solid-jobs 0.5.0 → 0.6.0

data/lib/solid_jobs/backends/base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Backends
+    class NotSupported < Error; end
+
+    # Contract shared by every storage/execution backend. SolidJobs owns the
+    # job API; adapters translate these operations to their engine.
+    #
+    # A "claim" is whatever #claim returns (an opaque reservation holder that
+    # exposes #id, #channel, #envelope and #attempt). It carries the
+    # reservation token / fencing token, so #ack and #retry_job take the
+    # claim instead of a bare job id.
+    #
+    # Delivery is at-least-once on every backend. Differences in guarantees
+    # are exposed through #capabilities and #supports?; operations a backend
+    # cannot honor raise NotSupported instead of silently degrading.
+    class Base
+      CAPABILITIES = %i[
+        scheduling fencing_tokens lease_extension get_job cancel deduplication
+        crash_recovery server_side_retries
+      ].freeze
+
+      # Persists one job. Returns { id:, duplicate: }.
+      def enqueue(_envelope, **_options) = raise(NotImplementedError)
+
+      # Persists a job that becomes ready at `at` (Unix seconds).
+      def schedule(_envelope, at:, **_options) = raise(NotImplementedError)
+
+      def enqueue_many(envelopes)
+        envelopes.each { |envelope| envelope.key?("run_at") ? schedule(envelope, at: envelope["run_at"]) : enqueue(envelope) }
+        []
+      end
+
+      # Reserves the next ready job of `channel`, or returns nil.
+      def claim(_channel) = raise(NotImplementedError)
+
+      def ack(_claim) = raise(NotImplementedError)
+
+      # Re-queues a claimed job after `delay` seconds (nil: backend policy).
+      def retry_job(_claim, delay: nil) = raise(NotImplementedError)
+
+      # Moves a claimed job to the dead state (no further retries).
+      def fail_job(_claim) = raise(NotSupported, "#{self.class.name} cannot fail jobs natively")
+
+      def get_job(_job_id) = raise(NotSupported, "#{self.class.name} cannot look up jobs by id")
+
+      # Reclaims jobs owned by dead workers; returns how many were recovered.
+      def recover(**_options) = 0
+
+      def capabilities = []
+
+      def supports?(capability) = capabilities.include?(capability)
+
+      def close = self
+    end
+  end
+end

data/lib/solid_jobs/backends/redis.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  module Backends
+    # SolidRedis adapter: the original Redis-backed storage, behind the
+    # backend contract. Publishing goes through the same pipeline as before.
+    class Redis < Base
+      CAPABILITIES = %i[scheduling crash_recovery deduplication].freeze
+
+      attr_reader :config
+
+      # Pass `url:` through the backend spec to target a different Redis
+      # server than the default configuration.
+      def initialize(config:, owned: false)
+        super()
+        @config = config
+        @owned = owned
+        @claimers = {}
+      end
+
+      def capabilities = CAPABILITIES
+
+      def enqueue(envelope, **_options)
+        envelope = Utilities.stringify_hash_keys(envelope)
+        envelope["id"] ||= SecureRandom.uuid
+        enqueue_many([envelope])
+        { id: envelope["id"], duplicate: false }
+      end
+
+      def schedule(envelope, at:, **options)
+        enqueue(Utilities.stringify_hash_keys(envelope).merge("run_at" => Float(at)), **options)
+      end
+
+      def enqueue_many(envelopes)
+        publisher = Publisher.new(config: @config)
+        barriers = publisher.persist_redis(envelopes)
+        PublicationBarrier.release(@config.redis_pool, barriers)
+        []
+      end
+
+      def claim(channel)
+        claimer(channel).next
+      end
+
+      def ack(claim)
+        claim.complete
+      end
+
+      def retry_job(claim, delay: nil)
+        delay ||= 0
+        payload = claim.envelope.merge("channel" => claim.channel)
+        @config.redis_pool.call("ZADD", Keyspace::RETRIES, Time.now.to_f + delay, JSON.generate(payload))
+        claim.complete
+      end
+
+      def recover(**)
+        Recovery.call(config: @config)
+      end
+
+      def close
+        @claimers.each_value { |(cfg, _)| cfg.close }
+        @claimers.clear
+        @config.close if @owned
+        self
+      end
+
+      private
+
+      def claimer(channel)
+        @claimers[channel.to_s] ||= begin
+          scoped = Blueprint.new(redis: @config.redis_config, concurrency: 1, channels: [channel.to_s])
+          [scoped, Claim.new(scoped, identity: @config.identity, processor_id: "backend-#{channel}")]
+        end
+        @claimers[channel.to_s].last
+      end
+    end
+  end
+end

data/lib/solid_jobs/backends/solid_engine.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "socket"
+require "uri"
+require "zlib"
+
+module SolidJobs
+  module Backends
+    # Native client and backend adapter for the SolidEngine storage engine
+    # (binary protocol v1 over Unix sockets or TCP).
+    #
+    #   SolidJobs.config.backend = :solid_engine
+    #   SolidJobs.config.url = "unix:///var/run/solidengine.sock"
+    class SolidEngine < Base
+      MAGIC = 0x474E4553
+      VERSION = 1
+      HEADER_LEN = 20
+      RESPONSE_BIT = 0x80
+      ERROR = 0xFF
+      DELAY_FROM_POLICY = 0xFFFF_FFFF_FFFF_FFFF
+
+      PING = 1
+      PUSH_JOB = 2
+      CLAIM_NEXT_JOB = 3
+      ACK_JOB = 4
+      NACK_JOB = 5
+      EXTEND_LEASE = 6
+      GET_JOB = 7
+      CANCEL_JOB = 8
+      WAIT_FOR_JOB = 9
+      GET_QUEUE_STATS = 10
+
+      ERROR_CODES = {
+        1 => :bad_request, 2 => :not_found, 3 => :stale_reservation, 4 => :invalid_state,
+        5 => :overloaded, 6 => :payload_too_large, 7 => :internal, 8 => :shutting_down,
+      }.freeze
+      JOB_STATES = %i[ready claimed completed scheduled dead cancelled].freeze
+      STATS_FIELDS = %i[
+        ready claimed scheduled completed dead cancelled pushed claims acks nacks lease_expirations
+      ].freeze
+
+      class Error < StandardError
+        attr_reader :code
+
+        def initialize(code, message)
+          @code = code
+          super("#{code}: #{message}")
+        end
+      end
+
+      class ProtocolError < StandardError; end
+
+      CAPABILITIES = %i[
+        scheduling fencing_tokens lease_extension get_job cancel deduplication
+        crash_recovery server_side_retries
+      ].freeze
+
+      Lease = Struct.new(:job_id, :reservation_id, :fencing_token, :lease_deadline, :attempts, :payload, :envelope,
+                         keyword_init: true) do
+        def id = envelope["id"]
+        def channel = envelope["channel"]
+        def attempt = attempts
+      end
+
+      attr_reader :url, :lease_ms
+
+      def initialize(url:, channel_ids: {}, lease_ms: 30_000, timeout: 5.0)
+        super()
+        @url = URI(url)
+        @channel_ids = channel_ids.transform_keys(&:to_s)
+        @lease_ms = Integer(lease_ms)
+        @timeout = timeout
+        @mutex = Mutex.new
+        @socket = nil
+        @request_id = 0
+      end
+
+      def ping
+        call(PING)
+        true
+      end
+
+      def enqueue(envelope, run_at: nil, priority: 0, max_attempts: 0, partition_key: "")
+        envelope = envelope.transform_keys(&:to_s)
+        id = (envelope["id"] ||= SecureRandom.uuid)
+        run_at ||= envelope.delete("run_at")
+        delay_ms = run_at ? [((run_at.to_f - Time.now.to_f) * 1000).ceil, 0].max : 0
+        body = pack_u128(uuid_to_int(id))
+        body << [channel_id(envelope.fetch("channel", "default")), priority, max_attempts, delay_ms].pack("S<CL<Q<")
+        body << pack_bytes(partition_key.to_s) << pack_bytes(JSON.generate(envelope))
+        _type, reply = call(PUSH_JOB, body)
+        { id: id, duplicate: reply.getbyte(16) != 0 }
+      end
+
+      def schedule(envelope, at: nil, run_at: nil, **options)
+        enqueue(envelope, run_at: at || run_at, **options)
+      end
+
+      def enqueue_many(envelopes)
+        envelopes.each do |envelope|
+          failures = envelope["max_failures"]
+          bounded = failures && !envelope["retry_within"]
+          enqueue(envelope, max_attempts: bounded ? Integer(failures) + 1 : 0)
+        end
+        []
+      end
+
+      def capabilities = CAPABILITIES
+
+      def claim(channel = "default", lease_ms: @lease_ms, wait_ms: 0)
+        body = [channel_id(channel), lease_ms, wait_ms].pack("S<L<L<")
+        _type, reply = call(CLAIM_NEXT_JOB, body, read_timeout: (wait_ms / 1000.0) + @timeout)
+        return nil if reply.getbyte(0) == 0
+
+        job_id = unpack_u128(reply, 1)
+        reservation_id, fencing, deadline, attempts, length = reply.byteslice(17, 32).unpack("Q<Q<Q<L<L<")
+        payload = reply.byteslice(49, length)
+        Lease.new(job_id: job_id, reservation_id: reservation_id, fencing_token: fencing,
+                  lease_deadline: deadline, attempts: attempts, payload: payload,
+                  envelope: JSON.parse(payload))
+      end
+
+      def ack(lease)
+        call(ACK_JOB, reservation(lease))
+        true
+      end
+
+      # Retries the job; delay: nil applies the engine's backoff policy.
+      def nack(lease, delay: nil, fail: false)
+        flags = fail ? 1 : 0
+        delay_ms = if fail then 0 elsif delay.nil? then DELAY_FROM_POLICY else (delay * 1000).ceil end
+        call(NACK_JOB, reservation(lease) + [flags, delay_ms].pack("CQ<"))
+        true
+      end
+
+      def fail_job(lease)
+        nack(lease, fail: true)
+      end
+
+      def retry_job(lease, delay: nil)
+        nack(lease, delay: delay)
+      end
+
+      # The engine reclaims expired leases itself; nothing to do client-side.
+      def recover(**) = 0
+
+      def extend_lease(lease, lease_ms: @lease_ms)
+        _type, reply = call(EXTEND_LEASE, reservation(lease) + [lease_ms].pack("L<"))
+        lease.lease_deadline = reply.unpack1("Q<")
+      end
+
+      def cancel(job_id)
+        call(CANCEL_JOB, pack_u128(uuid_to_int(job_id)))
+        true
+      end
+
+      def get_job(job_id) = job(job_id)
+
+      def job(job_id)
+        _type, reply = call(GET_JOB, pack_u128(uuid_to_int(job_id)))
+        channel, state, priority, attempts, max_attempts, fencing, deadline, scheduled_at, created_at, length =
+          reply.byteslice(16, reply.bytesize - 16).unpack("S<CCL<L<Q<Q<Q<Q<L<")
+        { channel_id: channel, state: JOB_STATES.fetch(state), priority: priority, attempts: attempts,
+          max_attempts: max_attempts, fencing_token: fencing, lease_deadline: deadline,
+          scheduled_at: scheduled_at, created_at: created_at, payload_length: length }
+      end
+
+      def stats(channel: nil)
+        body = channel ? [1, channel_id(channel)].pack("CS<") : [0, 0].pack("CS<")
+        _type, reply = call(GET_QUEUE_STATS, body)
+        STATS_FIELDS.zip(reply.unpack("Q<11")).to_h
+      end
+
+      def close
+        @mutex.synchronize do
+          @socket&.close
+          @socket = nil
+        end
+      end
+
+      def channel_id(name)
+        @channel_ids.fetch(name.to_s) { (Zlib.crc32(name.to_s) % 65_535) + 1 }
+      end
+
+      def uuid_to_int(id)
+        hex = id.to_s.delete("-")
+        raise ArgumentError, "job id must be a UUID: #{id.inspect}" unless hex.match?(/\A\h{32}\z/)
+
+        hex.to_i(16)
+      end
+
+      def int_to_uuid(value)
+        hex = value.to_s(16).rjust(32, "0")
+        "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
+      end
+
+      private
+
+      def reservation(lease)
+        pack_u128(lease.job_id) + [lease.reservation_id, lease.fencing_token].pack("Q<Q<")
+      end
+
+      def pack_u128(value)
+        [value & 0xFFFF_FFFF_FFFF_FFFF, value >> 64].pack("Q<Q<")
+      end
+
+      def unpack_u128(data, offset)
+        low, high = data.byteslice(offset, 16).unpack("Q<Q<")
+        (high << 64) | low
+      end
+
+      def pack_bytes(string)
+        string = string.b
+        [string.bytesize].pack("L<") + string
+      end
+
+      def call(type, body = "".b, read_timeout: @timeout)
+        @mutex.synchronize do
+          attempts = 0
+          begin
+            socket = (@socket ||= connect)
+            @request_id += 1
+            write_frame(socket, type, @request_id, body.b)
+            read_response(socket, type, @request_id, read_timeout)
+          rescue Errno::EPIPE, Errno::ECONNRESET, EOFError, IOError
+            @socket&.close
+            @socket = nil
+            attempts += 1
+            # Only idempotent reads are replayed after a lost connection.
+            retry if attempts < 2 && [PING, GET_JOB, GET_QUEUE_STATS].include?(type)
+            raise
+          end
+        end
+      end
+
+      def connect
+        case @url.scheme
+        when "unix" then UNIXSocket.new(@url.path)
+        when "tcp" then TCPSocket.new(@url.host, @url.port).tap { |s| s.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) }
+        else raise ArgumentError, "unsupported SolidEngine url scheme: #{@url.scheme.inspect}"
+        end
+      end
+
+      def write_frame(socket, type, request_id, body)
+        header = [MAGIC, VERSION, type, 0, request_id, body.bytesize].pack("L<CCS<Q<L<")
+        crc = Zlib.crc32(body, Zlib.crc32(header))
+        socket.write(header + body + [crc].pack("L<"))
+      end
+
+      def read_response(socket, request_type, request_id, timeout)
+        header = read_exact(socket, HEADER_LEN, timeout)
+        magic, version, type, _flags, id, length = header.unpack("L<CCS<Q<L<")
+        raise ProtocolError, "bad magic" unless magic == MAGIC
+        raise ProtocolError, "unsupported version #{version}" unless version == VERSION
+
+        body = read_exact(socket, length, timeout)
+        crc = read_exact(socket, 4, timeout).unpack1("L<")
+        raise ProtocolError, "checksum mismatch" unless Zlib.crc32(body, Zlib.crc32(header)) == crc
+        raise ProtocolError, "unexpected request id #{id}" unless id == request_id
+
+        if type == ERROR
+          code = ERROR_CODES.fetch(body.unpack1("S<"), :internal)
+          raise Error.new(code, body.byteslice(6, body.byteslice(2, 4).unpack1("L<")))
+        end
+        raise ProtocolError, "unexpected response type #{type}" unless type == (request_type | RESPONSE_BIT)
+
+        [type, body]
+      end
+
+      def read_exact(socket, count, timeout)
+        data = "".b
+        while data.bytesize < count
+          unless socket.wait_readable(timeout)
+            @socket&.close
+            @socket = nil
+            raise ProtocolError, "read timeout"
+          end
+          chunk = socket.read_nonblock(count - data.bytesize, exception: false)
+          raise EOFError if chunk.nil?
+          data << chunk unless chunk == :wait_readable
+        end
+        data
+      end
+    end
+  end
+end

data/lib/solid_jobs/backends/worker.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  module Backends
+    # Claim/execute/ack loop for channels served by a non-Redis backend
+    # (SolidEngine). One instance per thread; it owns its backend connection.
+    class Worker
+      IDLE_SLEEP = 0.02
+
+      attr_reader :processed, :failed
+
+      def initialize(config:, backend_name:, channels:, processor_id: 0)
+        @config = config
+        @backend = config.build_backend(backend_name)
+        @channels = channels.flat_map { |name, weight| Array.new(weight, name) }
+        @processor_id = processor_id
+        @executor = Executor.new(redis_config: config.redis_config, config: config)
+        @processed = 0
+        @failed = 0
+        @cursor = 0
+      end
+
+      def run(stop:, paused: -> { false })
+        until stop.call
+          if paused.call
+            sleep 0.05
+            next
+          end
+
+          claim = next_claim
+          if claim
+            perform(claim)
+          else
+            sleep IDLE_SLEEP
+          end
+        end
+        self
+      ensure
+        @backend.close
+      end
+
+      private
+
+      def next_claim
+        @channels.length.times do
+          channel = @channels[@cursor % @channels.length]
+          @cursor += 1
+          claim = @backend.claim(channel)
+          return claim if claim
+        end
+        nil
+      rescue SystemCallError, IOError, EOFError, Backends::SolidEngine::ProtocolError => error
+        @config.logger.warn("SolidJobs backend claim failed, retrying: #{error.class}: #{error.message}")
+        sleep 0.1
+        nil
+      end
+
+      def perform(claim)
+        envelope = claim.envelope
+        attributes = {reservation_id: claim.reservation_id, attempt: claim.attempt,
+                      **Instrumentation.worker_attributes(@processor_id)}
+        Instrumentation.emit(@config, :reserved, envelope, **attributes)
+        Instrumentation.emit(@config, :started, envelope, **attributes)
+        started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        keepalive = start_keepalive(claim)
+        begin
+          @executor.execute(envelope)
+        rescue ExecutionHalt
+          raise
+        rescue StandardError => error
+          duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started
+          Instrumentation.emit(@config, :failed, envelope, **attributes, duration: duration,
+                               error_class: error.class.name, error_message: error.message.to_s)
+          handle_failure(claim, envelope, error, attributes)
+          @failed += 1
+        else
+          duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started
+          Instrumentation.emit(@config, :completed, envelope, **attributes, duration: duration)
+          settle(claim) { @backend.ack(claim) }
+          Instrumentation.emit(@config, :acknowledged, envelope, **attributes)
+          @processed += 1
+        end
+      ensure
+        stop_keepalive(keepalive)
+      end
+
+      # Mirrors FailurePolicy: retry with backoff while allowed, then discard.
+      def handle_failure(claim, envelope, error, attributes)
+        failure_count = claim.attempt
+        task = resolve_task(envelope)
+        decision = task.respond_to?(:retry_delay_callback) ? task.retry_delay_callback(failure_count, error, envelope) : nil
+        if retry_allowed?(envelope, claim.attempt) && !%i[drop archive].include?(decision)
+          delay = decision.is_a?(Numeric) ? decision : backoff(failure_count)
+          settle(claim) { @backend.retry_job(claim, delay: delay) }
+          Instrumentation.emit(@config, :retry_scheduled, envelope, **attributes,
+                               error_class: error.class.name, error_message: error.message.to_s)
+        else
+          task.final_failure_callback(envelope, error) if decision != :drop && task.respond_to?(:final_failure_callback)
+          settle(claim) { @backend.fail_job(claim) }
+          Instrumentation.emit(@config, :dead, envelope, **attributes,
+                               error_class: error.class.name, error_message: error.message.to_s)
+        end
+      end
+
+      def retry_allowed?(envelope, attempt)
+        if envelope["retry_within"]
+          created = Float(envelope.fetch("created_ms", Utilities.realtime_milliseconds)) / 1_000
+          Time.now.to_f - created < Float(envelope["retry_within"])
+        else
+          attempt <= Integer(envelope.fetch("max_failures", 0))
+        end
+      end
+
+      def backoff(failure_count)
+        ceiling = [Float(@config.retry_base_delay) * (2**(failure_count - 1)), Float(@config.retry_max_delay)].min
+        (ceiling / 2) + (rand * ceiling / 2)
+      end
+
+      def resolve_task(envelope)
+        Utilities.constantize(envelope.fetch("task"))
+      rescue NameError
+        nil
+      end
+
+      # A stale reservation means the job was reclaimed: fencing did its job.
+      def settle(claim)
+        yield
+      rescue Backends::SolidEngine::Error => error
+        raise unless error.code == :stale_reservation
+
+        @config.logger.warn("Claim fencing rejected stale settlement: #{claim.id}")
+      rescue SystemCallError, IOError, EOFError, Backends::SolidEngine::ProtocolError => error
+        # Settlement unknown: leave the lease to expire so the job is redelivered (at-least-once).
+        @config.logger.warn("SolidJobs settlement failed, job will be redelivered: #{claim.id} #{error.class}: #{error.message}")
+      end
+
+      def start_keepalive(claim)
+        return unless @backend.supports?(:lease_extension)
+
+        interval = @backend.lease_ms / 3000.0
+        done = Queue.new
+        thread = Thread.new do
+          until done.pop(timeout: interval)
+            begin
+              @backend.extend_lease(claim)
+            rescue StandardError
+              break
+            end
+          end
+        end
+        [thread, done]
+      end
+
+      # Joined rather than killed: a killed thread could leave an unread reply on the socket.
+      def stop_keepalive(keepalive)
+        return unless keepalive
+
+        thread, done = keepalive
+        done << true
+        thread.join
+      end
+    end
+  end
+end