wurk 0.0.1

data/lib/wurk/fetcher/reaper.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require_relative '../component'
+require_relative '../keys'
+require_relative '../middleware/poison_pill'
+
+module Wurk
+  class Fetcher
+    # Orphan reclamation for the reliable fetcher (Pro super_fetch §3.2).
+    #
+    # The Reliable fetcher moves each job from a public queue into a
+    # per-process private list (`queue:<public>|<host>|<pid>|<idx>`) and
+    # leaves it there until the Processor ACKs. A SIGKILLed or crashed
+    # worker therefore strands its in-flight jobs in private lists that
+    # nobody will ever ACK. The Reaper is the recovery half: it periodically
+    # scans for private lists whose owning process is gone and atomically
+    # moves their jobs back to the public queue so a live worker re-runs them.
+    #
+    # Liveness is decided per owner:
+    #   * same host — the OS is authoritative: `Process.kill(0, pid)`. This
+    #     is instant and ignores a stale `processes` SET entry whose 60s TTL
+    #     hasn't lapsed yet, so a `kill -9`ed sibling is reclaimed the moment
+    #     the supervisor reaps it rather than 60s later. (Pid reuse by an
+    #     unrelated local process is the one blind spot — the supervisor
+    #     respawns with a fresh pid, so it does not arise in practice.)
+    #   * other host — we cannot ping the pid, so we trust the heartbeat:
+    #     the owner is alive iff some live `processes` member (one whose
+    #     `info` hash still exists) shares its `host:pid`. Cross-host reclaim
+    #     therefore waits out the 60s heartbeat TTL, exactly as the spec says.
+    #
+    # Re-pushed jobs run through Wurk::Middleware::PoisonPill, which caps a
+    # job at RECOVERY_THRESHOLD recoveries within 72h: past the cap the job
+    # is killed into the dead set instead of re-queued, so a job that crashes
+    # its worker every time can't loop forever.
+    #
+    # SCANs are scoped to the public queues this process serves and gated by
+    # a cluster-wide `SET NX EX` lock, so across a fleet only one process
+    # sweeps per interval ("1/min within process group" in the spec) and the
+    # keyspace touched is bounded to known queues.
+    #
+    # Spec: docs/target/sidekiq-pro.md §3.2.
+    class Reaper
+      include Component
+
+      # Sweep cadence in seconds; also the cluster-lock TTL so exactly one
+      # process sweeps per interval. 60s matches the heartbeat TTL — the
+      # floor below which cross-host orphans can't be detected anyway.
+      DEFAULT_INTERVAL = 60
+
+      LOCK_KEY = 'super_fetch:reaper'
+      SCAN_COUNT = 100
+      THREAD_NAME = 'wurk-reaper'
+
+      attr_reader :interval
+
+      def initialize(config, interval: DEFAULT_INTERVAL, lock_key: LOCK_KEY)
+        @config = config
+        @interval = interval
+        @lock_key = lock_key
+        @thread = nil
+        @done = false
+        @mutex = ::Mutex.new
+        @sleeper = ::ConditionVariable.new
+      end
+
+      # Spawns the sweep loop. Idempotent. The loop waits one interval before
+      # its first sweep so booting processes don't dogpile Redis and so an
+      # un-stopped launcher in a unit test never touches the keyspace.
+      def start
+        @mutex.synchronize do
+          return @thread if @thread
+
+          @done = false
+          @thread = spawn_loop_thread
+        end
+        @thread
+      end
+
+      def stop
+        @mutex.synchronize do
+          @done = true
+          @sleeper.signal
+        end
+        @thread&.join
+        @thread = nil
+      end
+
+      def running?
+        !@thread.nil? && @thread.alive?
+      end
+
+      # One cluster-gated sweep: a no-op (returns 0) unless this process wins
+      # the interval's lock. Used by the loop.
+      def reap
+        return 0 unless acquire_lock?
+
+        reclaim!
+      end
+
+      # One unguarded sweep over every served queue. Returns the number of
+      # jobs reclaimed (re-queued or killed). Public so boot paths and tests
+      # can drive a deterministic pass without the cluster lock.
+      def reclaim!
+        prefixes = live_process_prefixes
+        served_queues.sum { |public_q| reclaim_queue(public_q, prefixes) }
+      end
+
+      private
+
+      # Union of `queue:<name>` keys across every capsule this process serves.
+      # Scoping the scan to these keeps the keyspace we touch bounded and lets
+      # parallel test namespaces stay isolated.
+      def served_queues
+        @config.capsules.each_value
+               .flat_map(&:queues)
+               .uniq
+               .map { |name| Keys.queue(name) }
+      end
+
+      # SCAN for this public queue's private lists, reclaim the orphaned ones.
+      def reclaim_queue(public_q, prefixes)
+        reclaimed = 0
+        each_private_list(public_q) do |key, host, pid|
+          next if owner_alive?(host, pid, prefixes)
+
+          reclaimed += drain(key, public_q)
+        end
+        reclaimed
+      end
+
+      # Yields [private_list_key, host, pid] for each private list of
+      # `public_q`. MATCH `<public_q>|*` matches only this queue's private
+      # lists (public queue keys carry no `|`).
+      def each_private_list(public_q)
+        cursor = '0'
+        loop do
+          cursor, keys = redis { |c| c.call('SCAN', cursor, 'MATCH', "#{public_q}|*", 'COUNT', SCAN_COUNT) }
+          keys.each do |key|
+            host, pid = parse_owner(public_q, key)
+            yield key, host, pid if pid
+          end
+          break if cursor == '0'
+        end
+      end
+
+      # `<public_q>|<host>|<pid>|<idx>` → [host, pid] (pid as Integer), or
+      # [nil, nil] when the suffix isn't a well-formed `host|pid|idx` triple.
+      # Splitting the suffix off the known public-queue prefix tolerates a
+      # `|` inside the queue name itself.
+      def parse_owner(public_q, key)
+        suffix = key.delete_prefix("#{public_q}|")
+        return [nil, nil] if suffix == key
+
+        host, pid, idx = suffix.split('|')
+        return [nil, nil] unless host && integer?(pid) && integer?(idx)
+
+        [host, pid.to_i]
+      end
+
+      def integer?(str)
+        str.is_a?(String) && str.match?(/\A\d+\z/)
+      end
+
+      def owner_alive?(host, pid, prefixes)
+        return local_pid_alive?(pid) if host == hostname
+
+        prefixes.include?("#{host}:#{pid}")
+      end
+
+      def local_pid_alive?(pid)
+        ::Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+
+      # `host:pid` of every live process — a member of `processes` whose
+      # `info` hash still exists. A bare SET membership isn't enough: the
+      # member lingers after its 60s hash TTL until ProcessSet#cleanup prunes
+      # it, and we must treat that window as dead for cross-host reclaim.
+      def live_process_prefixes
+        redis do |conn|
+          members = conn.call('SMEMBERS', Keys::PROCESSES)
+          next ::Set.new if members.empty?
+
+          infos = conn.pipelined { |pipe| members.each { |m| pipe.call('HGET', m, 'info') } }
+          members.zip(infos).each_with_object(::Set.new) do |(member, info), set|
+            set << host_pid(member) if info
+          end
+        end
+      end
+
+      # identity is `<host>:<pid>:<nonce>`; the owner prefix is `<host>:<pid>`.
+      def host_pid(identity)
+        identity.split(':')[0..1].join(':')
+      end
+
+      # Drain one orphaned private list back to its public queue. Each job is
+      # moved with an atomic LMOVE (private tail → public tail) BEFORE the
+      # poison check, so a crash mid-drain leaves the job safely in the public
+      # queue (at-least-once), never lost. Poison jobs are killed to the dead
+      # set by PoisonPill.track! and then LREM'd out of the public queue.
+      def drain(private_list, public_q)
+        queue_name = public_q.delete_prefix(Keys::QUEUE_PREFIX)
+        count = 0
+        loop do
+          job = redis { |c| c.call('LMOVE', private_list, public_q, 'RIGHT', 'RIGHT') }
+          break unless job
+
+          count += 1
+          poison_off(public_q, job, queue_name)
+        end
+        count
+      rescue StandardError => e
+        handle_exception(e, context: THREAD_NAME)
+        count
+      end
+
+      def poison_off(public_q, job, queue_name)
+        return unless Middleware::PoisonPill.track!(job, queue: queue_name) == :poison
+
+        # track! already ZADDed the payload to the dead set; pull the copy we
+        # just LMOVE'd onto the public tail so it isn't also re-run.
+        redis { |c| c.call('LREM', public_q, -1, job) }
+      end
+
+      def acquire_lock?
+        redis { |c| c.call('SET', @lock_key, '1', 'NX', 'EX', @interval) } == 'OK'
+      end
+
+      def spawn_loop_thread
+        t = Thread.new { run_loop }
+        t.name = THREAD_NAME
+        t.report_on_exception = false
+        t
+      end
+
+      def run_loop
+        until done?
+          wait_next
+          break if done?
+
+          tick_once
+        end
+      end
+
+      def tick_once
+        reap
+      rescue StandardError => e
+        handle_exception(e, context: THREAD_NAME) if @config.respond_to?(:handle_exception)
+      end
+
+      def wait_next
+        @mutex.synchronize { @sleeper.wait(@mutex, @interval) unless @done }
+      end
+
+      def done?
+        @mutex.synchronize { @done }
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/reliable.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'socket'
+require_relative '../component'
+require_relative '../keys'
+require_relative '../fetcher'
+
+module Wurk
+  class Fetcher
+    # Default fetcher. Each public queue is paired with a per-process
+    # private list (`queue:<name>|<host>|<pid>|<idx>`); a job is moved
+    # atomically from the public tail to the private head via LMOVE, and
+    # stays there until the Processor explicitly ACKs (LREM). SIGKILL
+    # between fetch and ack leaves the job in the private list, where the
+    # next boot of this process reclaims it via bulk_requeue.
+    #
+    # Priority handling: iterate queues_cmd in order with non-blocking
+    # LMOVE, then fall back to a 2s BLMOVE on the first queue so an
+    # empty poll doesn't spin Redis. BLMOVE has no multi-key form, so
+    # blocking on a single queue is the best Redis gives us.
+    #
+    # Spec: docs/target/sidekiq-pro.md §3 (super_fetch),
+    # docs/target/sidekiq-free.md §15 (TIMEOUT=2).
+    class Reliable < Fetcher
+      include Component
+
+      TIMEOUT = 2
+
+      # Carries the public queue key, the raw (still-JSON) job payload,
+      # and the capsule we use to reach Redis. ACK removes from the private
+      # list; requeue pushes back to the public queue head so the job is
+      # next pulled. LREM count=1 is idempotent for our payloads since
+      # each job's JSON contains a unique `jid`.
+      UnitOfWork = Struct.new(:queue, :job, :config, keyword_init: true) do
+        def acknowledge
+          config.redis do |conn|
+            conn.call('LREM', Reliable.private_queue_name(queue), 1, job)
+          end
+        end
+
+        def queue_name
+          queue.delete_prefix(Keys::QUEUE_PREFIX)
+        end
+
+        def requeue
+          config.redis { |conn| conn.call('RPUSH', queue, job) }
+        end
+      end
+
+      # Class-level so UnitOfWork can compute the private list without
+      # carrying a back-reference to its parent fetcher. Index defaults to
+      # 0 — we run one fetcher per capsule today. Multi-processor topology
+      # (one private list per processor slot) is a future Manager concern.
+      def self.private_queue_name(public_queue, index = 0)
+        host = ENV['DYNO'] || Socket.gethostname
+        "#{public_queue}|#{host}|#{::Process.pid}|#{index}"
+      end
+
+      def initialize(capsule)
+        super()
+        @config = capsule
+        @done = false
+      end
+
+      def retrieve_work
+        return nil if @done
+
+        queues = queues_cmd
+        return nil if queues.empty?
+
+        queues.each do |public_q|
+          uow = lmove(public_q)
+          return uow if uow
+        end
+        blmove(queues.first)
+      end
+
+      # Called on shutdown for jobs the Processor couldn't finish in time.
+      # One pipelined RPUSH per public queue (head insert) so on next boot
+      # they're picked again ahead of fresh enqueues.
+      def bulk_requeue(in_progress)
+        return if in_progress.nil? || in_progress.empty?
+
+        grouped = in_progress.group_by(&:queue)
+        config.redis do |conn|
+          conn.pipelined do |pipe|
+            grouped.each do |public_q, uows|
+              pipe.call('RPUSH', public_q, *uows.map(&:job))
+            end
+          end
+        end
+      end
+
+      # Prefixed queue keys (`queue:<name>`) in fetch order. Strict mode
+      # preserves declaration order. Random/weighted shuffle each call —
+      # @queues is pre-expanded by weight in Capsule#queues=, so uniform
+      # shuffle yields weighted fairness; .uniq trims duplicates. Paused
+      # queues are filtered after shuffle so the membership test runs on
+      # the smallest possible set.
+      def queues_cmd
+        names = config.mode == :strict ? config.queues : config.queues.shuffle.uniq
+        paused = paused_names
+        names = names.reject { |q| paused.include?(q) } unless paused.empty?
+        names.map { |q| "#{Keys::QUEUE_PREFIX}#{q}" }
+      end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      # SMEMBERS of the `paused` SET. One round-trip per fetch pass; the
+      # set is tiny in practice (one entry per paused queue) so the cost
+      # is dominated by the BLMOVE that follows. Returns a Set for O(1)
+      # lookup against the (often weighted-expanded) queue list.
+      def paused_names
+        config.redis { |conn| conn.call('SMEMBERS', Keys::PAUSED_SET) }.to_set
+      end
+
+      def lmove(public_q)
+        priv = self.class.private_queue_name(public_q)
+        job = config.redis { |conn| conn.call('LMOVE', public_q, priv, 'RIGHT', 'LEFT') }
+        job ? UnitOfWork.new(queue: public_q, job: job, config: config) : nil
+      end
+
+      def blmove(public_q)
+        priv = self.class.private_queue_name(public_q)
+        # Extend the socket read-timeout past BLMOVE's own timeout so the
+        # default 1s pool timeout doesn't fire before BLMOVE returns.
+        job = config.redis do |conn|
+          conn.blocking_call(TIMEOUT + 1, 'BLMOVE', public_q, priv, 'RIGHT', 'LEFT', TIMEOUT)
+        end
+        job ? UnitOfWork.new(queue: public_q, job: job, config: config) : nil
+      end
+    end
+  end
+end

data/lib/wurk/fetcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wurk
+  # Abstract fetcher. Wurk::Fetcher::Reliable is the only implementation
+  # we ship and the only one we recommend — BLMOVE-based reliable fetch.
+  # No "basic fetch" mode.
+  class Fetcher
+    def retrieve_work; end
+    def bulk_requeue(in_progress); end
+  end
+end

data/lib/wurk/health.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'json'
+
+module Wurk
+  # Thin HTTP listener for k8s liveness/readiness probes. Optional, off by
+  # default — opt in with `config.health_check(port: 7433)`.
+  #
+  # Endpoints:
+  #   * GET /live  → 200 while the Launcher is running (not in quiet/stop).
+  #   * GET /ready → 200 only when Redis is reachable AND the heartbeat has
+  #                  fired within `ready_window` seconds. 503 otherwise.
+  # Anything else returns 404 JSON.
+  #
+  # The server uses a raw TCPServer and one accept thread. No Rack, no
+  # dependencies — it lives inside every worker process where Rails may or
+  # may not exist (standalone CLI, Embedded, swarm child). Bound to a
+  # dedicated port so it does not collide with the host application's HTTP.
+  #
+  # Spec: docs/target/sidekiq-ent.md §7.1.2 (`config.health_check`).
+  module Health
+    DEFAULT_PORT          = 7433
+    DEFAULT_BIND          = '0.0.0.0'
+    DEFAULT_READY_WINDOW  = 30
+
+    # The HTTP listener. Owns one TCPServer + one accept thread. Idempotent
+    # start/stop; safe to call from Launcher#run / Launcher#stop.
+    class Server
+      ACCEPT_TIMEOUT = 0.2
+
+      attr_reader :port, :bind
+
+      def initialize(launcher, port: DEFAULT_PORT, bind: DEFAULT_BIND, ready_window: DEFAULT_READY_WINDOW)
+        @launcher     = launcher
+        @config       = launcher.instance_variable_get(:@config)
+        @port         = port
+        @bind         = bind
+        @ready_window = ready_window
+        @server       = nil
+        @thread       = nil
+        @done         = false
+      end
+
+      def start
+        @server = ::TCPServer.new(@bind, @port)
+        # Capture the OS-assigned port when caller passed 0 (test pattern,
+        # also lets the kernel pick a free port at boot).
+        @port = @server.addr[1]
+        @done = false
+        @thread = ::Thread.new { run }
+        @thread.name = 'wurk-health'
+        self
+      rescue ::Errno::EADDRINUSE => e
+        # Swarm children all try to bind the same port — only the first wins.
+        # Don't crash the worker; just log and skip.
+        logger&.warn { "Wurk::Health: port #{@port} in use; health server NOT started (#{e.message})" }
+        @server = nil
+        @thread = nil
+        self
+      end
+
+      def stop
+        @done = true
+        srv = @server
+        @server = nil
+        srv&.close
+        @thread&.join(2)
+        @thread = nil
+      end
+
+      def running?
+        @thread&.alive? == true
+      end
+
+      private
+
+      def run
+        until @done
+          ready = ::IO.select([@server], nil, nil, ACCEPT_TIMEOUT)
+          next unless ready
+
+          begin
+            client, _addr = @server.accept_nonblock(exception: false)
+            handle(client) if client
+          rescue ::IO::WaitReadable
+            next
+          rescue ::StandardError => e
+            logger&.error { "Wurk::Health accept: #{e.class}: #{e.message}" }
+            next
+          end
+        end
+      rescue ::IOError, ::Errno::EBADF
+        # Server was closed during shutdown — expected.
+      end
+
+      def handle(client)
+        return unless ::IO.select([client], nil, nil, 1.0)
+        request_line = client.gets("\r\n")
+        return if request_line.nil?
+
+        method, path, = request_line.strip.split(' ', 3)
+        # Drain remaining headers; ignore the body (probes don't send one).
+        # Wait for readability before each gets so a stalled client can't
+        # block the single accept thread mid-headers.
+        loop do
+          break unless ::IO.select([client], nil, nil, 1.0)
+          line = client.gets("\r\n")
+          break if line.nil? || line == "\r\n"
+        end
+
+        body, status = response_for(method, path)
+        write_response(client, status, body)
+      rescue ::StandardError => e
+        logger&.error { "Wurk::Health request: #{e.class}: #{e.message}" }
+      ensure
+        client&.close
+      end
+
+      def response_for(method, path)
+        return [json('error', message: 'method not allowed'), 405] unless method == 'GET'
+
+        case path
+        when '/live'  then live_response
+        when '/ready' then ready_response
+        else [json('error', message: 'not found', path: path), 404]
+        end
+      end
+
+      def live_response
+        if @launcher.stopping?
+          [json('down', check: 'live', reason: 'stopping'), 503]
+        else
+          [json('ok', check: 'live'), 200]
+        end
+      end
+
+      def ready_response
+        redis_ok = ping_redis
+        beat_fresh = heartbeat_fresh?
+
+        if redis_ok && beat_fresh
+          [json('ok', check: 'ready'), 200]
+        else
+          reason = !redis_ok ? 'redis unreachable' : 'heartbeat stale'
+          [json('down', check: 'ready', reason: reason), 503]
+        end
+      end
+
+      def ping_redis
+        @config.redis { |conn| conn.call('PING') } == 'PONG'
+      rescue ::StandardError
+        false
+      end
+
+      def heartbeat_fresh?
+        hb = @launcher.instance_variable_get(:@heartbeat)
+        return false unless hb&.respond_to?(:last_beat_at)
+
+        last = hb.last_beat_at
+        return false if last.nil?
+
+        (::Time.now.to_f - last) < @ready_window
+      end
+
+      def json(status, **extra)
+        ::JSON.generate({ status: status }.merge(extra))
+      end
+
+      def write_response(client, status, body)
+        reason = case status
+                 when 200 then 'OK'
+                 when 404 then 'Not Found'
+                 when 405 then 'Method Not Allowed'
+                 when 503 then 'Service Unavailable'
+                 else 'Status'
+                 end
+
+        client.write(
+          "HTTP/1.1 #{status} #{reason}\r\n" \
+          "Content-Type: application/json\r\n" \
+          "Content-Length: #{body.bytesize}\r\n" \
+          "Connection: close\r\n\r\n" \
+          "#{body}"
+        )
+      end
+
+      def logger
+        @config&.logger
+      end
+    end
+  end
+end