wurk 0.0.1

data/lib/wurk/processor.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require_relative 'component'
+require_relative 'context'
+require_relative 'job_logger'
+require_relative 'job_retry'
+require_relative 'keys'
+
+module Wurk
+  # Inside each Manager, N Processors run in parallel. Each owns one thread,
+  # pulls a UnitOfWork from the capsule's fetcher, parses the payload, walks
+  # the server middleware chain, invokes `perform`, then ACKs (removes the
+  # payload from the per-process private list).
+  #
+  # Shutdown is two-stage:
+  #   * `terminate` flips a flag; the run loop exits between jobs.
+  #   * `kill` additionally raises `Wurk::Shutdown` into the thread so an
+  #     in-flight perform unwinds. The current UoW is NOT acked, so the
+  #     payload survives in the private list and is reclaimed on next boot.
+  #
+  # Spec: docs/target/sidekiq-free.md §14.
+  class Processor
+    include Component
+
+    attr_reader :thread, :job, :capsule
+
+    def initialize(capsule, &callback)
+      @capsule = capsule
+      @config = capsule
+      @callback = callback
+      @done = false
+      @job = nil
+      @thread = nil
+      @reloader = capsule.config[:reloader] || proc { |&b| b.call }
+      @job_logger = (capsule.config[:job_logger] || JobLogger).new(capsule.config)
+      @retrier = JobRetry.new(capsule)
+    end
+
+    # Sidekiq surface — positional boolean to match the drop-in contract.
+    def terminate(wait = false) # rubocop:disable Style/OptionalBooleanParameter
+      @done = true
+      return if @thread.nil?
+
+      @thread.value if wait
+    end
+
+    # Hard-stop: flips the flag *and* unwinds the in-flight job by raising
+    # Wurk::Shutdown into the worker thread. The UoW is intentionally not
+    # acked — the payload remains in the private list and is reclaimed on
+    # next boot via Reliable#bulk_requeue.
+    def kill(wait = false) # rubocop:disable Style/OptionalBooleanParameter
+      @done = true
+      return if @thread.nil?
+
+      @thread.raise ::Wurk::Shutdown
+      @thread.value if wait
+    end
+
+    def stopping?
+      @done
+    end
+
+    def start
+      @thread ||= safe_thread("#{@capsule.name}/processor", &method(:run)) # rubocop:disable Naming/MemoizedInstanceVariableName
+    end
+
+    # Capsule doesn't define handle_exception (it's a Configuration method);
+    # override Component's delegation so error handlers fire.
+    def handle_exception(ex, ctx = {})
+      @capsule.config.handle_exception(ex, ctx)
+    end
+
+    # Single iteration: fetch one UoW, process it. Public so tests can drive
+    # the loop step-by-step without spawning a thread.
+    def process_one
+      @job = fetch
+      process(@job) if @job
+      @job = nil
+    end
+
+    # Thread-safe global counter. Heartbeat reads + resets these every beat
+    # to publish per-process stats.
+    class Counter
+      def initialize
+        @value = 0
+        @lock = ::Mutex.new
+      end
+
+      def incr(amount = 1)
+        @lock.synchronize { @value += amount }
+      end
+
+      def reset
+        @lock.synchronize do
+          val = @value
+          @value = 0
+          val
+        end
+      end
+    end
+
+    # tid → { queue:, payload:, run_at: } for every Processor currently
+    # running a job. Read by Heartbeat each beat to publish into Redis.
+    class SharedWorkState
+      def initialize
+        @work = {}
+        @lock = ::Mutex.new
+      end
+
+      def set(tid, hash)
+        @lock.synchronize { @work[tid] = hash }
+      end
+
+      def delete(tid)
+        @lock.synchronize { @work.delete(tid) }
+      end
+
+      def dup
+        @lock.synchronize { @work.dup }
+      end
+
+      def size
+        @lock.synchronize { @work.size }
+      end
+
+      def clear
+        @lock.synchronize { @work.clear }
+      end
+    end
+
+    PROCESSED  = Counter.new
+    FAILURE    = Counter.new
+    EXPIRED    = Counter.new
+    WORK_STATE = SharedWorkState.new
+
+    private
+
+    def run
+      process_one until @done
+      @callback&.call(self)
+    rescue Wurk::Shutdown
+      @callback&.call(self)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      handle_exception(e, { context: '!shutdown' })
+      @callback&.call(self)
+      raise
+    end
+
+    def fetch
+      @capsule.fetcher.retrieve_work
+    rescue Wurk::Shutdown
+      nil
+    rescue StandardError => e
+      handle_exception(e, { context: 'Error fetching job' })
+      sleep(1)
+      nil
+    end
+
+    def process(uow)
+      jobstr = uow.job
+      queue  = uow.queue_name
+
+      job_hash = parse_or_kill(jobstr, uow)
+      return if job_hash.nil?
+
+      ack = false
+      begin
+        Thread.handle_interrupt(Wurk::Shutdown => :never) do
+          dispatch(job_hash, queue, jobstr) do |instance|
+            Thread.handle_interrupt(Wurk::Shutdown => :immediate) do
+              execute_job(instance, job_hash, queue)
+            end
+          end
+          ack = true
+        end
+      rescue Wurk::JobRetry::Handled
+        # JobRetry::Skip (subclass) or Handled — retry layer / middleware has
+        # already booked the outcome; safe to ack.
+        ack = true
+      rescue Wurk::Shutdown
+        # Don't ack — UoW stays in private list and is reclaimed on reboot.
+      ensure
+        uow.acknowledge if ack
+      end
+    end
+
+    # Parse JSON; on failure ZADD the raw payload to the dead set and ack.
+    # Returns nil to signal "no further processing".
+    def parse_or_kill(jobstr, uow)
+      Wurk.load_json(jobstr)
+    rescue ::JSON::ParserError => e
+      handle_exception(e, { context: 'Invalid JSON', jobstr: jobstr })
+      now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
+      @capsule.redis do |conn|
+        conn.call('ZADD', Keys::DEAD, now.to_s, jobstr)
+      end
+      uow.acknowledge
+      nil
+    end
+
+    # Wraps the actual perform in the dispatch onion: logger.prepare →
+    # retrier.global → logger.call → stats → reloader → instantiate
+    # → retrier.local → (yield to caller for middleware + perform).
+    def dispatch(job_hash, queue, jobstr)
+      @job_logger.prepare(job_hash) do
+        @retrier.global(jobstr, queue) do
+          @job_logger.call(job_hash, queue) do
+            stats(jobstr, queue) do
+              @reloader.call do
+                klass = Object.const_get(job_hash['class'])
+                instance = klass.new
+                instance.jid = job_hash['jid']
+                instance.bid = job_hash['bid'] if instance.respond_to?(:bid=)
+                instance._context = self
+                @retrier.local(instance, jobstr, queue) do
+                  yield instance
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def execute_job(instance, job_hash, queue)
+      @capsule.server_middleware.invoke(instance, job_hash, queue) do
+        instance.perform(*job_hash['args'])
+      end
+    end
+
+    # Bookkeeping wrapper. Publishes the in-flight job to WORK_STATE so the
+    # Heartbeat can mirror it into Redis (`<identity>:work`), and increments
+    # PROCESSED/FAILURE counters around the inner block.
+    def stats(jobstr, queue)
+      WORK_STATE.set(tid, queue: queue, payload: jobstr, run_at: ::Time.now.to_i)
+      begin
+        yield
+      rescue Exception # rubocop:disable Lint/RescueException
+        FAILURE.incr
+        raise
+      ensure
+        PROCESSED.incr
+        WORK_STATE.delete(tid)
+      end
+    end
+  end
+end

data/lib/wurk/queue.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative 'job_record'
+
+module Wurk
+  # Inspects / iterates one named queue against the canonical Sidekiq
+  # schema: LIST at `queue:<name>`, membership in the `queues` SET.
+  # Wire-compat is sacred — every Redis call below matches OSS exactly.
+  #
+  # Pause/unpause is a Pro feature upstream; Wurk ships it free. Membership
+  # of the `paused` SET drives both `paused?` and the fetcher's queue filter
+  # (see `Wurk::Fetcher::Reliable#queues_cmd`). In-flight jobs continue to
+  # completion — pausing only stops new fetches.
+  #
+  # Spec: docs/target/sidekiq-free.md §19.2, sidekiq-pro.md §6.
+  class Queue
+    include Enumerable
+
+    # Page size for LRANGE traversal. Matches upstream so dashboards
+    # observing Redis traffic see the same pattern.
+    PAGE_SIZE = 50
+
+    attr_reader :name
+    alias id name
+
+    # @return [Array<Queue>] one per known queue, sorted by name.
+    def self.all
+      names = Wurk.redis { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
+      names.sort.map { |n| new(n) }
+    end
+
+    def initialize(name = 'default')
+      @name = name.to_s
+      @rname = Keys.queue(@name)
+    end
+
+    def size
+      Wurk.redis { |conn| conn.call('LLEN', @rname) }
+    end
+
+    # Seconds since the oldest job (tail of LIST) was enqueued. 0.0 when empty.
+    def latency
+      payload = Wurk.redis { |conn| conn.call('LRANGE', @rname, -1, -1).first }
+      return 0.0 if payload.nil?
+
+      JobRecord.latency_from(Wurk.load_json(payload)['enqueued_at'])
+    rescue ::JSON::ParserError
+      0.0
+    end
+
+    # True iff this queue's name is a member of the `paused` SET. Wurk
+    # implements the Pro contract for free; fetchers consult the same set.
+    def paused?
+      Wurk.redis { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
+    end
+
+    # Pause new fetches against this queue. Idempotent — `SADD` returns
+    # 0 when the name was already present. In-flight jobs are untouched.
+    def pause! # rubocop:disable Naming/PredicateMethod
+      Wurk.redis { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
+      true
+    end
+
+    # Resume fetches. Idempotent.
+    def unpause! # rubocop:disable Naming/PredicateMethod
+      Wurk.redis { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
+      true
+    end
+
+    # Paged LRANGE traversal. Yields JobRecord per payload. Continues
+    # paging until Redis returns < PAGE_SIZE rows.
+    def each
+      page = 0
+      loop do
+        start  = page * PAGE_SIZE
+        stop   = start + PAGE_SIZE - 1
+        slice  = Wurk.redis { |conn| conn.call('LRANGE', @rname, start, stop) }
+        slice.each { |value| yield JobRecord.new(value, @name) }
+        break if slice.size < PAGE_SIZE
+
+        page += 1
+      end
+    end
+
+    # O(n) scan. Returns nil if no job matches.
+    def find_job(jid)
+      each { |record| return record if record.jid == jid }
+      nil
+    end
+
+    # UNLINK the list + drop the queue from the `queues` set. Pipelined
+    # so a partial failure leaves at most one of the two ops applied.
+    # Method name is Sidekiq wire-compat — `clear?` would break the alias.
+    def clear # rubocop:disable Naming/PredicateMethod
+      Wurk.redis do |conn|
+        conn.pipelined do |pipe|
+          pipe.call('UNLINK', @rname)
+          pipe.call('SREM', Keys::QUEUES_SET, @name)
+        end
+      end
+      true
+    end
+
+    def as_json(_options = nil)
+      { name: @name }
+    end
+  end
+end

data/lib/wurk/queues.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Wurk
+  # In-memory job store for `:fake` test mode (aliased to Sidekiq::Queues).
+  # Wurk::Client#raw_push routes payloads here instead of Redis when
+  # `Wurk::Testing.fake?`. Thread-safe so parallel test threads / a job that
+  # enqueues more jobs during `drain` don't corrupt the store.
+  #
+  # Spec: docs/target/sidekiq-free.md §24.2.
+  module Queues
+    @lock = ::Mutex.new
+    @by_queue = ::Hash.new { |h, k| h[k] = [] }
+
+    class << self
+      # Live array reference (not a copy), so the Sidekiq idiom
+      # `Sidekiq::Queues["q"].clear` mutates the underlying store like stock.
+      def [](queue)
+        @lock.synchronize { @by_queue[queue.to_s] }
+      end
+
+      def push(queue, _klass, job)
+        @lock.synchronize { @by_queue[queue.to_s] << job }
+      end
+
+      # Live hash of queue => [jobs], matching Sidekiq::Queues.jobs_by_queue.
+      def jobs_by_queue
+        @lock.synchronize { @by_queue }
+      end
+
+      def jobs_by_class
+        @lock.synchronize { @by_queue.values.flatten.group_by { |j| j['class'].to_s } }
+      end
+      alias jobs_by_worker jobs_by_class
+
+      # Every enqueued payload across all queues, flattened.
+      def jobs
+        @lock.synchronize { @by_queue.values.flatten }
+      end
+
+      def delete_for(jid, queue, _klass)
+        @lock.synchronize { @by_queue[queue.to_s].reject! { |j| j['jid'] == jid } }
+      end
+
+      def clear_for(queue, klass)
+        klass = klass.to_s
+        @lock.synchronize { @by_queue[queue.to_s].reject! { |j| j['class'].to_s == klass } }
+      end
+
+      def clear_all
+        @lock.synchronize { @by_queue.clear }
+      end
+
+      # --- drain helpers (lock released before the job runs, so a job that
+      # enqueues more work doesn't deadlock on the same Mutex) ---------------
+
+      def clear_class(klass)
+        klass = klass.to_s
+        @lock.synchronize { @by_queue.each_value { |jobs| jobs.reject! { |j| j['class'].to_s == klass } } }
+      end
+
+      def shift_class(klass)
+        klass = klass.to_s
+        @lock.synchronize do
+          @by_queue.each_value do |jobs|
+            idx = jobs.index { |j| j['class'].to_s == klass }
+            return jobs.delete_at(idx) if idx
+          end
+          nil
+        end
+      end
+
+      def shift_any
+        @lock.synchronize do
+          @by_queue.each_value { |jobs| return jobs.shift unless jobs.empty? }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/wurk/rails.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# One-line require for Rails hosts: `require "wurk/rails"`.
+# Loads core Wurk, then defines the engine and railtie that mounts the
+# dashboard and forks the swarm after `after_initialize`.
+
+require_relative "../wurk"
+require_relative "engine"
+require_relative "railtie"

data/lib/wurk/railtie.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Wurk
+  # Boot the swarm after the host app has fully initialized.
+  # Skip when: WURK_DISABLED=1, Rails console mode, or Rails test env.
+  # See docs/idea/03-process-model.md for the exact ordering.
+  class Railtie < ::Rails::Railtie
+    config.after_initialize do |app|
+      next if ENV["WURK_DISABLED"] == "1"
+      next if defined?(::Rails::Console)
+      next if ::Rails.env.test?
+
+      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology)
+      swarm.boot
+      # Embedded mode keeps the Rails process serving HTTP; supervise must
+      # run somewhere or signal_queue never drains, crashed children never
+      # respawn, and memory pressure checks never fire. Background thread
+      # leaves the host's main thread free for Rails.
+      Thread.new do
+        swarm.supervise
+      rescue StandardError => e
+        Wurk.configuration.logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
+      end
+    end
+  end
+end

data/lib/wurk/redis_pool.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'redis-client'
+require 'connection_pool'
+
+module Wurk
+  # Per-process pool over redis-client + connection_pool. Never share a socket
+  # across forks: the parent closes the pool before fork, each child opens a
+  # fresh one (see docs/idea/03-process-model.md, steps 3 and 5).
+  #
+  # Retry policy on conn-level errors: close + retry once for messages
+  # prefixed READONLY / NOREPLICAS / UNBLOCKED. Spec: docs/target/sidekiq-free.md §26.
+  class RedisPool
+    DEFAULT_URL     = ENV.fetch('REDIS_URL', 'redis://localhost:6379/0')
+    DEFAULT_TIMEOUT = 1.0
+    DEFAULT_NAME    = 'default'
+
+    # Server-side messages where Sidekiq (and therefore Wurk) closes the
+    # connection and retries the block exactly once. Any other RedisClient::Error
+    # propagates immediately.
+    RETRYABLE_MSG = /\A(READONLY|NOREPLICAS|UNBLOCKED)/
+
+    attr_reader :size, :url, :timeout, :name
+
+    def initialize(size:, url: DEFAULT_URL, timeout: DEFAULT_TIMEOUT, name: DEFAULT_NAME)
+      @size    = size
+      @url     = url
+      @timeout = timeout
+      @name    = name
+      @pool    = ConnectionPool.new(size: size, timeout: timeout) { build_client }
+    end
+
+    def with
+      @pool.with do |conn|
+        attempts = 0
+        begin
+          yield conn
+        rescue RedisClient::Error => e
+          raise unless RETRYABLE_MSG.match?(e.message.to_s)
+          raise if attempts >= 1
+
+          attempts += 1
+          safe_close(conn)
+          retry
+        end
+      end
+    end
+
+    def disconnect!
+      @pool.shutdown { |conn| safe_close(conn) }
+    end
+
+    def info
+      with { |conn| parse_info(conn.call('INFO')) }
+    end
+
+    private
+
+    def build_client
+      RedisClient.config(url: @url, timeout: @timeout).new_client
+    end
+
+    def safe_close(conn)
+      conn.close
+    rescue StandardError
+      nil
+    end
+
+    def parse_info(raw)
+      raw.to_s.each_line.with_object({}) do |line, h|
+        line = line.strip
+        next if line.empty? || line.start_with?('#')
+
+        key, val = line.split(':', 2)
+        h[key] = val if key && val
+      end
+    end
+  end
+end

data/lib/wurk/retry_set.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative 'job_set'
+
+module Wurk
+  # ZSET keyed by next-retry timestamp (score = epoch seconds). Wire-compat
+  # with Sidekiq — the `retry` key, score format, and JSON payload all match.
+  #
+  # Spec: docs/target/sidekiq-free.md §19.5.
+  class RetrySet < JobSet
+    # Optional `name` allows tests to operate on a namespaced ZSET; production
+    # callers always use the default `'retry'` key (wire-compat with Sidekiq).
+    def initialize(name = 'retry')
+      super
+    end
+  end
+end