wurk 0.0.1

data/lib/wurk/scheduled.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require_relative 'component'
+require_relative 'keys'
+require_relative 'lua'
+require_relative 'lua/loader'
+require_relative 'client'
+require_relative 'process_set'
+
+module Wurk
+  # Promotes due jobs from the `retry` and `schedule` sorted sets back onto
+  # their target queues. One Poller thread per process; collectively they
+  # drain both SETS via an atomic Lua pop-by-score (loaded via the EVALSHA
+  # cache, retried once on NOSCRIPT). Polling cadence scales with cluster
+  # size so total scheduler traffic stays constant as processes are added.
+  #
+  # Spec: docs/target/sidekiq-free.md §16. Pluggable via `config[:scheduled_enq]`.
+  module Scheduled
+    SETS = %w[retry schedule].freeze
+
+    # Atomic pop-by-score for retry/schedule. Source must match
+    # Wurk::Lua::ZPOPBYSCORE byte-for-byte — they share the same SHA.
+    LUA_ZPOPBYSCORE = Wurk::Lua::ZPOPBYSCORE
+
+    # Drains both SETS each call. Iterates per-set inside a single pooled
+    # checkout so the EVALSHA + LPUSH loop avoids re-checkout per job.
+    class Enq
+      include Component
+
+      LUA_ZPOPBYSCORE = Wurk::Lua::ZPOPBYSCORE
+
+      def initialize(container)
+        @config = container
+        @done = false
+        @client = Client.new(config: container)
+      end
+
+      # Pops every due job from each sorted set and re-pushes through the
+      # client. `now` is captured once per set so a slow loop on one ZSET
+      # can't keep grabbing newly-scheduled jobs from a moving window.
+      def enqueue_jobs(sorted_sets = SETS)
+        @config.redis do |conn|
+          sorted_sets.each { |sset| drain_set(conn, sset) }
+        end
+      end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def drain_set(conn, sset)
+        now = real_time.to_s
+        loop do
+          break if @done
+
+          jobstr = Wurk::Lua::Loader.eval_cached(conn, :zpopbyscore, keys: [sset], argv: [now])
+          break unless jobstr
+
+          @client.push(Wurk.load_json(jobstr))
+        end
+      end
+
+      def real_time
+        ::Process.clock_gettime(::Process::CLOCK_REALTIME)
+      end
+    end
+
+    # Single thread that wakes on a randomized interval, drains both ZSETs,
+    # then sleeps again. Random spread prevents the cluster from dogpiling
+    # Redis at the top of each cadence.
+    class Poller
+      include Component
+
+      INITIAL_WAIT = 10
+
+      attr_accessor :rnd
+
+      def initialize(config)
+        @config = config
+        @enq = (config[:scheduled_enq] || Enq).new(config)
+        @done = false
+        @mutex = ::Mutex.new
+        @sleeper = ::ConditionVariable.new
+        @thread = nil
+        @rnd = ::Random.new
+        @last_cleanup_ms = 0
+      end
+
+      # Spawns the scheduler thread. INITIAL_WAIT delays the first sweep so
+      # a fleet-wide deploy doesn't have every freshly-booted process hit
+      # Redis simultaneously.
+      def start
+        @thread ||= safe_thread('scheduler') do # rubocop:disable Naming/MemoizedInstanceVariableName
+          initial_wait
+          until @done
+            enqueue
+            wait
+          end
+          logger.info('Scheduler exiting...')
+        end
+      end
+
+      # Idempotent. Wakes the sleeping thread so it observes @done and exits.
+      # Also propagates the stop signal to @enq so any in-flight drain loop
+      # short-circuits instead of running to completion.
+      def terminate
+        @mutex.synchronize do
+          @done = true
+          @enq.terminate
+          @sleeper.signal
+        end
+      end
+
+      # Called on every wake. Any raise inside the Enq is reported and the
+      # loop continues — a transient Redis blip must not kill the scheduler.
+      def enqueue
+        @enq.enqueue_jobs
+      rescue StandardError => e
+        handle_exception(e, { context: 'scheduler' })
+      end
+
+      private
+
+      # INITIAL_WAIT (10s) staggers the fleet's first sweep after a deploy so
+      # freshly-booted processes don't hit Redis in unison. Overridable via
+      # `config[:scheduler_initial_wait]` (tests want a near-zero first sweep).
+      def initial_wait
+        wait = @config[:scheduler_initial_wait] || INITIAL_WAIT
+        @mutex.synchronize do
+          @sleeper.wait(@mutex, wait) unless @done
+        end
+      end
+
+      def wait
+        @mutex.synchronize do
+          @sleeper.wait(@mutex, random_poll_interval) unless @done
+        end
+      end
+
+      # interval = process_count * average_scheduled_poll_interval
+      # <10 procs: jitter `interval * rand + interval/2`
+      # ≥10 procs: jitter `interval * rand * 2`
+      # The two regimes produce comparable expected wait times but the
+      # high-cluster form widens the spread so 100+ processes don't cluster.
+      def random_poll_interval
+        count = process_count
+        interval = poll_interval_average(count)
+        if count < 10
+          (interval * @rnd.rand) + (interval / 2.0)
+        else
+          interval * @rnd.rand * 2
+        end
+      end
+
+      def poll_interval_average(count)
+        @config[:poll_interval_average] || scaled_poll_interval(count)
+      end
+
+      def scaled_poll_interval(count)
+        count * @config[:average_scheduled_poll_interval]
+      end
+
+      # SCARD on the `processes` SET, floor of 1 so a freshly-booted process
+      # (not yet in the set) still computes a non-zero interval.
+      def process_count
+        pcount = cleanup
+        pcount = 1 if pcount < 1
+        pcount
+      end
+
+      # Returns the current `processes` SCARD. Rate-limited to 1/min: the
+      # full ProcessSet prune is expensive (SMEMBERS + per-id HGET), so we
+      # only invoke it when at least 60s have passed; intermediate calls
+      # just SCARD and trust the previous prune.
+      def cleanup
+        @config.redis do |conn|
+          if mono_ms - @last_cleanup_ms > 60_000
+            @last_cleanup_ms = mono_ms
+            ProcessSet.new(true).size
+          else
+            conn.call('SCARD', Keys::PROCESSES)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wurk/scheduled_set.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'job_set'
+
+module Wurk
+  # ZSET of jobs scheduled to run at a future time (score = epoch seconds).
+  # The scheduled-poller pops eligible members and re-enqueues via the
+  # client. Wire-compat with Sidekiq's `schedule` key.
+  #
+  # Spec: docs/target/sidekiq-free.md §19.5.
+  class ScheduledSet < JobSet
+    # Optional `name` allows tests to operate on a namespaced ZSET; production
+    # callers always use the default `'schedule'` key (wire-compat with Sidekiq).
+    def initialize(name = 'schedule')
+      super
+    end
+  end
+end

data/lib/wurk/sorted_entry.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require_relative 'job_record'
+
+module Wurk
+  # One entry inside a sorted-set view (Retry/Scheduled/Dead). Carries the
+  # member's `score` alongside the JobRecord so callers can re-target the
+  # exact (score, value) pair when mutating Redis — sorted-set membership is
+  # by value, but ZREM-by-value is faster than ZRANGEBYSCORE+filter.
+  #
+  # The `id` field ("<score>|<jid>") is the Sidekiq wire-compat identifier
+  # used by dashboards and third-party tooling. Don't reformat it.
+  #
+  # Spec: docs/target/sidekiq-free.md §19.4.
+  class SortedEntry < JobRecord
+    attr_reader :score, :parent
+
+    # @param parent [JobSet, nil] the owning set; nil when constructed bare.
+    # @param score [Numeric] ZSET score (Float seconds since epoch).
+    # @param item [String, Hash] raw JSON or parsed payload.
+    def initialize(parent, score, item)
+      super(item)
+      @score = score.to_f
+      @parent = parent
+    end
+
+    def id = "#{score}|#{jid}"
+
+    def at = ::Time.at(score).utc
+
+    # Removes this entry from the parent set. Prefers exact-value match
+    # (idempotent across duplicates with the same jid), falls back to
+    # (score, jid) when constructed without a cached `value`.
+    def delete
+      if @value
+        @parent.delete_by_value(@parent.name, @value)
+      else
+        @parent.delete_by_jid(@score, jid)
+      end
+    end
+
+    # ZINCRBY to shift the score; positive deltas reschedule into the future.
+    # Sidekiq passes the absolute target time; we compute the delta here so
+    # the call survives clock skew between caller and Redis.
+    def reschedule(at) # rubocop:disable Naming/MethodParameterName
+      Wurk.redis { |conn| conn.call('ZINCRBY', @parent.name, at.to_f - @score, value) }
+    end
+
+    # Removes this entry, decrements `retry_count` by one (so the worker treats
+    # the next attempt as a re-do, not a fresh retry), and re-enqueues via the
+    # client. Wire-compat with Sidekiq's "Retry now" UI action.
+    def add_to_queue
+      remove_job do |message|
+        message['retry_count'] = message['retry_count'].to_i - 1 if message['retry_count']
+        Client.new.push(message)
+      end
+    end
+
+    # Same flow as add_to_queue but keeps `retry_count` intact. Used for the
+    # "retry" action from the retry set (count was already incremented when
+    # the job entered retry; don't double-bump).
+    def retry
+      remove_job do |message|
+        Client.new.push(message)
+      end
+    end
+
+    # Removes this entry from its parent set and writes it to the dead set.
+    # `notify_failure: false` because the kill is user-initiated (UI action),
+    # not a retry-exhausted event — death_handlers don't fire.
+    def kill
+      remove_job do |message|
+        DeadSet.new.kill(Wurk.dump_json(message), notify_failure: false)
+      end
+    end
+
+    def error? = !item['error_class'].nil?
+
+    private
+
+    # Pulls the message out of Redis, yields it for the caller's re-enqueue
+    # work, and returns the parsed hash. Done with the cached value when
+    # available so LREM-like ZREM matches the exact bytes.
+    # Returns nil without yielding when the parent removal fails — prevents
+    # duplicate side effects (e.g. retry pushing twice) if another caller
+    # already removed the entry.
+    def remove_job
+      message = item.dup
+      return nil unless @parent.remove_job(self)
+
+      yield message
+      message
+    end
+  end
+end

data/lib/wurk/stats.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require 'date'
+
+module Wurk
+  # Read-only inspector for cluster state in Redis. The cheap counters are
+  # eagerly pipelined at initialize so a single instance can answer many
+  # questions without re-querying; the unbounded ones (`enqueued`,
+  # `workers_size`, `queue_summaries`) re-fetch lazily.
+  #
+  # Wire-compat is sacred: every key matches the Sidekiq OSS schema exactly.
+  # Spec: docs/target/sidekiq-free.md §19.1.
+  class Stats
+    # Sidekiq exposes this as a `Data` class. Third-party gems destructure on
+    # `name`/`size`/`latency`/`paused?`, so the shape can't change.
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    def initialize
+      fetch_stats_fast!
+    end
+
+    def processed       = @stats.fetch(:processed)
+    def failed          = @stats.fetch(:failed)
+    def expired         = @stats.fetch(:expired)
+    def scheduled_size  = @stats.fetch(:scheduled_size)
+    def retry_size      = @stats.fetch(:retry_size)
+    def dead_size       = @stats.fetch(:dead_size)
+    def processes_size  = @stats.fetch(:processes_size)
+
+    # Sum of LLEN across every known queue. Linear in queue count — the
+    # spec labels this "slow" upstream; don't put it on a hot path.
+    def enqueued
+      queues.each_value.sum
+    end
+
+    # Sum of the `busy` HASH field across every live process identity.
+    # Pipelined but unbounded by process count.
+    def workers_size
+      Wurk.redis do |conn|
+        identities = conn.call('SMEMBERS', Keys::PROCESSES)
+        next 0 if identities.empty?
+
+        busy = conn.pipelined do |pipe|
+          identities.each { |id| pipe.call('HGET', id, 'busy') }
+        end
+        busy.sum(&:to_i)
+      end
+    end
+
+    # @return [Hash{String=>Integer}] queue name → LLEN.
+    def queues
+      Wurk.redis do |conn|
+        names = conn.call('SMEMBERS', Keys::QUEUES_SET)
+        next {} if names.empty?
+
+        sizes = conn.pipelined do |pipe|
+          names.each { |q| pipe.call('LLEN', Keys.queue(q)) }
+        end
+        names.zip(sizes.map(&:to_i)).to_h
+      end
+    end
+
+    # @return [Array<QueueSummary>] one per known queue.
+    def queue_summaries
+      Wurk.redis do |conn|
+        names = conn.call('SMEMBERS', Keys::QUEUES_SET)
+        next [] if names.empty?
+
+        paused_set = conn.call('SMEMBERS', 'paused')
+        results = conn.pipelined do |pipe|
+          names.each do |q|
+            pipe.call('LLEN', Keys.queue(q))
+            pipe.call('LRANGE', Keys.queue(q), -1, -1)
+          end
+        end
+        build_summaries(names, results, paused_set)
+      end
+    end
+
+    # Latency (secs) of the `default` queue — the most-asked-about gauge.
+    def default_queue_latency
+      now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+      payload = Wurk.redis { |c| c.call('LRANGE', Keys.queue('default'), -1, -1) }.first
+      compute_latency(payload, now_ms)
+    end
+
+    # Resets the named global counters. With no args, clears `processed`,
+    # `failed`, and `expired`. SET … 0 (not DEL — keeps the key around so
+    # reads stay `Integer` not `nil`).
+    def reset(*stats)
+      all      = %w[failed processed expired]
+      to_clear = stats.empty? ? all : all & stats.flatten.map(&:to_s)
+      Wurk.redis do |conn|
+        conn.pipelined do |pipe|
+          to_clear.each { |s| pipe.call('SET', "stat:#{s}", 0) }
+        end
+      end
+    end
+
+    private
+
+    # Single pipeline for the cheap counters. Eagerly invoked at initialize
+    # so callers can read many fields without paying per-method round trips.
+    FAST_QUERIES = [
+      ['GET',   'stat:processed'],
+      ['GET',   'stat:failed'],
+      ['GET',   Keys::STAT_EXPIRED],
+      ['ZCARD', Keys::SCHEDULE],
+      ['ZCARD', Keys::RETRY],
+      ['ZCARD', Keys::DEAD],
+      ['SCARD', Keys::PROCESSES]
+    ].freeze
+    FAST_KEYS = %i[processed failed expired scheduled_size retry_size dead_size processes_size].freeze
+    private_constant :FAST_QUERIES, :FAST_KEYS
+
+    def fetch_stats_fast!
+      raw = Wurk.redis do |conn|
+        conn.pipelined { |pipe| FAST_QUERIES.each { |args| pipe.call(*args) } }
+      end
+      @stats = FAST_KEYS.zip(raw.map(&:to_i)).to_h
+    end
+
+    def build_summaries(names, results, paused_set)
+      now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+      names.each_with_index.map do |name, i|
+        QueueSummary.new(
+          name: name,
+          size: results[i * 2].to_i,
+          latency: compute_latency(results[(i * 2) + 1].first, now_ms),
+          paused: paused_set.include?(name)
+        )
+      end
+    end
+
+    # `enqueued_at` may be Float (epoch secs, legacy) or Integer (epoch ms,
+    # current). Spec §5 calls out the dual format; handle both. Malformed
+    # JSON or non-numeric `enqueued_at` shouldn't crash a dashboard read —
+    # fall back to 0.
+    def compute_latency(payload_json, now_ms)
+      return 0.0 if payload_json.nil?
+
+      enq = Float(Wurk.load_json(payload_json)['enqueued_at'] || 0)
+      enq_ms = enq < 10_000_000_000 ? enq * 1_000 : enq
+      diff = (now_ms - enq_ms) / 1_000.0
+      diff.negative? ? 0.0 : diff
+    rescue ::JSON::ParserError, ::TypeError, ::ArgumentError
+      0.0
+    end
+
+    # Per-day historical processed/failed/expired counts. Reads
+    # `stat:processed:YYYY-MM-DD`, `stat:failed:YYYY-MM-DD`, and
+    # `stat:expired:YYYY-MM-DD` strings; missing days return 0. Range
+    # 1..1825 (5 years) mirrors upstream.
+    class History
+      MAX_DAYS = 1_825
+
+      def initialize(days_previous, start_date = nil, pool: nil)
+        raise ArgumentError, "days_previous must be in 1..#{MAX_DAYS}" unless (1..MAX_DAYS).cover?(days_previous)
+
+        @days_previous = days_previous
+        @start_date    = start_date || ::Date.today
+        @pool          = pool
+      end
+
+      def processed = date_stat_hash('processed')
+      def failed    = date_stat_hash('failed')
+      def expired   = date_stat_hash('expired')
+
+      private
+
+      def date_stat_hash(stat)
+        keys = (0...@days_previous).map { |i| (@start_date - i).strftime('%Y-%m-%d') }
+        values = with_redis do |conn|
+          conn.pipelined { |pipe| keys.each { |d| pipe.call('GET', "stat:#{stat}:#{d}") } }
+        end
+        keys.zip(values.map(&:to_i)).to_h
+      end
+
+      def with_redis(&)
+        if @pool
+          @pool.with(&)
+        else
+          Wurk.redis(&)
+        end
+      end
+    end
+  end
+end

data/lib/wurk/swarm/child_boot.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative '../launcher'
+require_relative '../fetcher/reliable'
+
+module Wurk
+  class Swarm
+    # Step 5 of the boot ordering. Runs inside each forked child:
+    #   * reset signal traps inherited from the parent,
+    #   * reconnect ActiveRecord (if loaded) + open a fresh Redis pool,
+    #   * apply the slot's queues + concurrency to the default capsule,
+    #   * install child signal handlers (TERM/INT drain, TSTP quiet,
+    #     USR2 reopen logs),
+    #   * launch the Wurk::Launcher and block until shutdown.
+    #
+    # Kept separate from Wurk::Swarm so the parent supervisor stays
+    # focused on PID supervision (SRP).
+    class ChildBoot
+      CHILD_SIGNALS = { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp, 'USR2' => :usr2 }.freeze
+
+      def initialize(config, slot, index)
+        @config = config
+        @slot = slot
+        @index = index
+        @signal_queue = ::Thread::Queue.new
+      end
+
+      def run
+        reset_inherited_signals
+        reconnect_after_fork
+        Wurk.server = true
+        apply_slot_to_config
+        launcher = Wurk::Launcher.new(@config)
+        install_signal_handlers(launcher)
+        launcher.run
+        wait_loop(launcher)
+        exit 0
+      rescue StandardError, ::Wurk::Shutdown => e
+        @config.logger.error { "swarm child ##{@index} (#{::Process.pid}) crashed: #{e.class}: #{e.message}" }
+        exit 1
+      end
+
+      private
+
+      # Parent installed traps for TERM/INT/TSTP/CONT/USR1 — the child
+      # needs its own behavior, not the parent's.
+      def reset_inherited_signals
+        %w[TERM INT TSTP CONT USR1 USR2].each { |s| ::Signal.trap(s, 'DEFAULT') }
+      end
+
+      def reconnect_after_fork
+        @config.reset_redis_pools!
+        return unless defined?(::ActiveRecord::Base)
+
+        begin
+          ::ActiveRecord::Base.establish_connection
+        rescue StandardError
+          nil
+        end
+      end
+
+      def apply_slot_to_config
+        cap = @config.default_capsule
+        cap.queues = @slot.queues
+        cap.concurrency = @slot.concurrency
+        # Fetcher defaulting + lazy-ivar materialization now happens in
+        # Configuration#freeze! (Capsule#prepare!), called by Launcher#run
+        # below — for every entry point, not just the swarm.
+      end
+
+      def install_signal_handlers(launcher)
+        CHILD_SIGNALS.each { |sig, sym| ::Signal.trap(sig) { @signal_queue << sym } }
+        @dispatcher = Thread.new { dispatch_signals(launcher) }
+      end
+
+      # TSTP/USR2 keep looping; TERM/INT run the full launcher.stop
+      # (which blocks on manager drain) and then return — wait_loop
+      # joins this thread, so the main child thread can't `exit 0`
+      # mid-drain. Otherwise quiet would flip launcher.stopping? true
+      # and the main thread would race past the unfinished managers.
+      def dispatch_signals(launcher)
+        loop do
+          case @signal_queue.pop
+          when :term
+            launcher.stop
+            return
+          when :tstp then launcher.quiet
+          when :usr2 then reopen_logs
+          end
+        end
+      end
+
+      def reopen_logs
+        log = @config.logger
+        log.reopen if log.respond_to?(:reopen)
+      rescue StandardError
+        nil
+      end
+
+      def wait_loop(_launcher)
+        @dispatcher.join
+      end
+    end
+  end
+end