wurk 1.3.1 → 1.5.0

data/lib/wurk/metrics/accumulator.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Wurk
+  module Metrics
+    # The in-memory half of Wurk::Metrics::History. Every execution folds into a
+    # `{pool => {minute => {class => [processed, failed, ms]}}}` tree under one
+    # mutex; Wurk::Metrics::Flusher drains it into Redis on a timer. `HINCRBY`
+    # is additive, so N folded executions leave Redis in the same state as the N
+    # individual pipelines the hot path used to send.
+    #
+    # Knows nothing about Redis — History owns the write, this owns the counts.
+    #
+    # The minute is the bucket the job *ran* in, decided by the caller at record
+    # time. That is what keeps the batching a visibility lag instead of a
+    # misattribution: a job that runs at 12:03:59 and flushes at 12:04:02 still
+    # counts toward 12:03. Signed off in
+    # docs/plans/2026/08/06/101-faster-than-sidekiq/00-semantics-signoff.md §2.
+    #
+    # Keyed by the pool the recording middleware handed us (nil = the process
+    # default), so a multi-capsule process flushes each capsule's counts through
+    # the pool that capsule's jobs ran on.
+    class Accumulator
+      # Ceiling on minute buckets retained per pool. Only reachable through
+      # #merge_back: a flush that keeps failing puts its counts back every tick,
+      # and a Redis outage that outlives the process would otherwise grow a
+      # structure fed by the job hot path without bound. The newest buckets win
+      # — they are the ones the dashboard is still drawing — and the oldest are
+      # dropped, which these counters have always been allowed to do (a Redis
+      # failure during a metrics write has never changed a job's outcome).
+      MAX_RETAINED_MINUTES = 60
+
+      def initialize
+        @lock = ::Mutex.new
+        @pools = {}
+      end
+
+      # Hot path. Three hash lookups under the mutex; allocates only the first
+      # time a (pool, minute, class) triple is seen.
+      def add(pool, klass, minute, ms, success)
+        @lock.synchronize do
+          counts = (((@pools[pool] ||= {})[minute] ||= {})[klass] ||= [0, 0, 0])
+          counts[success ? 0 : 1] += 1
+          counts[2] += ms
+        end
+      end
+
+      # Hands the whole tree over and starts a fresh one, so recording never
+      # blocks behind the flush's Redis round trip.
+      def drain
+        @lock.synchronize do
+          drained = @pools
+          @pools = {}
+          drained
+        end
+      end
+
+      # Puts one pool's counts back after its write failed, so the next tick
+      # retries them instead of dropping the window. Merged rather than
+      # assigned: jobs kept recording into the fresh tree while the flush was in
+      # flight, and those counts have not been written yet either.
+      def merge_back(pool, minutes)
+        @lock.synchronize do
+          into = (@pools[pool] ||= {})
+          minutes.each { |minute, classes| merge_minute(into, minute, classes) }
+          trim(into)
+        end
+      end
+
+      def empty?
+        @lock.synchronize { @pools.empty? }
+      end
+
+      private
+
+      def merge_minute(into, minute, classes)
+        target = (into[minute] ||= {})
+        classes.each do |klass, counts|
+          existing = target[klass]
+          if existing
+            counts.each_with_index { |count, i| existing[i] += count }
+          else
+            target[klass] = counts
+          end
+        end
+      end
+
+      def trim(minutes)
+        excess = minutes.size - MAX_RETAINED_MINUTES
+        return if excess <= 0
+
+        minutes.keys.min(excess).each { |minute| minutes.delete(minute) }
+      end
+    end
+  end
+end

data/lib/wurk/metrics/flusher.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../component'
+require_relative '../timer_loop'
+require_relative 'history'
+
+module Wurk
+  module Metrics
+    # Drains Wurk::Metrics::History's in-process accumulator into Redis every
+    # `History::FLUSH_INTERVAL` seconds.
+    #
+    # Deliberately NOT leader-gated, unlike every other periodic component here
+    # (Metrics::Rollup, Metrics::QueueRollup, Wurk::History). Those publish
+    # *cluster* state that one process should own; this publishes counters that
+    # only exist inside this process's memory and that no other process can
+    # write. A leader gate would strand every follower's metrics until it died.
+    #
+    # One tick per process, not per capsule: the accumulator is process-wide and
+    # already keyed by the pool each capsule's jobs recorded through.
+    class Flusher
+      include Component
+
+      # `accumulator` is a collaborator, not an option: every real boot flushes
+      # the process-wide one. Tests inject their own so a broken pool stays
+      # inside the test that built it.
+      def initialize(config, accumulator: History::ACCUMULATOR)
+        @config = config
+        @accumulator = accumulator
+        @timer = TimerLoop.new(History::FLUSH_INTERVAL)
+        @thread = nil
+      end
+
+      def start
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('metrics-flush') { @timer.run { tick } }
+      end
+
+      # Flushes once more in an `ensure`: this stops the only thread that would
+      # have written the window accumulated since the last tick, so without it a
+      # graceful shutdown would drop up to FLUSH_INTERVAL of counters — the cost
+      # the sign-off accepts for a *hard* kill only.
+      #
+      # Launcher#stop joins the manager drains before the teardown tail reaches
+      # us, so by the time this runs no job is still recording.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout): a
+      # wedged thread must stay tracked so #start's guard returns it instead of
+      # calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two threads draining the same accumulator.
+      def terminate
+        @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
+      ensure
+        tick
+      end
+
+      # Never raises: this runs on a `safe_thread`, whose watchdog re-raises
+      # after reporting, and a reported-then-dead flusher would silently stop
+      # every counter in the process for a blip Redis recovers from. The
+      # accumulator has already merged the failed window back for the next tick.
+      def tick
+        History.flush(@accumulator)
+      rescue StandardError => e
+        handle_exception(e, { context: 'metrics-flush' })
+      end
+    end
+  end
+end

data/lib/wurk/metrics/history.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative '../middleware'
+require_relative '../pool_checkout'
+require_relative 'accumulator'
 
 module Wurk
   module Metrics
@@ -30,9 +32,10 @@ module Wurk
     # running we keep its bucket around for the retention window measured from
     # *last write*, not from first write. So we EXPIRE unconditionally.
     #
-    # The middleware is hot-path — every successful job pays for it. Writes are
-    # pipelined in a single round-trip per job (1 HINCRBY × 2 + 1 EXPIRE per
-    # bucket × 2 buckets = 6 commands, batched).
+    # The middleware is hot-path — every job pays for it — so it does not touch
+    # Redis at all. Each execution folds into a process-wide Accumulator and
+    # Wurk::Metrics::Flusher writes the whole tree out every FLUSH_INTERVAL, in
+    # the same 6-commands-per-(class, minute) shape the per-job pipeline sent.
     class History
       include Wurk::Middleware::ServerMiddleware
 
@@ -43,6 +46,22 @@ module Wurk
       MINUTE_KEY_PREFIX = 'j|'
       DATE_FORMAT = '%Y%m%d'           # YYYYMMDD — matches Sidekiq's j| key
 
+      # Ceiling on how stale an unflushed counter can be, in seconds. A
+      # constant, never a config knob: it exists to bound the dashboard's lag,
+      # not to be tuned. Sidekiq — the engine these numbers get compared
+      # against — flushes its own process stats on a 10s heartbeat and writes
+      # nothing per job, so this is the tighter of the two. Signed off in
+      # docs/plans/2026/08/06/101-faster-than-sidekiq/00-semantics-signoff.md §2.
+      FLUSH_INTERVAL = 5
+
+      # Process-wide, like Processor::PROCESSED — the counters belong to the
+      # process, not to a middleware instance (the chain builds one per capsule)
+      # and not to a job.
+      ACCUMULATOR = Accumulator.new
+
+      # Hourly buckets are already class-scoped, so their fields are bare.
+      HOUR_FIELDS = %w[p f ms].freeze
+
       def call(_worker, job, _queue)
         klass = job['class']
         started = monotonic_ms
@@ -53,8 +72,11 @@ module Wurk
           result
         ensure
           duration = (monotonic_ms - started).round
-          # Best-effort: a metrics write failure must never propagate into
-          # the job result. The processor already finalized the ack path.
+          # Best-effort: a metrics failure must never propagate into the job
+          # result. The processor already finalized the ack path. Recording is
+          # in-memory now, so what this catches is a bad payload (a `class` that
+          # is not a String) rather than a Redis blip — the Redis half moved to
+          # Flusher, which reports on its own thread.
           begin
             self.class.record(klass, duration, success: success, redis_pool: redis_pool)
           rescue StandardError => e
@@ -64,39 +86,46 @@ module Wurk
       end
 
       class << self
-        # Single Redis round-trip per job. `success: true` → `<klass>|p`;
-        # `success: false` → `<klass>|f`. `<klass>|ms` accumulates total
-        # runtime in milliseconds for *both* outcomes (so an operator can
-        # ask "how much wall-clock time has FooJob consumed?" without
-        # branching on outcome).
-        def record(klass, duration_ms, success:, redis_pool: nil, at: ::Time.now)
+        # No Redis. `success: true` → `<klass>|p`; `success: false` →
+        # `<klass>|f`. `<klass>|ms` accumulates total runtime in milliseconds
+        # for *both* outcomes (so an operator can ask "how much wall-clock time
+        # has FooJob consumed?" without branching on outcome).
+        #
+        # `at:` defaults to nil rather than `::Time.now`: this runs once per job
+        # and the bucket only needs whole UTC minutes, which #minute_bucket
+        # reads straight off the clock as an Integer.
+        def record(klass, duration_ms, success:, redis_pool: nil, at: nil)
           return if klass.nil? || klass.empty?
 
           ms = duration_ms.to_i
           ms = 0 if ms.negative?
-          buckets = { minute: minute_key(at), hour: hour_key(klass, at) }
-          with_pool(redis_pool) { |conn| pipeline_write(conn, klass, ms, success, buckets) }
+          ACCUMULATOR.add(redis_pool, klass, minute_bucket(at), ms, success)
           nil
         end
 
-        # The minute bucket carries per-class `<klass>|p|f|ms` fields; the
-        # hourly bucket is already class-scoped so its fields are bare
-        # `p|f|ms`. Pipeline all 6 commands in one round-trip.
-        def pipeline_write(conn, klass, ms, success, buckets)
-          outcome = success ? 'p' : 'f'
-          class_outcome = "#{klass}|#{outcome}"
-          class_ms = "#{klass}|ms"
-          conn.pipelined do |pipe|
-            incr_bucket(pipe, buckets[:minute], [class_outcome, class_ms, ms, MID_TERM])
-            incr_bucket(pipe, buckets[:hour], [outcome, 'ms', ms, MID_TERM])
+        # Drains an accumulator into Redis, one pipeline per pool. Raises the
+        # first pool's failure after every other pool has had its turn —
+        # Wurk::Metrics::Flusher owns turning that into an error-handler call.
+        #
+        # Only the *failed* pool's counts are merged back. Putting the whole
+        # drained tree back would re-send the writes that already landed, and
+        # HINCRBY would count them twice.
+        #
+        # The argument is a collaborator, not an option: a process has exactly
+        # one accumulator and nothing in wurk passes a second. Tests pass their
+        # own so a deliberately broken pool cannot leak into a parallel test's
+        # flush through the process-wide one.
+        def flush(accumulator = ACCUMULATOR)
+          error = nil
+          accumulator.drain.each do |pool, minutes|
+            write(pool, minutes)
+          rescue StandardError => e
+            accumulator.merge_back(pool, minutes)
+            error = e
           end
-        end
+          raise error if error
 
-        def incr_bucket(pipe, key, parts)
-          outcome_field, ms_field, ms, ttl = parts
-          pipe.call('HINCRBY', key, outcome_field, 1)
-          pipe.call('HINCRBY', key, ms_field, ms)
-          pipe.call('EXPIRE', key, ttl)
+          nil
         end
 
         # Public formatters — Wurk::Metrics::Query reuses these so the two
@@ -111,6 +140,47 @@ module Wurk
           t = time.utc
           "#{klass}-#{t.strftime(DATE_FORMAT)}-#{t.hour}"
         end
+
+        private
+
+        # Whole UTC minutes since the epoch. Integer arithmetic on a raw clock
+        # read, so nothing is allocated per job.
+        def minute_bucket(at)
+          return ::Process.clock_gettime(::Process::CLOCK_REALTIME, :second) / 60 unless at
+
+          at.to_i / 60
+        end
+
+        def write(pool, minutes)
+          with_pool(pool) do |conn|
+            conn.pipelined do |pipe|
+              minutes.each do |minute, classes|
+                at = ::Time.at(minute * 60).utc
+                key = minute_key(at)
+                classes.each { |klass, counts| write_class(pipe, key, at, klass, counts) }
+              end
+            end
+          end
+        end
+
+        def write_class(pipe, bucket_key, at, klass, counts)
+          incr_bucket(pipe, bucket_key, ["#{klass}|p", "#{klass}|f", "#{klass}|ms"], counts)
+          incr_bucket(pipe, hour_key(klass, at), HOUR_FIELDS, counts)
+        end
+
+        # `p`/`f` are emitted only when non-zero, so a bucket ends up with the
+        # exact field set the per-job path produced: `HINCRBY <field> 0` would
+        # materialize a counter no unbatched write ever created, and the read
+        # side would start reporting a `f: 0` series for classes that have never
+        # failed. `ms` is unconditional — the per-job path always incremented
+        # it, including by zero.
+        def incr_bucket(pipe, key, fields, counts)
+          processed, failed, ms = counts
+          pipe.call('HINCRBY', key, fields[0], processed) if processed.positive?
+          pipe.call('HINCRBY', key, fields[1], failed) if failed.positive?
+          pipe.call('HINCRBY', key, fields[2], ms)
+          pipe.call('EXPIRE', key, MID_TERM)
+        end
       end
 
       private
@@ -124,11 +194,11 @@ module Wurk
         cfg.handle_exception(err, context: 'Wurk::Metrics::History')
       end
 
-      def self.with_pool(pool, &)
+      def self.with_pool(pool, idempotent: false, &)
         if pool
-          pool.with(&)
+          PoolCheckout.with(pool, idempotent, &)
         else
-          Wurk.redis(&)
+          Wurk.redis(idempotent:, &)
         end
       end
       private_class_method :with_pool

data/lib/wurk/metrics/queue_rollup.rb

@@ -51,11 +51,23 @@ module Wurk
       end
 
       def start
-        @thread ||= safe_thread('queue-metrics') { @timer.run { tick } } # rubocop:disable Naming/MemoizedInstanceVariableName
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('queue-metrics') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a sample still in
+      # flight would write the same buckets as the next leader's first one.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two threads HSETting the same buckets.
       def terminate
         @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated: only the elected leader samples, so N workers don't each

data/lib/wurk/metrics/rollup.rb

@@ -59,11 +59,23 @@ module Wurk
       end
 
       def start
-        @thread ||= safe_thread('metrics-rollup') { @timer.run { tick } } # rubocop:disable Naming/MemoizedInstanceVariableName
+        return @thread if @thread
+
+        @timer.reset
+        @thread = safe_thread('metrics-rollup') { @timer.run { tick } }
       end
 
+      # Blocks until the thread is really gone: the launcher releases the
+      # cluster lock immediately after this returns, and a roll still in flight
+      # would write the same buckets as the next leader's first one.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged thread must stay tracked so #start's guard returns it instead
+      # of calling @timer.reset, which would un-terminate the loop it is still
+      # inside and leave two threads HSETting the same buckets.
       def terminate
         @timer.terminate
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Leader-gated: only the elected leader writes the cluster-total series,

data/lib/wurk/metrics/statsd.rb

@@ -35,6 +35,15 @@ module Wurk
       METRIC_PREFIX = 'sidekiq.'
       DEFAULT_SAMPLE_RATE = 1.0
 
+      # Unresolved state of the {.client} memo, distinct from a resolved nil.
+      UNSET = :unset
+      private_constant :UNSET
+
+      # Seeded in the class body (not `class << self`, where `self` is the
+      # singleton class) so {.client} is a bare ivar read with no `defined?`
+      # guard on the hot path.
+      @client = UNSET
+
       class << self
         attr_accessor :options
 
@@ -91,19 +100,35 @@ module Wurk
         # Resolves the live client: invokes the configured `dogstatsd` proc
         # exactly once per process and memoizes. Returns nil when no proc
         # is configured, so callers get a clean no-op without raising.
+        #
+        # The *unconfigured* answer is memoized too, which is what makes the
+        # no-client path free: Client#emit_enqueued asks once per payload, so
+        # re-reading `Wurk.configuration` here would cost a config lookup per
+        # job on every bulk push. Both invalidation points call {.reset!}.
         def client
-          return @client if defined?(@client) && !@client.nil?
+          memo = @client
+          return memo unless UNSET.equal?(memo)
 
           builder = Wurk.configuration.respond_to?(:dogstatsd) ? Wurk.configuration.dogstatsd : nil
-          return nil if builder.nil?
-
           @client = builder.respond_to?(:call) ? builder.call : builder
         end
 
-        # Test/lifecycle hook. Reset between specs and after fork so the
-        # parent's socket doesn't bleed into children.
+        # {.client} without the raise: a builder proc that blows up is reported
+        # through the error handler and treated as "no client", so misconfigured
+        # metrics can never fail the work they were measuring. Callers on a
+        # hot path use this to skip building anything the emit would drop.
+        def safe_client
+          client
+        rescue StandardError => e
+          handle_error(e)
+          nil
+        end
+
+        # Test/lifecycle hook. Reset between specs, after fork so the parent's
+        # socket doesn't bleed into children, and on every `config.dogstatsd=`
+        # — a memoized nil would otherwise hide a client configured later.
         def reset!
-          @client = nil
+          @client = UNSET
         end
 
         private
@@ -118,8 +143,7 @@ module Wurk
       end
 
       def call(_worker, job, queue) # rubocop:disable Metrics/AbcSize
-        client = safe_client
-        return yield if client.nil?
+        return yield if self.class.safe_client.nil?
 
         klass = job['class']
         opts  = per_job_options(klass, job, queue)
@@ -146,16 +170,6 @@ module Wurk
 
       private
 
-      # Wraps `self.class.client` so a misconfigured builder proc never turns
-      # a metrics-init failure into a job failure. Returns nil on error (the
-      # caller falls through to a plain `yield`).
-      def safe_client
-        self.class.client
-      rescue StandardError => e
-        self.class.send(:handle_error, e)
-        nil
-      end
-
       # Per-spec §9.2: caller-supplied proc may override tags / sample_rate
       # on a per-job basis. The job's own `dd_rate` field, when present,
       # always wins — it's the per-push override hinted in §8.

data/lib/wurk/middleware/chain.rb

@@ -7,9 +7,10 @@ module Wurk
     # pushes to index 0, `insert_before`/`insert_after` anchor relative to an
     # existing entry, `invoke` walks the chain handing the inner block to each.
     #
-    # `retrieve` instantiates a fresh instance per entry on every job — entries
-    # store klass + ctor args, not the live object. This keeps middleware
-    # thread-safe by construction and matches Sidekiq's lifecycle.
+    # Entries store klass + ctor args, not the live object: every job gets a
+    # fresh instance per entry (`invoke` builds each as it reaches it, `retrieve`
+    # builds them all up front). This keeps middleware thread-safe by
+    # construction and matches Sidekiq's lifecycle.
     #
     # Spec: docs/target/sidekiq-free.md §10.1.
     class Chain
@@ -83,23 +84,28 @@ module Wurk
       # Walks the chain inside-out. Each middleware receives a block that
       # advances to the next; the innermost block is the caller's `&block`,
       # whose return value is propagated back out. Empty chain: `yield`.
+      #
+      # Hot path — one job pays this per invocation, so instantiation is folded
+      # into an index walk over `@entries`: no instance array from `retrieve`,
+      # no traversal lambda, and no `shift` (which memmoves the array per entry).
+      # Cost is one allocation per entry (the middleware itself), zero to walk.
       def invoke(*args, &block)
         raise ArgumentError, 'middleware chain requires a block' unless block
         return yield if @entries.empty?
 
-        chain = retrieve
-        traverse = lambda do
-          if chain.empty?
-            block.call
-          else
-            chain.shift.call(*args, &traverse)
-          end
-        end
-        traverse.call
+        traverse(@entries, 0, args, &block)
       end
 
       private
 
+      def traverse(entries, index, args, &block)
+        return yield if index == entries.size
+
+        entries[index].make_new(@config).call(*args) do
+          traverse(entries, index + 1, args, &block)
+        end
+      end
+
       # Custom dup semantics: deep-copy the entries array so a child chain's
       # mutations don't bleed into the parent (or sibling capsules).
       def initialize_copy(orig)
@@ -108,18 +114,29 @@ module Wurk
       end
 
       # Holds the klass + ctor args for a registered middleware. Defers
-      # instantiation until `retrieve` runs so each job gets a fresh object.
+      # instantiation until `retrieve`/`invoke` runs so each job gets a fresh
+      # object.
+      #
+      # Whether the class declares `config=` is resolved once, at registration,
+      # so the common case (the setter comes from `include
+      # Wurk::Middleware::ServerMiddleware`) skips a `respond_to?` on every job
+      # × every entry. It is a fast path, not the rule: the contract is "set
+      # `config` if the instance is respondable" (spec §10.1), which also covers
+      # a setter defined on the singleton in `initialize`, one reached through
+      # `method_missing`, or one added to the class after it was registered — so
+      # a class that does not declare it still gets asked.
       class Entry
         attr_reader :klass
 
         def initialize(klass, *args)
           @klass = klass
           @args = args
+          @takes_config = klass.public_method_defined?(:config=)
         end
 
         def make_new(config = nil)
           instance = @klass.new(*@args)
-          instance.config = config if config && instance.respond_to?(:config=)
+          instance.config = config if config && (@takes_config || instance.respond_to?(:config=))
           instance
         end
       end

data/lib/wurk/middleware/interrupt_handler.rb

@@ -8,14 +8,15 @@ module Wurk
   module Middleware
     # Server middleware. Catches `Wurk::Job::Interrupted` raised by an
     # IterableJob mid-iteration (or any cooperatively-cancelled job),
-    # re-pushes the job to the head of its queue so it resumes from the
+    # re-pushes the job to the tail of its queue so it resumes from the
     # persisted cursor, and raises `Wurk::JobRetry::Skip` so the retry
     # layer treats this as a clean exit rather than an error.
     #
-    # The re-push uses LPUSH (head of queue) so the same job is the next
-    # one to be fetched after restart. The job JSON is unchanged: cursor
-    # state lives in the `it-<jid>` HASH (see IterableJob persistence),
-    # not in the payload.
+    # The re-push uses RPUSH (tail of queue) so the same job is the next
+    # one to be fetched: the fetcher's LMOVE pops from the RIGHT (tail),
+    # so this job is fetched ahead of fresh LPUSH'd enqueues. The job
+    # JSON is unchanged: cursor state lives in the `it-<jid>` HASH (see
+    # IterableJob persistence), not in the payload.
     #
     # Auto-registered at the top of the server chain when this file is
     # required. Top-of-chain is important: a downstream middleware must
@@ -36,7 +37,7 @@ module Wurk
 
       def repush(job, queue)
         payload = Wurk.dump_json(job)
-        redis_pool.with { |conn| conn.call('LPUSH', "queue:#{queue}", payload) }
+        redis_pool.with { |conn| conn.call('RPUSH', "queue:#{queue}", payload) }
       end
     end
   end