wurk 1.7.6 → 1.8.0

data/lib/wurk/client/buffered.rb

@@ -9,6 +9,11 @@ module Wurk
     # creation or batch-context pushes (`bid` on payload): BATCH_PUSH has
     # atomic counter side-effects we can't safely replay.
     #
+    # One buffer serves every pool, so each entry remembers the Redis it was
+    # headed for (its {Origin}) and is only ever replayed there: a sharded app
+    # pushing through `Client.via(shard_a)` must not have its outage backlog
+    # land on shard B because B's producer happened to push next.
+    #
     # Spec: docs/target/sidekiq-pro.md §5.
     module Buffered
       DEFAULT_BUFFER_CAP = 1_000
@@ -40,6 +45,31 @@ module Wurk
       # Returned by the append helpers when everything fit.
       NOTHING_UNDELIVERED = [].freeze
 
+      Entry = Struct.new(:payload, :origin)
+
+      # Where a buffered payload has to be replayed. What must NOT be captured
+      # is a pool object the config hands out: `reset_redis_pools!` — every
+      # fork, every embedded teardown — disconnects it and drops it for a
+      # lazily rebuilt one, and ConnectionPool#shutdown is terminal, so a
+      # pinned instance replays into dead sockets for the rest of the process's
+      # life. For such a pool the config (a Configuration or a Capsule, which
+      # survives the rebuild) is kept and asked again at replay time. A pool
+      # the config does not own — `Client.new(pool:)`, `Client.via(pool)` — is
+      # a second Redis nothing else can produce, so that one stays pinned,
+      # stale or not: replaying it anywhere else writes to the wrong server.
+      Origin = Struct.new(:config, :pool) do
+        def self.for(client)
+          pool = client.send(:pool)
+          config = client.instance_variable_get(:@config)
+          owned = config.respond_to?(:redis_pool) && config.redis_pool.equal?(pool)
+          owned ? new(config, nil) : new(nil, pool)
+        end
+
+        def resolve
+          config ? config.redis_pool : pool
+        end
+      end
+
       # Eagerly initialized: `||=` inside an accessor is not atomic — two
       # threads racing first-touch could end up holding distinct Mutex
       # instances and lose all synchronization on the shared buffer.
@@ -60,8 +90,6 @@ module Wurk
       @owner_pid = ::Process.pid
 
       class << self
-        attr_accessor :buffer_client_factory
-
         # Idempotent. Prepends the wrapper module into Wurk::Client so push /
         # push_bulk drain the buffer before each call and raw_push catches
         # connection errors. Safe to call from multiple threads.
@@ -117,11 +145,10 @@ module Wurk
             @buffer = []
             @buffer_cap = nil
             @overflow_mode = nil
-            @buffer_client_factory = nil
           end
-          # Stop before dropping: an unstopped drainer thread survives with
-          # its factory nil'd out from under it and ticks forever against
-          # nothing, leaking the thread and everything its closure retains.
+          @drops = DropLog.new
+          # Stop before dropping: an unstopped drainer thread would otherwise
+          # tick on forever, unreachable, leaking the thread.
           install_mutex.synchronize do
             @drainer&.stop
             @drainer = nil
@@ -146,8 +173,7 @@ module Wurk
         # `@drainer` is dropped, never `stop`ped — its `@lock` carries the same
         # inherited-mutex hazard. A parent-configured drainer is replaced by an
         # equivalent fresh one so an opted-in child keeps flushing the buffer it
-        # fills itself; the captured client factory goes with it, since it
-        # closes over the parent's pre-fork Redis pool.
+        # fills itself.
         #
         # Deliberately unsynchronized: the child has exactly one thread here,
         # and waiting on the very mutex being replaced is what would hang it.
@@ -158,7 +184,7 @@ module Wurk
           @install_mutex         = Mutex.new
           @buffer_mutex          = Mutex.new
           @buffer                = []
-          @buffer_client_factory = nil
+          @drops                 = DropLog.new
           interval               = @drainer&.interval
           @drainer               = nil
           start_drainer!(interval: interval) if interval
@@ -172,62 +198,32 @@ module Wurk
         #                                    raises one Overflow carrying every
         #                                    payload that did not fit.
         # Drops batched payloads — caller is expected to re-raise for those.
-        # If client is provided, captures its pool for drainer to use by default.
-        def enbuffer(payloads, client: nil)
-          capture_pool_from_client(client)
+        # `client` is the one whose push failed; every payload is tagged with
+        # the Redis that client was writing to (see Origin).
+        def enbuffer(payloads, client:)
+          origin = Origin.for(client)
+          entries = payloads.map { |payload| Entry.new(payload, origin) }
 
           cap  = buffer_cap
           mode = overflow_mode
           undelivered = buffer_mutex.synchronize do
-            mode == :raise ? append_within_capacity(payloads, cap) : append_dropping_oldest(payloads, cap)
+            mode == :raise ? append_within_capacity(entries, cap) : append_dropping_oldest(entries, cap)
           end
 
-          raise Overflow, undelivered unless undelivered.empty?
+          raise Overflow, undelivered.map(&:payload) unless undelivered.empty?
+        ensure
+          drops.report(cap)
         end
 
         private
 
-        # Remember how the buffering client reaches Redis, so the drainer
-        # replays into the same server it was pushed to unless explicitly
-        # overridden.
-        def capture_pool_from_client(client)
-          return unless client && !buffer_client_factory
-
-          pool = client.instance_variable_get(:@redis_pool)
-          # A nil capture must not install a factory: it would pin the drainer
-          # to the DEFAULT pool forever (the `!buffer_client_factory` guard
-          # blocks any later, correct capture) — wrong Redis for jobs pushed
-          # through an explicit-pool client. A pool-less client already resolves
-          # its config at push time, and so does the fallback factory.
-          return unless pool
-
-          resolver = pool_resolver(client, pool)
-          self.buffer_client_factory = -> { Wurk::Client.new(pool: resolver.call) }
-        end
-
-        # What must NOT be captured is the pool object. `reset_redis_pools!` —
-        # every fork, every embedded teardown — disconnects a pool and drops it
-        # for a lazily rebuilt one, and ConnectionPool#shutdown is terminal, so
-        # a pinned instance leaves the drainer replaying into dead sockets for
-        # the rest of the process's life. The config (a Configuration or a
-        # Capsule) is what survives that rebuild, so ask it again at drain time
-        # whenever the client's pool is the one it hands out. A pool the config
-        # does not own is a second Redis nothing else can produce — that one
-        # stays pinned, stale or not, because replaying it anywhere else writes
-        # to the wrong server.
-        def pool_resolver(client, pool)
-          config = client.instance_variable_get(:@config)
-          config_owns = config.respond_to?(:redis_pool) && config.redis_pool.equal?(pool)
-          config_owns ? -> { config.redis_pool } : -> { pool }
-        end
-
-        # Both append helpers run with buffer_mutex held and return the payloads
+        # Both append helpers run with buffer_mutex held and return the entries
         # they could not take.
 
-        def append_dropping_oldest(payloads, cap)
-          payloads.each do |p|
-            buffer.shift if buffer.size >= cap
-            buffer << p
+        def append_dropping_oldest(entries, cap)
+          entries.each do |entry|
+            drops.record(buffer.shift.payload) if buffer.size >= cap
+            buffer << entry
           end
           NOTHING_UNDELIVERED
         end
@@ -238,69 +234,78 @@ module Wurk
         # negative when the cap was lowered after the buffer filled — clamped,
         # so an over-full buffer rejects the whole call instead of raising on
         # `first`/`drop`.
-        def append_within_capacity(payloads, cap)
-          room = (cap - buffer.size).clamp(0, payloads.size)
-          if room == payloads.size
-            buffer.concat(payloads)
+        def append_within_capacity(entries, cap)
+          room = (cap - buffer.size).clamp(0, entries.size)
+          if room == entries.size
+            buffer.concat(entries)
             return NOTHING_UNDELIVERED
           end
 
-          buffer.concat(payloads.first(room))
-          payloads.drop(room)
+          buffer.concat(entries.first(room))
+          entries.drop(room)
         end
 
         public
 
-        # Drain payloads through `raw_push` on the given client. Stops on
-        # the first transient failure (ConnectionError past the pool's own
-        # retries, or a starved checkout), preserving order at the head of
-        # the buffer so the next push retries the same payload. Emits statsd
-        # `jobs.recovered.push` per drained payload, plus the `jobs.enqueued`
-        # the buffering push deliberately did not emit — the replay is where
-        # the job actually reaches Redis, so a buffered-then-drained job counts
-        # once as enqueued and once as recovered.
+        # Replay, through `raw_push` on `client`, the buffered payloads headed
+        # for the Redis that client writes to — oldest first; entries for any
+        # other Redis are left in place, so a push to a healthy shard neither
+        # misroutes nor waits on a dead one's backlog. Stops on the first
+        # transient failure (ConnectionError past the pool's own retries, or a
+        # starved checkout) and puts that payload back at the head, so the next
+        # push retries it first. Emits statsd `jobs.recovered.push` per drained
+        # payload, plus the `jobs.enqueued` the buffering push deliberately did
+        # not emit — the replay is where the job actually reaches Redis, so a
+        # buffered-then-drained job counts once as enqueued and once as
+        # recovered.
         def drain!(client)
-          drained = 0
-          while (payload = pop_head)
-            begin
-              replayed = attempt_replay(client, payload)
-            rescue StandardError
-              # Non-connection failures (OOM, LOADING, READONLY…) must not
-              # drop the popped payload — restore it before propagating, or
-              # a recovering-but-not-ready Redis silently eats one buffered
-              # job per drain tick.
-              buffer_mutex.synchronize { buffer.unshift(payload) }
-              raise
-            end
+          return 0 if buffer_mutex.synchronize { buffer.empty? }
 
-            unless replayed
-              buffer_mutex.synchronize { buffer.unshift(payload) }
-              break
-            end
+          target = client.send(:pool)
+          drained = 0
+          while (entry = take_next(target))
+            break unless replay(client, entry)
 
-            client.send(:emit_enqueued, [payload])
-            Wurk::Metrics::Statsd.increment('jobs.recovered.push')
             drained += 1
           end
+          drops.end_burst if drained.positive?
+          drained
+        end
+
+        # The background drainer's pass: every origin in the buffer, each
+        # through a client of its own, so one Redis still down does not hold
+        # back another that has recovered. Returns the total replayed; the
+        # first non-transient error is re-raised after every origin had its turn.
+        def drain_all!
+          failure = nil
+          drained = buffered_pools.sum do |pool|
+            drain!(Wurk::Client.new(pool: pool))
+          rescue StandardError => e
+            failure ||= e
+            0
+          end
+          raise failure if failure
+
           drained
         end
 
-        # Internal — visible for tests. Treat as private.
+        # Internal — visible for tests. Treat as private. Holds Entry structs.
         def buffer
           @buffer ||= []
         end
 
+        attr_reader :drops
+
         # Start a background drain thread that wakes every `interval`
         # seconds and tries to flush the buffer. Idempotent — replaces
         # any prior drainer with one at the new interval. Issue #19
         # requirement: "Background drain thread flushes on reconnect" —
         # handles the case where push activity stops mid-outage so the
         # passive (drain-on-next-push) path never fires.
-        def start_drainer!(interval: Drainer::DEFAULT_INTERVAL, client_factory: nil)
+        def start_drainer!(interval: Drainer::DEFAULT_INTERVAL)
           install_mutex.synchronize do
             @drainer&.stop
-            factory = client_factory || buffer_client_factory || -> { Wurk::Client.new }
-            @drainer = Drainer.new(interval: interval, client_factory: factory)
+            @drainer = Drainer.new(interval: interval)
             @drainer.start
           end
         end
@@ -320,8 +325,45 @@ module Wurk
 
         attr_reader :install_mutex, :buffer_mutex
 
-        def pop_head
-          buffer_mutex.synchronize { buffer.shift }
+        # The oldest entry bound for `target`, removed from the buffer.
+        def take_next(target)
+          buffer_mutex.synchronize do
+            index = buffer.index { |entry| entry.origin.resolve.equal?(target) }
+            index && buffer.delete_at(index)
+          end
+        end
+
+        def buffered_pools
+          buffer_mutex.synchronize { buffer.map { |entry| entry.origin.resolve } }.uniq(&:object_id)
+        end
+
+        # The entry once it is in Redis, nil when it went back to the buffer.
+        # Same-origin order is all that matters (other entries go to another
+        # server), so a payload that could not be replayed goes back at the
+        # absolute head: ahead of every later entry for its own Redis.
+        def replay(client, entry)
+          begin
+            replayed = attempt_replay(client, entry.payload)
+          rescue StandardError
+            # Non-connection failures (OOM, LOADING, READONLY…) must not drop
+            # the taken payload — restore it before propagating, or a
+            # recovering-but-not-ready Redis silently eats one buffered job per
+            # drain tick.
+            restore(entry)
+            raise
+          end
+          unless replayed
+            restore(entry)
+            return
+          end
+
+          client.send(:emit_enqueued, [entry.payload])
+          Wurk::Metrics::Statsd.increment('jobs.recovered.push')
+          entry
+        end
+
+        def restore(entry)
+          buffer_mutex.synchronize { buffer.unshift(entry) }
         end
 
         # Drain marks the thread so our prepended raw_push re-raises the
@@ -338,12 +380,82 @@ module Wurk
         end
       end
 
-      # Background drain thread. Wakes every `interval` seconds and tries
-      # `Buffered.drain!` against a fresh Wurk::Client. drain! already
-      # short-circuits on the first transient failure, so a still-down Redis
-      # just leaves the buffer alone for this tick — no exponential
-      # backoff or explicit "reconnect detection" needed; the inner
-      # connection retry already lives inside `client.raw_push`.
+      # The jobs `:drop_oldest` evicts. A drop is a job the caller was told was
+      # enqueued and that will now never run, so it must never be silent. One
+      # ERROR per burst — the first drop since a replay last reached Redis —
+      # because an outage that outlasts the cap drops on every push, and a line
+      # per job would bury the one that says what happened. Every drop still
+      # counts, as the statsd `jobs.dropped.push` counter.
+      #
+      # #record runs under the buffer's mutex; #report runs after it is
+      # released, so neither the log write nor statsd holds up other pushes.
+      class DropLog
+        def initialize
+          @lock = Mutex.new
+          @pending = []
+          @burst = false
+        end
+
+        def record(payload)
+          @lock.synchronize { @pending << payload }
+        end
+
+        def report(cap)
+          dropped, announce = @lock.synchronize do
+            next [nil, false] if @pending.empty?
+
+            taken = @pending
+            @pending = []
+            first = !@burst
+            @burst = true
+            [taken, first]
+          end
+          return unless dropped
+
+          log(dropped.first, cap) if announce
+          dropped.size.times { count_drop }
+        end
+
+        # A replay reached Redis, so the outage that filled the buffer is over:
+        # the next overflow is a new incident and gets its own ERROR.
+        def end_burst
+          @lock.synchronize { @burst = false }
+        end
+
+        private
+
+        # Runs in Buffered.enbuffer's `ensure`: anything raised here would
+        # replace the push's own result (its return, or the Overflow it is
+        # raising), so the metric is strictly best-effort.
+        def count_drop
+          Wurk::Metrics::Statsd.increment('jobs.dropped.push')
+        rescue StandardError
+          nil
+        end
+
+        def log(first, cap)
+          Wurk.configuration.logger.error do
+            "reliable_push buffer full (cap=#{cap}): dropping the oldest buffered jobs to make room — " \
+              "#{first['class']} jid=#{first['jid']} is the first of this burst and will never run. " \
+              'Further drops are counted in statsd jobs.dropped.push until Redis takes a replay. Raise ' \
+              '`Wurk::Client.reliable_push_buffer`, or set `reliable_push_overflow = :raise` to handle ' \
+              'overflow yourself.'
+          end
+        rescue StandardError
+          nil
+        end
+      end
+
+      # Eager for the reason the mutexes are: a lazy `||=` could hand two
+      # racing first pushes different logs.
+      @drops = DropLog.new
+
+      # Background drain thread. Wakes every `interval` seconds and runs
+      # `Buffered.drain_all!`. drain! already short-circuits on the first
+      # transient failure, so a still-down Redis just leaves its entries alone
+      # for this tick — no exponential backoff or explicit "reconnect
+      # detection" needed; the inner connection retry already lives inside
+      # `client.raw_push`.
       class Drainer
         DEFAULT_INTERVAL = 2.0
         STOP_JOIN_TIMEOUT = 5.0
@@ -352,13 +464,12 @@ module Wurk
         # rebuild an equivalent one in the child without touching its lock.
         attr_reader :interval
 
-        def initialize(interval: DEFAULT_INTERVAL, client_factory: -> { Wurk::Client.new })
+        def initialize(interval: DEFAULT_INTERVAL)
           unless interval.is_a?(Numeric) && interval.positive?
             raise ArgumentError, 'interval must be a positive Numeric'
           end
 
           @interval = interval
-          @client_factory = client_factory
           @done = false
           @thread = nil
           @wake = ConditionVariable.new
@@ -397,15 +508,25 @@ module Wurk
             wait_interval
             break if @done
 
-            begin
-              Buffered.drain!(@client_factory.call)
-            rescue StandardError
-              # Swallow — next tick retries. Don't let a transient blow up
-              # the daemon thread.
-            end
+            tick
           end
         end
 
+        # A raise must not end the thread — the next tick retries — but it is
+        # reported: a transient outage never gets here (drain! absorbs it), so
+        # whatever does is something an operator needs to see.
+        def tick
+          Buffered.drain_all!
+        rescue StandardError => e
+          report(e)
+        end
+
+        def report(error)
+          Wurk.configuration.handle_exception(error, { context: 'reliable_push drainer' })
+        rescue StandardError
+          nil
+        end
+
         # Mutex+ConditionVariable lets `stop` wake the thread immediately
         # instead of waiting up to `interval` seconds for sleep to return.
         def wait_interval
@@ -499,7 +620,7 @@ module Wurk
       # Activate reliable_push! mode globally. Idempotent — call from the
       # top level of an initializer (NOT inside Wurk.configure_*). Spec:
       # docs/target/sidekiq-pro.md §5.
-      def reliable_push! # rubocop:disable Naming/PredicateMethod
+      def reliable_push!
         Buffered.install!
         true
       end

data/lib/wurk/client.rb

@@ -50,6 +50,13 @@ module Wurk
     # pays a single nil check per write phase.
     DELIVERED_KEY = :wurk_client_delivered
 
+    # Thread-local write-progress marker for a caller that has to know whether
+    # a failed push may already be in Redis — {API::Idempotency} opens it at
+    # :clean around a produce request. #raw_push moves it to :attempted when a
+    # write goes out and to :applied once one lands; nil (no caller watching)
+    # costs one lookup per write phase.
+    WRITE_STATE_KEY = :wurk_client_write_state
+
     attr_accessor :redis_pool
 
     def initialize(pool: nil, config: nil, chain: nil)
@@ -266,10 +273,10 @@ module Wurk
       Array.new(count) { now + (rand * window) }
     end
 
-    # Inside an autoflush `Batch#jobs` block immediate batched pushes are
-    # accumulated in the buffer rather than written; it flushes every N jobs
-    # (when autoflush is an Integer) and Batch#jobs drains the remainder at
-    # block exit. Scheduled (`at`) or non-batched payloads bypass the buffer.
+    # Inside a `Batch#jobs` block immediate batched pushes are accumulated in
+    # the buffer rather than written; it flushes every N jobs (when autoflush
+    # is an Integer) and Batch#jobs drains the remainder at block exit.
+    # Scheduled (`at`) or non-batched payloads bypass the buffer.
     #
     # Adds happen one payload at a time so an `autoflush = N` actually bounds
     # the pipeline size — a bulk push of 100 with N=2 must flush 2/2/... not
@@ -293,16 +300,28 @@ module Wurk
 
       # No apply-safety claim: every command below appends (LPUSH, ZADD, the
       # batch Lua's counters), so a block replayed after a lost reply is a
-      # second copy of the job. A post-write timeout raises out of here instead
-      # — {Client::Buffered} turns that into an outage-buffer entry, and a plain
-      # Client hands it to whoever called `perform_async`. The pool's pre-apply
-      # retry only fires while this block has landed nothing, so a queue group
-      # that already went out is never re-pushed by a replay.
-      pool.with { |conn| atomic_push(conn, payloads) }
+      # second copy of the job. The pool's pre-apply retry only fires while this
+      # block has landed nothing, so a queue group that already went out is
+      # never re-pushed by a block replay. That is the pool's guarantee, not the
+      # socket's: redis-client re-sends the one in-flight pipeline once on a
+      # dropped connection (RedisPool::DEFAULT_RECONNECT_ATTEMPTS, Sidekiq's
+      # setting too), so a reply lost mid-pipeline can still land that group
+      # twice. Only an error that outlasts that re-send raises out of here —
+      # {Client::Buffered} turns it into an outage-buffer entry, and a plain
+      # Client hands it to whoever called `perform_async`.
+      tracked_push(payloads)
       nil
     end
 
-    # Batch autoflush path: accumulate each non-scheduled batched payload into
+    # The write itself, moving WRITE_STATE_KEY along for a caller watching it.
+    def tracked_push(payloads)
+      state = Thread.current[WRITE_STATE_KEY]
+      Thread.current[WRITE_STATE_KEY] = :attempted if state == :clean
+      pool.with { |conn| atomic_push(conn, payloads) }
+      Thread.current[WRITE_STATE_KEY] = :applied if state
+    end
+
+    # Batch#jobs path: accumulate each non-scheduled batched payload into
     # the active buffer, flushing every N adds (when `buffer.ready?`).
     def buffer_add(buffer, payloads)
       payloads.each do |payload|
@@ -379,8 +398,8 @@ module Wurk
     end
 
     # Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
-    # code; the eager `script_load_all` after fork keeps the script cache
-    # hot for the life of the connection. The retry uses EVAL (source-embedded)
+    # code; the swarm child's post-fork cache check (Loader.load_missing)
+    # keeps the script cache hot for the life of the connection. The retry uses EVAL (source-embedded)
     # instead of EVALSHA so a freshly-loaded script can't race the retry and
     # NOSCRIPT a second time under heavy CI load (WorkerTest 3.4/7.2 flake).
     # `script_load_all` still primes the cache so the *next* pipeline returns
@@ -527,6 +546,7 @@ module Wurk
     # ledger.
     def mark_delivered(payloads)
       Thread.current[DELIVERED_KEY]&.concat(payloads)
+      Thread.current[WRITE_STATE_KEY] &&= :applied
     end
 
     # Best-effort `sidekiq.jobs.enqueued` counter — one increment per payload

data/lib/wurk/compat.rb

@@ -5,6 +5,8 @@
 #
 # Spec: docs/target/sidekiq-{free,pro,ent}.md.
 
+require_relative 'loader'
+
 # The `Sidekiq::*` compatibility namespace. Every public Wurk class is exposed
 # here under its Sidekiq name so an existing Sidekiq/Pro/Enterprise app runs on
 # Wurk after a one-line gem swap. `Sidekiq::Job` / `Sidekiq::Worker` are
@@ -136,10 +138,15 @@ module Sidekiq
   JobLogger        = Wurk::JobLogger
   JobRecord        = Wurk::JobRecord
   JobRetry         = Wurk::JobRetry
+  # sidekiq-unique-jobs reopens these with `class Sidekiq::JobSet` to prepend
+  # its lock release; without the alias that defines a fresh, unused class and
+  # deleted jobs never give their locks back.
+  JobSet           = Wurk::JobSet
   JobUtil          = Wurk::JobUtil
   Keys             = Wurk::Keys
   Launcher         = Wurk::Launcher
   Limiter          = Wurk::Limiter
+  Loader           = Wurk::Loader
   Logger           = Wurk::Logger
   Manager          = Wurk::Manager
   Metrics          = Wurk::Metrics
@@ -160,6 +167,7 @@ module Sidekiq
   ScheduledSet     = Wurk::ScheduledSet
   Shutdown         = Wurk::Shutdown
   SortedEntry      = Wurk::SortedEntry
+  SortedSet        = Wurk::SortedSet
   Stats            = Wurk::Stats
   # No `Sidekiq::Status` alias on purpose, same reason as `Sidekiq::Cron`
   # (#204): the sidekiq-status gem owns that namespace and opens it with
@@ -212,5 +220,10 @@ module Sidekiq
     def testing? = Wurk.testing?
     def load_json(str) = Wurk.load_json(str)
     def dump_json(obj) = Wurk.dump_json(obj)
+    def loader = Wurk.loader
   end
 end
+
+# Upstream fires this at the end of sidekiq/api.rb; Wurk's API is fully loaded
+# by the time its aliases are.
+Wurk.loader.run_load_hooks(:api)