wurk 1.3.0 → 1.4.0

data/lib/wurk/client/buffered.rb

@@ -21,24 +21,43 @@ module Wurk
       OVERFLOW_MODES        = %i[drop_oldest raise].freeze
       DEFAULT_OVERFLOW_MODE = :drop_oldest
 
-      # Raised by `enbuffer` when the cap would be exceeded under
-      # `overflow_mode == :raise`. Inherits from RuntimeError so callers
-      # can rescue narrowly. The payload that triggered the overflow rides
-      # along so the caller can persist/log/forward it.
+      # Raised when the cap would be exceeded under `overflow_mode == :raise`.
+      # Inherits from RuntimeError so callers can rescue narrowly. Carries
+      # EVERY payload the call failed to deliver — the tail that did not fit,
+      # plus, on a mixed push, the batched payloads that never buffer — so a
+      # caller can persist/log/forward the lot. `cause` is the connection error
+      # that sent the push to the buffer in the first place.
       class Overflow < RuntimeError
-        attr_reader :payload
+        attr_reader :payloads
 
-        def initialize(payload)
-          @payload = payload
-          super("reliable_push buffer is full (cap=#{Buffered.buffer_cap})")
+        def initialize(payloads)
+          @payloads = payloads
+          super("reliable_push buffer is full (cap=#{Buffered.buffer_cap}), " \
+                "#{payloads.size} payload(s) undelivered")
         end
       end
 
+      # Returned by the append helpers when everything fit.
+      NOTHING_UNDELIVERED = [].freeze
+
       # Eagerly initialized: `||=` inside an accessor is not atomic — two
       # threads racing first-touch could end up holding distinct Mutex
       # instances and lose all synchronization on the shared buffer.
-      INSTALL_MUTEX = Mutex.new
-      BUFFER_MUTEX  = Mutex.new
+      #
+      # Module ivars rather than constants so `reset_after_fork!` can replace
+      # them outright. MRI abandons a mutex whose owner thread didn't survive
+      # the fork (rb_thread_atfork), but that's an implementation detail rather
+      # than a documented guarantee, and it does NOT cover a fork taken from
+      # inside either critical section — there the child inherits the lock
+      # still owned, and its first `Client#push` (which drains, so it
+      # synchronizes, before pushing) blocks forever. Two allocations per fork
+      # buys immunity from both.
+      @install_mutex = Mutex.new
+      @buffer_mutex  = Mutex.new
+
+      # Process that owns the state above; a mismatch means we're running in a
+      # fork and the inherited copy has to go.
+      @owner_pid = ::Process.pid
 
       class << self
         attr_accessor :buffer_client_factory
@@ -102,38 +121,68 @@ module Wurk
           end
         end
 
+        # Fork hook, called from the `Process._fork` prepend below and from
+        # `Swarm::ChildBoot#reconnect_after_fork`. Whichever runs first wins
+        # and returns true; the pid guard makes the other a no-op returning
+        # false, so a caller can tell which one rebuilt the state.
+        #
+        # A child inherits a copy of every ivar here: the buffered payloads,
+        # the Drainer (whose thread did not survive the fork), and both mutexes
+        # (see their definition for why replacing them matters).
+        #
+        # The child DROPS its inherited payloads rather than replaying them:
+        # the parent still holds the same buffer and replays it on its own next
+        # push, so a child that also drained would enqueue every buffered job
+        # once per fork — `(children + 1) x N` duplicates. Only the parent
+        # replays.
+        #
+        # `@drainer` is dropped, never `stop`ped — its `@lock` carries the same
+        # inherited-mutex hazard. A parent-configured drainer is replaced by an
+        # equivalent fresh one so an opted-in child keeps flushing the buffer it
+        # fills itself; the captured client factory goes with it, since it
+        # closes over the parent's pre-fork Redis pool.
+        #
+        # Deliberately unsynchronized: the child has exactly one thread here,
+        # and waiting on the very mutex being replaced is what would hang it.
+        def reset_after_fork! # rubocop:disable Naming/PredicateMethod
+          return false if @owner_pid == ::Process.pid
+
+          @owner_pid             = ::Process.pid
+          @install_mutex         = Mutex.new
+          @buffer_mutex          = Mutex.new
+          @buffer                = []
+          @buffer_client_factory = nil
+          interval               = @drainer&.interval
+          @drainer               = nil
+          start_drainer!(interval: interval) if interval
+          true
+        end
+
         # Append payloads to the buffer. Behavior on cap exhaustion depends
         # on `overflow_mode`:
         #   * :drop_oldest (default, spec) — ring buffer, oldest evicted.
-        #   * :raise                       — Overflow raised, buffer left
-        #                                    unchanged for already-appended
-        #                                    siblings in the same call; the
-        #                                    offending payload is attached
-        #                                    to the exception.
+        #   * :raise                       — fills the remaining capacity, then
+        #                                    raises one Overflow carrying every
+        #                                    payload that did not fit.
         # Drops batched payloads — caller is expected to re-raise for those.
         # If client is provided, captures its pool for drainer to use by default.
         def enbuffer(payloads, client: nil)
           capture_pool_from_client(client)
 
-          cap = buffer_cap
+          cap  = buffer_cap
           mode = overflow_mode
-          buffer_mutex.synchronize do
-            payloads.each do |p|
-              if buffer.size >= cap
-                raise Overflow, p if mode == :raise
-
-                buffer.shift # :drop_oldest
-              end
-              buffer << p
-            end
+          undelivered = buffer_mutex.synchronize do
+            mode == :raise ? append_within_capacity(payloads, cap) : append_dropping_oldest(payloads, cap)
           end
+
+          raise Overflow, undelivered unless undelivered.empty?
         end
 
         private
 
-        # Capture the pool from the provided client and set it as the default
-        # factory for the drainer. Ensures buffered jobs are drained to the
-        # same pool they were pushed to, unless explicitly overridden.
+        # Remember how the buffering client reaches Redis, so the drainer
+        # replays into the same server it was pushed to unless explicitly
+        # overridden.
         def capture_pool_from_client(client)
           return unless client && !buffer_client_factory
 
@@ -141,10 +190,56 @@ module Wurk
           # A nil capture must not install a factory: it would pin the drainer
           # to the DEFAULT pool forever (the `!buffer_client_factory` guard
           # blocks any later, correct capture) — wrong Redis for jobs pushed
-          # through an explicit-pool client.
+          # through an explicit-pool client. A pool-less client already resolves
+          # its config at push time, and so does the fallback factory.
           return unless pool
 
-          self.buffer_client_factory = -> { Wurk::Client.new(pool: pool) }
+          resolver = pool_resolver(client, pool)
+          self.buffer_client_factory = -> { Wurk::Client.new(pool: resolver.call) }
+        end
+
+        # What must NOT be captured is the pool object. `reset_redis_pools!` —
+        # every fork, every embedded teardown — disconnects a pool and drops it
+        # for a lazily rebuilt one, and ConnectionPool#shutdown is terminal, so
+        # a pinned instance leaves the drainer replaying into dead sockets for
+        # the rest of the process's life. The config (a Configuration or a
+        # Capsule) is what survives that rebuild, so ask it again at drain time
+        # whenever the client's pool is the one it hands out. A pool the config
+        # does not own is a second Redis nothing else can produce — that one
+        # stays pinned, stale or not, because replaying it anywhere else writes
+        # to the wrong server.
+        def pool_resolver(client, pool)
+          config = client.instance_variable_get(:@config)
+          config_owns = config.respond_to?(:redis_pool) && config.redis_pool.equal?(pool)
+          config_owns ? -> { config.redis_pool } : -> { pool }
+        end
+
+        # Both append helpers run with buffer_mutex held and return the payloads
+        # they could not take.
+
+        def append_dropping_oldest(payloads, cap)
+          payloads.each do |p|
+            buffer.shift if buffer.size >= cap
+            buffer << p
+          end
+          NOTHING_UNDELIVERED
+        end
+
+        # The split has to be decided before any mutation: raising from inside
+        # the append loop leaves every payload after the rejected one neither
+        # buffered, nor enqueued, nor attached to the exception. `room` goes
+        # negative when the cap was lowered after the buffer filled — clamped,
+        # so an over-full buffer rejects the whole call instead of raising on
+        # `first`/`drop`.
+        def append_within_capacity(payloads, cap)
+          room = (cap - buffer.size).clamp(0, payloads.size)
+          if room == payloads.size
+            buffer.concat(payloads)
+            return NOTHING_UNDELIVERED
+          end
+
+          buffer.concat(payloads.first(room))
+          payloads.drop(room)
         end
 
         public
@@ -153,7 +248,10 @@ module Wurk
         # the first transient failure (ConnectionError past the pool's own
         # retries, or a starved checkout), preserving order at the head of
         # the buffer so the next push retries the same payload. Emits statsd
-        # `jobs.recovered.push` per drained payload.
+        # `jobs.recovered.push` per drained payload, plus the `jobs.enqueued`
+        # the buffering push deliberately did not emit — the replay is where
+        # the job actually reaches Redis, so a buffered-then-drained job counts
+        # once as enqueued and once as recovered.
         def drain!(client)
           drained = 0
           while (payload = pop_head)
@@ -173,6 +271,7 @@ module Wurk
               break
             end
 
+            client.send(:emit_enqueued, [payload])
             Wurk::Metrics::Statsd.increment('jobs.recovered.push')
             drained += 1
           end
@@ -191,7 +290,7 @@ module Wurk
         # handles the case where push activity stops mid-outage so the
         # passive (drain-on-next-push) path never fires.
         def start_drainer!(interval: Drainer::DEFAULT_INTERVAL, client_factory: nil)
-          INSTALL_MUTEX.synchronize do
+          install_mutex.synchronize do
             @drainer&.stop
             factory = client_factory || buffer_client_factory || -> { Wurk::Client.new }
             @drainer = Drainer.new(interval: interval, client_factory: factory)
@@ -200,25 +299,19 @@ module Wurk
         end
 
         def stop_drainer!
-          INSTALL_MUTEX.synchronize do
+          install_mutex.synchronize do
             @drainer&.stop
             @drainer = nil
           end
         end
 
         def drainer_running?
-          INSTALL_MUTEX.synchronize { @drainer&.running? == true }
+          install_mutex.synchronize { @drainer&.running? == true }
         end
 
         private
 
-        def install_mutex
-          INSTALL_MUTEX
-        end
-
-        def buffer_mutex
-          BUFFER_MUTEX
-        end
+        attr_reader :install_mutex, :buffer_mutex
 
         def pop_head
           buffer_mutex.synchronize { buffer.shift }
@@ -248,6 +341,10 @@ module Wurk
         DEFAULT_INTERVAL = 2.0
         STOP_JOIN_TIMEOUT = 5.0
 
+        # Read by `Buffered.reset_after_fork!` off the *inherited* drainer, to
+        # rebuild an equivalent one in the child without touching its lock.
+        attr_reader :interval
+
         def initialize(interval: DEFAULT_INTERVAL, client_factory: -> { Wurk::Client.new })
           unless interval.is_a?(Numeric) && interval.positive?
             raise ArgumentError, 'interval must be a positive Numeric'
@@ -326,16 +423,69 @@ module Wurk
 
         private
 
+        # Opens the delivery ledger Client writes into. It lives here rather
+        # than in Client because this rescue is its only reader: a Client
+        # without reliable_push! never allocates it.
         def raw_push(payloads)
+          Thread.current[DELIVERED_KEY] = []
           super
         rescue RedisClient::ConnectionError, ConnectionPool::TimeoutError
           raise if Thread.current[Buffered::DRAINING_KEY]
 
-          bidless, batched = payloads.partition { |p| !p['bid'] }
-          Buffered.enbuffer(bidless, client: self) if bidless.any?
+          bidless, batched = undelivered(payloads).partition { |p| !p['bid'] }
+          enbuffer_bidless(bidless, batched)
           raise unless batched.empty?
+
+          # Client#push subtracts these from the enqueued metric: they are in
+          # the buffer, not in Redis. Whatever the group did deliver stays out
+          # of the set and still counts.
+          bidless
+        ensure
+          Thread.current[DELIVERED_KEY] = nil
+        end
+
+        # The payloads this push is not known to have written. A push spanning
+        # several queues, or one mixing plain and batched jobs, fails after
+        # some of its groups already landed; buffering those would replay them
+        # into duplicate jobs once the outage clears. The ledger holds the very
+        # Hash objects Client just handed to Redis, hence the identity subtract.
+        def undelivered(payloads)
+          delivered = Thread.current[DELIVERED_KEY]
+          return payloads if delivered.empty?
+
+          reject_by_identity(payloads, delivered)
+        end
+
+        # Bare `raise` re-raises whatever `$!` holds: the connection error from
+        # the caller's rescue, or the Overflow once we're inside this one.
+        def enbuffer_bidless(bidless, batched)
+          Buffered.enbuffer(bidless, client: self) if bidless.any?
+        rescue Buffered::Overflow => e
+          # An overflow pre-empts the caller's connection-error re-raise, which
+          # would strand the batched payloads silently — they never buffer. One
+          # exception, every payload that failed to get through; `cause` stays
+          # the connection error rather than the folded-in Overflow.
+          raise if batched.empty?
+
+          raise Buffered::Overflow, e.payloads + batched, cause: e.cause
+        end
+      end
+
+      # Ruby >= 3.1 routes every `fork` / `Process.fork` through
+      # `Process._fork`, which is the only way to catch the forks Wurk never
+      # sees: a Puma or Unicorn parent that preloaded the app — and may already
+      # be holding buffered payloads — spawning its workers. Registered at
+      # require time because `reliable_push!` can be installed after the fork
+      # that copied the state. Guarded on the fork-less runtimes (JRuby), where
+      # there is no `super` to call.
+      module ForkHook
+        def _fork
+          pid = super
+          Buffered.reset_after_fork! if pid.zero?
+          pid
         end
       end
+      ::Process.singleton_class.prepend(ForkHook) if ::Process.respond_to?(:_fork)
     end
 
     class << self

data/lib/wurk/client.rb

@@ -26,6 +26,15 @@ module Wurk
     SCHEDULED_BATCH_SIZE      = 100
     SPREAD_INTERVAL_FLOOR     = 5
 
+    # Thread-local slot holding the payloads of the current push whose Redis
+    # write is confirmed applied. {Client::Buffered} subtracts them from the set
+    # it re-buffers when a *later* phase of the same push loses the connection,
+    # so an already-written job is never replayed into a second copy.
+    # Thread-local because one Client instance serves every producer thread;
+    # opened and closed by Buffered, the only reader, so an un-prepended Client
+    # pays a single nil check per write phase.
+    DELIVERED_KEY = :wurk_client_delivered
+
     attr_accessor :redis_pool
 
     def initialize(pool: nil, config: nil, chain: nil)
@@ -51,8 +60,8 @@ module Wurk
       return nil unless payload
 
       verify_json(payload)
-      raw_push([payload])
-      emit_enqueued([payload])
+      buffered = raw_push([payload])
+      emit_enqueued([payload], buffered)
       payload['jid']
     end
 
@@ -169,8 +178,8 @@ module Wurk
         payloads = build_bulk_payloads(slice, base, ats)
         compacted = payloads.compact
         if compacted.any?
-          raw_push(compacted)
-          emit_enqueued(compacted)
+          buffered = raw_push(compacted)
+          emit_enqueued(compacted, buffered)
         end
         jids.concat(payloads.map { |p| p && p['jid'] })
       end
@@ -222,16 +231,32 @@ module Wurk
     # Adds happen one payload at a time so an `autoflush = N` actually bounds
     # the pipeline size — a bulk push of 100 with N=2 must flush 2/2/... not
     # 100 in one shot.
+    #
+    # Returns the payloads it did NOT get to Redis: always nil here, since a
+    # plain Client either writes them all or raises. {Client::Buffered}
+    # overrides the contract — the payloads it diverted into the outage buffer
+    # come back so #push can keep them out of the enqueued metric.
     def raw_push(payloads)
       # Test modes short-circuit the Redis write (and the batch buffer): :fake
       # collects payloads in-memory, :inline runs them now. Client middleware
       # has already run by this point, matching Sidekiq.
-      return ::Wurk::Testing.dispatch_push(payloads) if ::Wurk::Testing.enabled?
+      if ::Wurk::Testing.enabled?
+        ::Wurk::Testing.dispatch_push(payloads)
+        return nil
+      end
 
       buffer = Thread.current[Wurk::Batch::BUFFER_KEY]
       return buffer_add(buffer, payloads) if buffer && payloads.all? { |p| p['bid'] && !p['at'] }
 
+      # No apply-safety claim: every command below appends (LPUSH, ZADD, the
+      # batch Lua's counters), so a block replayed after a lost reply is a
+      # second copy of the job. A post-write timeout raises out of here instead
+      # — {Client::Buffered} turns that into an outage-buffer entry, and a plain
+      # Client hands it to whoever called `perform_async`. The pool's pre-apply
+      # retry only fires while this block has landed nothing, so a queue group
+      # that already went out is never re-pushed by a replay.
       pool.with { |conn| atomic_push(conn, payloads) }
+      nil
     end
 
     # Batch autoflush path: accumulate each non-scheduled batched payload into
@@ -262,7 +287,10 @@ module Wurk
     # duplicate the scheduled entry.
     def push_scheduled_split(conn, payloads)
       batched, plain = payloads.partition { |j| j['bid'] }
-      conn.pipelined { |pipe| push_scheduled(pipe, plain) } unless plain.empty?
+      unless plain.empty?
+        conn.pipelined { |pipe| push_scheduled(pipe, plain) }
+        mark_delivered(plain)
+      end
       push_batched_scheduled_pipelined(conn, batched) unless batched.empty?
     end
 
@@ -282,7 +310,7 @@ module Wurk
     def push_immediate(conn, payloads)
       now = now_in_millis
       batched, plain = payloads.partition { |j| j['bid'] }
-      conn.pipelined { |pipe| push_plain(pipe, plain, now) } unless plain.empty?
+      push_plain(conn, plain, now) unless plain.empty?
       push_batched_pipelined(conn, batched, now) unless batched.empty?
     end
 
@@ -313,15 +341,31 @@ module Wurk
       conn.pipelined { |pipe| push_batched_scheduled(pipe, batched, eval_method: :eval_with_source) }
     end
 
+    # One pipeline per queue, marked delivered the moment its reply is in.
+    # A push can die between groups — or land its whole plain phase and then
+    # lose the connection in the batched phase above — and whatever Redis
+    # already accepted must stay out of the reliable_push buffer; replaying it
+    # would enqueue a second copy of a job that ran fine.
+    #
+    # The group in flight when the socket drops stays unmarked and so is
+    # replayed: a lost reply is indistinguishable from a lost command, so that
+    # residual is at-least-once by construction. The split bounds it to one
+    # queue group per failed push instead of the entire payload set.
+    #
+    # Cost is a round trip per distinct queue. The single-queue push — every
+    # `perform_async`, every same-class `push_bulk` — still writes exactly the
+    # one SADD + LPUSH pipeline it did before.
     def push_plain(conn, payloads, now)
-      grouped = payloads.group_by { |j| j['queue'] }
-      conn.call('SADD', 'queues', *grouped.keys)
-      grouped.each do |queue, jobs|
+      payloads.group_by { |j| j['queue'] }.each do |queue, jobs|
         serialized = jobs.map do |j|
           j['enqueued_at'] = now
           Wurk.dump_json(j)
         end
-        conn.call('LPUSH', "queue:#{queue}", *serialized)
+        conn.pipelined do |pipe|
+          pipe.call('SADD', 'queues', queue)
+          pipe.call('LPUSH', "queue:#{queue}", *serialized)
+        end
+        mark_delivered(jobs)
       end
     end
 
@@ -341,7 +385,7 @@ module Wurk
           :batch_push,
           keys: ["b-#{j['bid']}", "b-#{j['bid']}-jids", "queue:#{j['queue']}", 'queues',
                  "b-#{j['bid']}-died", 'dead-batches'],
-          argv: [j['queue'], j['jid'], Wurk.dump_json(j), j['bid']]
+          argv: [j['queue'], j['jid'], Wurk.dump_json(j), j['bid'], Wurk::Batch::DEFAULT_EXPIRY_SECONDS]
         )
       end
     end
@@ -360,7 +404,8 @@ module Wurk
           conn,
           :batch_schedule,
           keys: ['schedule', "b-#{j['bid']}", "b-#{j['bid']}-jids"],
-          argv: [j['at'].to_s, Wurk.dump_json(j.except('enqueued_at', 'at')), j['jid']]
+          argv: [j['at'].to_s, Wurk.dump_json(j.except('enqueued_at', 'at')), j['jid'],
+                 Wurk::Batch::DEFAULT_EXPIRY_SECONDS]
         )
       end
     end
@@ -369,11 +414,30 @@ module Wurk
       @redis_pool || Thread.current[:wurk_via_pool] || @config.redis_pool
     end
 
+    # Record a write Redis has acknowledged, for {Client::Buffered} to subtract
+    # from what it re-buffers: one push spans several pipelines (plain vs
+    # batched, immediate vs scheduled), and the ledger is what keeps a group
+    # that already landed out of the buffer when a later group fails. It
+    # accumulates across pool attempts and is never pruned — #raw_push claims no
+    # apply-safety, and RedisPool refuses to replay such a block once one of its
+    # round trips has completed, so the only replay left starts from an empty
+    # ledger.
+    def mark_delivered(payloads)
+      Thread.current[DELIVERED_KEY]&.concat(payloads)
+    end
+
     # Best-effort `sidekiq.jobs.enqueued` counter — one increment per payload
     # that actually made it past middleware AND Redis. Tags follow the same
     # `worker:`/`queue:` shape as Wurk::Metrics::Statsd so dashboards built
     # for the server-side emissions work unchanged.
-    def emit_enqueued(payloads)
+    #
+    # `buffered` is what reliable_push swallowed into its outage buffer (see
+    # #raw_push). Those payloads are not enqueued: the ring buffer may still
+    # evict them, and the drain that does land one counts it then — so booking
+    # them here would inflate the counter on an outage and double-count every
+    # payload that later replays.
+    def emit_enqueued(payloads, buffered = nil)
+      payloads = reject_by_identity(payloads, buffered) if buffered && !buffered.empty?
       payloads.each do |p|
         Wurk::Metrics::Statsd.increment(
           'jobs.enqueued',
@@ -381,5 +445,14 @@ module Wurk
         )
       end
     end
+
+    # Set difference by object identity — `==` would fold two jobs carrying the
+    # same fields into one. Both sides are always the very Hash objects this
+    # push built, so identity is both exact and cheaper than hashing them.
+    def reject_by_identity(payloads, excluded)
+      seen = {}.compare_by_identity
+      excluded.each { |p| seen[p] = true }
+      payloads.reject { |p| seen.key?(p) }
+    end
   end
 end

data/lib/wurk/compat.rb

@@ -170,7 +170,7 @@ module Sidekiq
     def configure_client(&) = Wurk.configure_client(&)
     def configure_embed(&) = Wurk.configure_embed(&)
     def default_configuration = Wurk.default_configuration
-    def redis(&) = Wurk.redis(&)
+    def redis(idempotent: false, &) = Wurk.redis(idempotent:, &)
     def redis_pool = Wurk.redis_pool
     def logger = Wurk.logger
 

data/lib/wurk/component.rb

@@ -63,8 +63,8 @@ module Wurk
       config.logger
     end
 
-    def redis(&)
-      config.redis(&)
+    def redis(idempotent: false, &)
+      config.redis(idempotent:, &)
     end
 
     def handle_exception(ex, ctx = {})

data/lib/wurk/configuration.rb

@@ -4,8 +4,10 @@ require 'etc'
 require 'logger'
 require_relative 'middleware/chain'
 require_relative 'capsule'
+require_relative 'pool_checkout'
 require_relative 'context'
 require_relative 'topology'
+require_relative 'redis_options'
 
 module Wurk
   # Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
@@ -169,8 +171,14 @@ module Wurk
 
     # --- Redis ------------------------------------------------------------
 
+    # Validated here, in the process running the initializer, rather than later
+    # in whichever process first builds a pool. The swarm's children are the ones
+    # that construct pools, so a bad key used to kill every child on boot while
+    # the parent stayed up and healthy — Running pod, passing probe, zero jobs
+    # processed (#283).
     def redis=(hash)
       guard_frozen!
+      RedisOptions.validate!(hash)
       @redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
     end
 
@@ -191,8 +199,8 @@ module Wurk
       build_redis_pool(size: size, name: name)
     end
 
-    def redis(&)
-      redis_pool.with(&)
+    def redis(idempotent: false, &)
+      PoolCheckout.with(redis_pool, idempotent, &)
     end
 
     # --- Web dashboard Redis pool ----------------------------------------