wurk 1.3.1 → 1.5.0

data/lib/wurk/client.rb

@@ -26,6 +26,29 @@ module Wurk
     SCHEDULED_BATCH_SIZE      = 100
     SPREAD_INTERVAL_FLOOR     = 5
 
+    # Batched (`bid`) payloads per EVALSHA pipeline. Ours, not Sidekiq's — it
+    # has no batches. Sized to DEFAULT_BATCH_SIZE so no existing caller's
+    # round-trip count moves: `push_bulk` already hands #raw_push at most that
+    # many payloads, so its batched pipeline stays exactly one round trip.
+    #
+    # The cap is for the one path that isn't pre-sliced: `autoflush = true`
+    # buffers a whole `Batch#jobs` block, so #flush_batched can be handed an
+    # unbounded payload set. Unsliced that is one pipeline holding every
+    # command and every reply in memory at once, and — Lua being atomic and
+    # single-threaded — one uninterrupted server-side sweep that blocks every
+    # other client for its duration. Same reasoning as the LIMIT on
+    # RELIABLE_SCHEDULE_PROMOTE.
+    BATCH_PIPELINE_SLICE      = 1_000
+
+    # Thread-local slot holding the payloads of the current push whose Redis
+    # write is confirmed applied. {Client::Buffered} subtracts them from the set
+    # it re-buffers when a *later* phase of the same push loses the connection,
+    # so an already-written job is never replayed into a second copy.
+    # Thread-local because one Client instance serves every producer thread;
+    # opened and closed by Buffered, the only reader, so an un-prepended Client
+    # pays a single nil check per write phase.
+    DELIVERED_KEY = :wurk_client_delivered
+
     attr_accessor :redis_pool
 
     def initialize(pool: nil, config: nil, chain: nil)
@@ -51,8 +74,8 @@ module Wurk
       return nil unless payload
 
       verify_json(payload)
-      raw_push([payload])
-      emit_enqueued([payload])
+      buffered = raw_push([payload])
+      emit_enqueued([payload], buffered)
       payload['jid']
     end
 
@@ -129,7 +152,21 @@ module Wurk
 
     private
 
+    # #push and #push_bulk verify at different points and both match Sidekiq
+    # exactly: push walks the payload the chain handed back (sidekiq
+    # client.rb:101 — normalize → middleware → verify → raw_push), bulk walks it
+    # inside the innermost block (sidekiq client.rb:165). Push used to do both,
+    # and since `strict_args_mode` defaults to :raise the second full recursive
+    # args walk was never skipped.
+    #
+    # Bulk keeps its walk inside the block on purpose: a client middleware that
+    # halts the job there short-circuits the walk, and hoisting it out would
+    # raise on args that middleware was about to drop.
     def invoke_chain(normed)
+      @chain.invoke(normed['class'], normed, normed['queue'], pool) { normed }
+    end
+
+    def invoke_chain_verified(normed)
       @chain.invoke(normed['class'], normed, normed['queue'], pool) do
         verify_json(normed)
         normed
@@ -169,8 +206,8 @@ module Wurk
         payloads = build_bulk_payloads(slice, base, ats)
         compacted = payloads.compact
         if compacted.any?
-          raw_push(compacted)
-          emit_enqueued(compacted)
+          buffered = raw_push(compacted)
+          emit_enqueued(compacted, buffered)
         end
         jids.concat(payloads.map { |p| p && p['jid'] })
       end
@@ -182,7 +219,7 @@ module Wurk
         item = base.merge('args' => job_args)
         item['at'] = ats[idx] if ats
         normed = normalize_item(item)
-        invoke_chain(normed)
+        invoke_chain_verified(normed)
       end
     end
 
@@ -222,16 +259,32 @@ module Wurk
     # Adds happen one payload at a time so an `autoflush = N` actually bounds
     # the pipeline size — a bulk push of 100 with N=2 must flush 2/2/... not
     # 100 in one shot.
+    #
+    # Returns the payloads it did NOT get to Redis: always nil here, since a
+    # plain Client either writes them all or raises. {Client::Buffered}
+    # overrides the contract — the payloads it diverted into the outage buffer
+    # come back so #push can keep them out of the enqueued metric.
     def raw_push(payloads)
       # Test modes short-circuit the Redis write (and the batch buffer): :fake
       # collects payloads in-memory, :inline runs them now. Client middleware
       # has already run by this point, matching Sidekiq.
-      return ::Wurk::Testing.dispatch_push(payloads) if ::Wurk::Testing.enabled?
+      if ::Wurk::Testing.enabled?
+        ::Wurk::Testing.dispatch_push(payloads)
+        return nil
+      end
 
       buffer = Thread.current[Wurk::Batch::BUFFER_KEY]
       return buffer_add(buffer, payloads) if buffer && payloads.all? { |p| p['bid'] && !p['at'] }
 
+      # No apply-safety claim: every command below appends (LPUSH, ZADD, the
+      # batch Lua's counters), so a block replayed after a lost reply is a
+      # second copy of the job. A post-write timeout raises out of here instead
+      # — {Client::Buffered} turns that into an outage-buffer entry, and a plain
+      # Client hands it to whoever called `perform_async`. The pool's pre-apply
+      # retry only fires while this block has landed nothing, so a queue group
+      # that already went out is never re-pushed by a replay.
       pool.with { |conn| atomic_push(conn, payloads) }
+      nil
     end
 
     # Batch autoflush path: accumulate each non-scheduled batched payload into
@@ -262,7 +315,10 @@ module Wurk
     # duplicate the scheduled entry.
     def push_scheduled_split(conn, payloads)
       batched, plain = payloads.partition { |j| j['bid'] }
-      conn.pipelined { |pipe| push_scheduled(pipe, plain) } unless plain.empty?
+      unless plain.empty?
+        conn.pipelined { |pipe| push_scheduled(pipe, plain) }
+        mark_delivered(plain)
+      end
       push_batched_scheduled_pipelined(conn, batched) unless batched.empty?
     end
 
@@ -282,10 +338,31 @@ module Wurk
     def push_immediate(conn, payloads)
       now = now_in_millis
       batched, plain = payloads.partition { |j| j['bid'] }
-      conn.pipelined { |pipe| push_plain(pipe, plain, now) } unless plain.empty?
+      push_plain(conn, plain, now) unless plain.empty?
       push_batched_pipelined(conn, batched, now) unless batched.empty?
     end
 
+    # One pipeline per BATCH_PIPELINE_SLICE payloads, each marked delivered the
+    # moment its reply is in — same contract as push_plain_group, and for the
+    # same reason: a slice that Redis already accepted must stay out of the
+    # reliable_push ledger, or a later slice's failure would report it as
+    # undelivered.
+    def push_batched_pipelined(conn, batched, now)
+      batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
+        eval_batched_slice(conn) { |pipe, eval_method| push_batched(pipe, slice, now, eval_method: eval_method) }
+        mark_delivered(slice)
+      end
+    end
+
+    # Same slicing and NOSCRIPT recovery as push_batched_pipelined, for the
+    # scheduled batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
+    def push_batched_scheduled_pipelined(conn, batched)
+      batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
+        eval_batched_slice(conn) { |pipe, eval_method| push_batched_scheduled(pipe, slice, eval_method: eval_method) }
+        mark_delivered(slice)
+      end
+    end
+
     # Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
     # code; the eager `script_load_all` after fork keeps the script cache
     # hot for the life of the connection. The retry uses EVAL (source-embedded)
@@ -293,45 +370,76 @@ module Wurk
     # NOSCRIPT a second time under heavy CI load (WorkerTest 3.4/7.2 flake).
     # `script_load_all` still primes the cache so the *next* pipeline returns
     # to the EVALSHA fast path.
-    def push_batched_pipelined(conn, batched, now)
-      conn.pipelined { |pipe| push_batched(pipe, batched, now) }
+    #
+    # Replaying the slice is safe precisely because every command in it is the
+    # same script: a flushed cache NOSCRIPTs all of them and applies none.
+    # Recovery is per slice, so the slices already acknowledged above are never
+    # re-sent. Mirrors Fetcher::Reliable#requeue_pipelined.
+    def eval_batched_slice(conn)
+      conn.pipelined { |pipe| yield(pipe, :eval_cached) }
     rescue RedisClient::CommandError => e
       raise unless e.message.to_s.start_with?('NOSCRIPT')
 
       Wurk::Lua::Loader.script_load_all(conn)
-      conn.pipelined { |pipe| push_batched(pipe, batched, now, eval_method: :eval_with_source) }
+      conn.pipelined { |pipe| yield(pipe, :eval_with_source) }
     end
 
-    # Same NOSCRIPT-recovery shape as push_batched_pipelined, for the scheduled
-    # batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
-    def push_batched_scheduled_pipelined(conn, batched)
-      conn.pipelined { |pipe| push_batched_scheduled(pipe, batched) }
-    rescue RedisClient::CommandError => e
-      raise unless e.message.to_s.start_with?('NOSCRIPT')
+    # One pipeline per queue, marked delivered the moment its reply is in.
+    # A push can die between groups — or land its whole plain phase and then
+    # lose the connection in the batched phase above — and whatever Redis
+    # already accepted must stay out of the reliable_push buffer; replaying it
+    # would enqueue a second copy of a job that ran fine.
+    #
+    # The group in flight when the socket drops stays unmarked and so is
+    # replayed: a lost reply is indistinguishable from a lost command, so that
+    # residual is at-least-once by construction. The split bounds it to one
+    # queue group per failed push instead of the entire payload set.
+    #
+    # Cost is a round trip per distinct queue. The single-queue push — every
+    # `perform_async`, every same-class `push_bulk` — still writes exactly the
+    # one SADD + LPUSH pipeline it did before.
+    #
+    # `uniform_queue` short-circuits the common case (one job, or many jobs
+    # all destined for the same queue) without paying for the `group_by`
+    # Hash + per-group Array allocations; only a genuinely mixed-queue batch
+    # falls through to grouping.
+    def push_plain(conn, payloads, now)
+      queue = uniform_queue(payloads)
+      return push_plain_group(conn, queue, payloads, now) if queue
 
-      Wurk::Lua::Loader.script_load_all(conn)
-      conn.pipelined { |pipe| push_batched_scheduled(pipe, batched, eval_method: :eval_with_source) }
+      payloads.group_by { |j| j['queue'] }.each { |q, jobs| push_plain_group(conn, q, jobs, now) }
     end
 
-    def push_plain(conn, payloads, now)
-      grouped = payloads.group_by { |j| j['queue'] }
-      conn.call('SADD', 'queues', *grouped.keys)
-      grouped.each do |queue, jobs|
-        serialized = jobs.map do |j|
-          j['enqueued_at'] = now
-          Wurk.dump_json(j)
-        end
-        conn.call('LPUSH', "queue:#{queue}", *serialized)
+    def uniform_queue(payloads)
+      first = payloads[0]['queue']
+      return first if payloads.size == 1
+
+      first if payloads.all? { |j| j['queue'] == first }
+    end
+
+    def push_plain_group(conn, queue, jobs, now)
+      serialized = jobs.map do |j|
+        j['enqueued_at'] = now
+        Wurk.dump_json(j)
+      end
+      conn.pipelined do |pipe|
+        pipe.call('SADD', 'queues', queue)
+        pipe.call('LPUSH', "queue:#{queue}", *serialized)
       end
+      mark_delivered(jobs)
     end
 
     # Batched jobs route through BATCH_PUSH: increments b-<bid> total+pending,
     # SADDs jid into the live set, registers the queue, LPUSHes the payload —
-    # all atomically. One Redis round-trip per job (no pipeline grouping)
-    # because the lua needs per-job KEYS bound. Acceptable cost: batch
-    # enqueue is not the hot path; correctness is. `eval_method` is the
+    # all atomically. The Lua binds per-job KEYS, so grouping N jobs into one
+    # EVALSHA isn't available; they ride one pipeline instead (the `conn` here
+    # is always the pipeline #push_batched_pipelined opened), so the cost is
+    # N commands and one round trip, not N round trips. `eval_method` is the
     # Wurk::Lua::Loader entry point (`:eval_cached` for the hot EVALSHA path,
-    # `:eval_with_source` for the EVAL-source retry).
+    # `:eval_with_source` for the EVAL-source retry) — neither reads the reply,
+    # which is what makes the pipelined form legal: `eval_cached`'s inline
+    # NOSCRIPT rescue can't fire against a buffered call, so recovery is the
+    # caller's finalize-time rescue.
     def push_batched(conn, payloads, now, eval_method: :eval_cached)
       payloads.each do |j|
         j['enqueued_at'] = now
@@ -341,7 +449,7 @@ module Wurk
           :batch_push,
           keys: ["b-#{j['bid']}", "b-#{j['bid']}-jids", "queue:#{j['queue']}", 'queues',
                  "b-#{j['bid']}-died", 'dead-batches'],
-          argv: [j['queue'], j['jid'], Wurk.dump_json(j), j['bid']]
+          argv: [j['queue'], j['jid'], Wurk.dump_json(j), j['bid'], Wurk::Batch::DEFAULT_EXPIRY_SECONDS]
         )
       end
     end
@@ -350,9 +458,9 @@ module Wurk
     # total/pending increment registers the job in its batch at creation, and
     # the ZADD defers it onto `schedule`. Payload is stripped of `at`/
     # `enqueued_at` exactly like push_scheduled — `enqueued_at` is stamped fresh
-    # at promotion, never while the job sits scheduled (spec §7.1). One Redis
-    # round-trip per job because the Lua binds per-job KEYS; scheduled batch
-    # enqueue is not the hot path.
+    # at promotion, never while the job sits scheduled (spec §7.1). Per-job
+    # KEYS again, so one EVALSHA per job — pipelined by
+    # #push_batched_scheduled_pipelined into one round trip per slice.
     def push_batched_scheduled(conn, payloads, eval_method: :eval_cached)
       payloads.each do |j|
         Wurk::Lua::Loader.public_send(
@@ -360,7 +468,8 @@ module Wurk
           conn,
           :batch_schedule,
           keys: ['schedule', "b-#{j['bid']}", "b-#{j['bid']}-jids"],
-          argv: [j['at'].to_s, Wurk.dump_json(j.except('enqueued_at', 'at')), j['jid']]
+          argv: [j['at'].to_s, Wurk.dump_json(j.except('enqueued_at', 'at')), j['jid'],
+                 Wurk::Batch::DEFAULT_EXPIRY_SECONDS]
         )
       end
     end
@@ -369,11 +478,36 @@ module Wurk
       @redis_pool || Thread.current[:wurk_via_pool] || @config.redis_pool
     end
 
+    # Record a write Redis has acknowledged, for {Client::Buffered} to subtract
+    # from what it re-buffers: one push spans several pipelines (plain vs
+    # batched, immediate vs scheduled), and the ledger is what keeps a group
+    # that already landed out of the buffer when a later group fails. It
+    # accumulates across pool attempts and is never pruned — #raw_push claims no
+    # apply-safety, and RedisPool refuses to replay such a block once one of its
+    # round trips has completed, so the only replay left starts from an empty
+    # ledger.
+    def mark_delivered(payloads)
+      Thread.current[DELIVERED_KEY]&.concat(payloads)
+    end
+
     # Best-effort `sidekiq.jobs.enqueued` counter — one increment per payload
     # that actually made it past middleware AND Redis. Tags follow the same
     # `worker:`/`queue:` shape as Wurk::Metrics::Statsd so dashboards built
     # for the server-side emissions work unchanged.
-    def emit_enqueued(payloads)
+    #
+    # `buffered` is what reliable_push swallowed into its outage buffer (see
+    # #raw_push). Those payloads are not enqueued: the ring buffer may still
+    # evict them, and the drain that does land one counts it then — so booking
+    # them here would inflate the counter on an outage and double-count every
+    # payload that later replays.
+    #
+    # Resolve the client once for the whole batch and bail before touching a
+    # payload: unconfigured is the common case, and the tags below cost two
+    # Strings and an Array per job for `increment` to immediately drop.
+    def emit_enqueued(payloads, buffered = nil)
+      return if Wurk::Metrics::Statsd.safe_client.nil?
+
+      payloads = reject_by_identity(payloads, buffered) if buffered && !buffered.empty?
       payloads.each do |p|
         Wurk::Metrics::Statsd.increment(
           'jobs.enqueued',
@@ -381,5 +515,14 @@ module Wurk
         )
       end
     end
+
+    # Set difference by object identity — `==` would fold two jobs carrying the
+    # same fields into one. Both sides are always the very Hash objects this
+    # push built, so identity is both exact and cheaper than hashing them.
+    def reject_by_identity(payloads, excluded)
+      seen = {}.compare_by_identity
+      excluded.each { |p| seen[p] = true }
+      payloads.reject { |p| seen.key?(p) }
+    end
   end
 end

data/lib/wurk/compat.rb

@@ -170,7 +170,7 @@ module Sidekiq
     def configure_client(&) = Wurk.configure_client(&)
     def configure_embed(&) = Wurk.configure_embed(&)
     def default_configuration = Wurk.default_configuration
-    def redis(&) = Wurk.redis(&)
+    def redis(idempotent: false, &) = Wurk.redis(idempotent:, &)
     def redis_pool = Wurk.redis_pool
     def logger = Wurk.logger
 

data/lib/wurk/component.rb

@@ -37,8 +37,34 @@ module Wurk
 
     # --- identity -------------------------------------------------------
 
+    # Base36 `thread.object_id ^ pid` — the id in every log line and the key
+    # each Processor publishes its in-flight job under. Constant for the life
+    # of a thread inside one process and read several times per job, so it is
+    # memoized per thread (frozen: it is used as a Hash key, and an unfrozen
+    # String key is duped on every store).
+    #
+    # The pid is memoized alongside it because the thread that calls fork keeps
+    # its thread-locals in the child, where the pid — and therefore the tid —
+    # has changed. Without the guard a forked child would report the parent's
+    # tid and collide with it in `<identity>:work`.
+    #
+    # Thread-local, not `Thread#[]`: the latter is fiber-local, so a job that
+    # runs inside a Fiber (or any Enumerator) would miss the memo and allocate
+    # a fresh String on every read — the identity the memo exists to cache is
+    # the thread's, and it does not change when a fiber does.
+    def self.tid
+      thread = Thread.current
+      memo = thread.thread_variable_get(:wurk_tid)
+      pid = ::Process.pid
+      return memo[1] if memo && memo[0] == pid
+
+      id = (thread.object_id ^ pid).to_s(36).freeze
+      thread.thread_variable_set(:wurk_tid, [pid, id].freeze)
+      id
+    end
+
     def tid
-      (Thread.current.object_id ^ ::Process.pid).to_s(36)
+      Component.tid
     end
 
     def hostname
@@ -63,8 +89,8 @@ module Wurk
       config.logger
     end
 
-    def redis(&)
-      config.redis(&)
+    def redis(idempotent: false, &)
+      config.redis(idempotent:, &)
     end
 
     def handle_exception(ex, ctx = {})
@@ -110,10 +136,16 @@ module Wurk
     # Spawns a named thread that runs `block` under `watchdog(name)`. The
     # parent must retain the returned Thread; otherwise GC may not, but
     # report_on_exception is disabled so we don't double-log on death.
+    #
+    # Priority resolution matches Sidekiq (component.rb:44-48): explicit
+    # argument, then `config.thread_priority`, then -1. Ruby's default of 0
+    # buys a 100ms timeslice; each negative step halves it, so -1 keeps a
+    # CPU-heavy capsule from starving its siblings for a whole tick.
     def safe_thread(name, priority: nil, &block)
+      resolved = priority || config.thread_priority || DEFAULT_THREAD_PRIORITY
       Thread.new do
         Thread.current.name = name
-        Thread.current.priority = priority || DEFAULT_THREAD_PRIORITY
+        Thread.current.priority = resolved
         Thread.current.report_on_exception = false
         watchdog(name, &block)
       end

data/lib/wurk/configuration.rb

@@ -4,6 +4,7 @@ require 'etc'
 require 'logger'
 require_relative 'middleware/chain'
 require_relative 'capsule'
+require_relative 'pool_checkout'
 require_relative 'context'
 require_relative 'topology'
 require_relative 'redis_options'
@@ -86,7 +87,16 @@ module Wurk
     #   config.dogstatsd = -> { Datadog::Statsd.new('host', 8125) }
     #
     # Spec: docs/target/sidekiq-pro.md §9.1.
-    attr_accessor :dogstatsd
+    attr_reader :dogstatsd
+
+    # Assignment drops Statsd's resolved-client memo. That memo caches the
+    # "nothing configured" answer too (so the no-client emit path allocates
+    # nothing), which means a builder wired up after the first emit would
+    # otherwise never be picked up.
+    def dogstatsd=(builder)
+      @dogstatsd = builder
+      Wurk::Metrics::Statsd.reset!
+    end
 
     def initialize(options = {})
       @options = deep_dup_defaults.merge(options)
@@ -198,8 +208,8 @@ module Wurk
       build_redis_pool(size: size, name: name)
     end
 
-    def redis(&)
-      redis_pool.with(&)
+    def redis(idempotent: false, &)
+      PoolCheckout.trusted(redis_pool, idempotent, &)
     end
 
     # --- Web dashboard Redis pool ----------------------------------------
@@ -242,6 +252,11 @@ module Wurk
       @directory[name] = instance
     end
 
+    # Memoizes on a miss, and the first miss can land long after boot (an
+    # extension resolved on its first tick), which is why `freeze!` leaves
+    # `@directory` writable: frozen, that lookup raised FrozenError instead of
+    # building the default. `register` — the host-facing half — still refuses
+    # writes past the freeze, so the closed surface is unchanged.
     def lookup(name, default_class = nil)
       @directory[name] ||= default_class&.new
     end
@@ -481,14 +496,39 @@ module Wurk
       mb&.positive? ? mb * 1024 : nil
     end
 
+    # The pre-fork half of `freeze!`, and the only half a forking parent can
+    # run: capsules stay writable until each child has applied its slot
+    # (ChildBoot#apply_slot_to_config) and opened its own Redis pools. What is
+    # left is slot-independent — the options Hash and every capsule's middleware
+    # chains — so the swarm parent settles it once and every child inherits the
+    # result copy-on-write instead of allocating and dirtying its own copy.
+    # Freezing the options here also makes a post-fork option write raise in the
+    # child that wrote it, rather than silently diverging from its siblings.
+    #
+    # `@directory` is deliberately left out — see #lookup.
+    def prepare_for_fork!
+      # The capsule every swarm child configures (ChildBoot reaches for it by
+      # name), and the one a client-only config builds on its first enqueue.
+      # Materialized here so its chains are shared rather than rebuilt N times
+      # — and so `freeze!` can't close `@capsules` around a name that is only
+      # ever resolved later, which turned that first resolution into a
+      # FrozenError on the frozen Hash.
+      default_capsule
+      @capsules.each_value(&:prepare_shared!)
+      @options.freeze
+      @frozen = true
+      self
+    end
+
+    # Guarded on the capsule table, not `@frozen`: a swarm child reaches this
+    # with `prepare_for_fork!` already run in its parent, so `@frozen` is true
+    # while the capsules it has just configured are still open.
     def freeze!
-      return self if @frozen
+      return self if @capsules.frozen?
 
+      prepare_for_fork!
       @capsules.each_value(&:freeze)
       @capsules.freeze
-      @options.freeze
-      @directory.freeze
-      @frozen = true
       self
     end
 

data/lib/wurk/context.rb

@@ -14,7 +14,7 @@ module Wurk
     # context is restored on exit, even if the block raises — safe to nest.
     def self.with(hash)
       prior = Thread.current[KEY]
-      Thread.current[KEY] = (prior || {}).merge(hash)
+      Thread.current[KEY] = prior ? prior.merge(hash) : hash
       yield
     ensure
       Thread.current[KEY] = prior