wurk 1.5.0 → 1.7.0

data/lib/wurk/fetcher/capped.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require_relative '../keys'
+require_relative '../lua'
+require_relative '../queue_slot'
+
+module Wurk
+  class Fetcher
+    # Everything the reliable fetcher does differently when one of its queues
+    # carries a global concurrency cap (`config.global_concurrency`).
+    #
+    # Prepended onto {Fetcher::Reliable}, so the separation is structural rather
+    # than a claim in a comment: an install that caps nothing resolves one
+    # boolean at boot and every fetch pass hands straight back to `super` — the
+    # unchanged loop — without touching another line in this file. The cost of
+    # the whole feature to everyone else is that one test and the frame it sits
+    # in, which is why `rake bench` unconfigured and `bench/command_count.rb` are
+    # the gate on this slice rather than an argument about it.
+    #
+    # The cap itself — the ledger of who is running what — is {Wurk::QueueSlot}
+    # and `lib/wurk/lua/queue_slot.lua`. This is only about *fetching* under one:
+    # asking the question in the same round trip that claims the job, and
+    # deciding what a worker does with a queue that says no.
+    module Capped
+      # How long a queue that answered "the cluster is at capacity" is skipped
+      # locally before this fetcher asks again.
+      #
+      # Without it a capped queue sitting at capacity costs a round trip per
+      # processor thread per pass, forever — the fetcher spins on a `no` that
+      # cannot change faster than a job finishes somewhere in the cluster. With
+      # it the cost is bounded at four questions a second per capped queue for
+      # the whole capsule, and the price is that capacity freed on another host
+      # is picked up up to a quarter second late. Much shorter and the round
+      # trips come back; much longer and a cap starts to behave like a throttle.
+      # Not a config knob — a cap is a ceiling on how much runs, never a promise
+      # about latency.
+      CAPPED_BACKOFF = 0.25
+
+      # The two of fetch_slot.lua's three answers a caller acts on. Its third,
+      # `1`, means the queue was empty — which costs and means exactly what an
+      # empty uncapped queue does, so nothing compares against it.
+      AT_CAPACITY = 0
+      CLAIMED = 2
+
+      # One capped queue's ceiling, its slot key, and the local clock that keeps
+      # a queue at capacity from costing a round trip per pass. Built once per
+      # capped queue at boot: the fetch path never reads a config Hash, and a cap
+      # cannot change under a running fetcher anyway (`global_concurrency=`
+      # refuses to run once the configuration is frozen).
+      #
+      # One per queue per fetcher, so a capsule's processor threads share it: a
+      # refusal is a statement about the cluster, not about the thread that heard
+      # it, and suppressing the retry for all of them is the whole saving. They
+      # race on `blocked_until` — the loser writes an equally fresh deadline, and
+      # the worst a torn read can cost is one extra round trip. A gate inherited
+      # across a fork can carry a stale deadline for the same reason it is
+      # harmless: at worst the child's first pass on that queue waits out a
+      # backoff it did not earn.
+      Gate = Struct.new(:capacity, :slot_key, :blocked_until) do
+        def blocked?(now) = now < blocked_until
+
+        def block!(now)
+          self.blocked_until = now + CAPPED_BACKOFF
+        end
+      end
+
+      # Resolved here and never again. The fetch path must not read a config
+      # Hash per pass to discover that this install caps nothing, so the answer
+      # is a boolean and a Hash of prebuilt gates, both settled while the
+      # capsule is materializing its fetcher (`Capsule#prepare!`).
+      def initialize(capsule)
+        super
+        @gates = capsule.config.global_concurrency.to_h do |name, capacity|
+          [name, Gate.new(capacity, Keys.queue_slot(name).freeze, 0.0)]
+        end.freeze
+        @capped = !@gates.empty?
+      end
+
+      private
+
+      # #walk for a capsule that caps at least one of its queues; every other
+      # install falls straight through to the loop this decorates.
+      #
+      # A capped queue at capacity is skipped *locally* — no round trip, and the
+      # pass carries on to the queues behind it. That is the fairness
+      # requirement: one full queue must never stop this worker serving the
+      # others, and must not be able to spin the fetcher either. Skipping is also
+      # what keeps the held ACK, which is only ever handed to a queue we are
+      # about to talk to.
+      def walk(queues)
+        return super unless @capped
+
+        ack = take_pending_ack
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        queues.each do |public_q|
+          # `[private list, unprefixed name]`, the tuple #queue_keys caches; the
+          # name is what a cap is declared against.
+          keys = queue_keys(public_q)
+          gate = @gates[keys[1]]
+          next if gate&.blocked?(now)
+
+          uow = gate ? claim(public_q, keys, gate, ack, now) : lmove(public_q, ack)
+          ack = nil
+          return uow if uow
+        end
+        wait_capped(queues, ack)
+      end
+
+      # The gated twin of #lmove: fetch_slot.lua answers the cap and performs the
+      # LMOVE in one call, so a capped queue costs the same single round trip an
+      # uncapped one does and the held ACK still rides in front of it. Bracketing
+      # the existing fetch with a separate acquire and release instead measured
+      # 2.2x-2.6x the uncapped cost (the slice's measurement doc); that shape is
+      # what this method exists to avoid.
+      #
+      # Claims the same apply-safety #lmove does, for the same reason: a claim
+      # whose reply was lost leaves the job in *this* process's private list
+      # un-ACKed, which the next boot's Reaper already reclaims, and the script's
+      # own replay arm refreshes this token's hold rather than counting it twice
+      # — the token is built here, before the retried block, so a replay carries
+      # the same one.
+      #
+      # A token per claim rather than per thread, because the matching release
+      # is deferred: an ACK handed back by a failed flush must not be able to
+      # take the hold of whatever this thread is running by then. See
+      # QueueSlot.claim_token.
+      def claim(public_q, keys, gate, ack, now)
+        priv, name = keys
+        token = QueueSlot.claim_token
+        script_keys = [gate.slot_key, public_q, priv]
+        status, payload = config.redis(idempotent: true) do |conn|
+          fetch_slot(conn, script_keys, gate.capacity, token, ack)
+        end
+        return hold(unit_of_work(public_q, priv, name, payload), gate.slot_key, token) if status == CLAIMED
+
+        # The refusal is the only answer worth remembering: it cannot change
+        # faster than a job finishes somewhere in the cluster, so re-asking on
+        # every pass is how a full queue turns into a spin. An empty queue is
+        # remembered by nobody, exactly as on the uncapped path.
+        gate.block!(now) if status == AT_CAPACITY
+        nil
+      rescue StandardError
+        # The window #lmove guards: the ACK left its slot but never reached
+        # Redis, so hand it back or a finished job runs a second time.
+        defer_ack(ack) if ack
+        raise
+      end
+
+      # The claim's other half — the bookkeeping the script's ZADD implies.
+      #
+      # The unit carries the slot it was admitted under so its release can name
+      # that slot without re-deriving anything on the way out
+      # (UnitOfWork#write_ack), and the process ledger carries it so the
+      # heartbeat keeps the hold alive under a job that outlives the TTL.
+      def hold(uow, slot_key, token)
+        QueueSlot::HELD.hold(token, slot_key)
+        uow.slot_key = slot_key
+        uow.slot_token = token
+        uow
+      end
+
+      # One EVALSHA, or that same EVALSHA behind the held ACK in one pipeline —
+      # the shape #lmove already uses, so the gate costs no round trip of its
+      # own either way.
+      def fetch_slot(conn, keys, capacity, token, ack)
+        argv = [capacity, token, QueueSlot::TTL_SECONDS]
+        return Wurk::Lua::Loader.eval_cached(conn, :fetch_slot, keys: keys, argv: argv) unless ack
+
+        Wurk::Lua::Loader.pipelined_eval(conn) do |pipe, eval_method|
+          ack.write_ack(pipe)
+          Wurk::Lua::Loader.public_send(eval_method, pipe, :fetch_slot, keys: keys, argv: argv)
+        end.last
+      end
+
+      # #walk's blocking fall-through, for a capsule that caps a queue. BLMOVE
+      # has no gated form — a script cannot block — so a capped queue is never
+      # the one we park on: the block goes to the first *uncapped* queue in fetch
+      # order, and when every fetchable queue is capped there is nothing to park
+      # on at all, so we idle the wall clock a timed-out block would have spent.
+      # Sidekiq's BasicFetch idles on the same terms.
+      #
+      # While a capped queue is sitting at capacity the block is shortened to the
+      # gate's own re-check, so capacity freed on another host is picked up in a
+      # backoff instead of a poll interval. An idle capsule keeps today's
+      # cadence: nothing is blocked, so nothing is shortened.
+      def wait_capped(queues, ack)
+        # A pass that skipped every capped queue never handed its ACK to
+        # anything. A blocking call cannot carry one, so give it back rather than
+        # drop it — an LREM that is never sent leaves a finished job in the
+        # private list for the next boot's reaper to run again.
+        defer_ack(ack) if ack
+        wait = capped_wait(queues)
+        open_q = queues.find { |public_q| gate_for(public_q).nil? }
+        return blmove(open_q, wait) if open_q
+
+        # Which also hands back the slot this thread's last job ran under: a
+        # held ACK carries that ZREM, and a thread idling out a backoff with no
+        # job in hand must not keep capacity somebody else could be using. The
+        # two cases are the same one — an ACK survives the loop above only when
+        # every queue was gate-blocked, and a gate-blocked queue is a capped
+        # queue, so there was never an uncapped one to park on either.
+        flush_pending_acks
+        sleep wait
+        nil
+      end
+
+      def capped_wait(queues)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        blocked = queues.any? { |public_q| gate_for(public_q)&.blocked?(now) }
+        blocked ? [poll_interval, CAPPED_BACKOFF].min : poll_interval
+      end
+
+      def gate_for(public_q)
+        @gates[queue_keys(public_q)[1]]
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/reliable.rb

@@ -2,10 +2,11 @@
 
 require 'socket'
 require_relative '../component'
+require_relative '../pid_cache'
 require_relative '../keys'
 require_relative '../lua'
 require_relative '../fetcher'
-require_relative '../middleware/poison_pill'
+require_relative 'capped'
 
 module Wurk
   class Fetcher
@@ -33,6 +34,12 @@ module Wurk
     # docs/target/sidekiq-free.md §15 (TIMEOUT=2).
     class Reliable < Fetcher
       include Component
+      # Global per-queue concurrency, as a decorator over #walk. Prepended
+      # rather than mixed in so there stays exactly one copy of the uncapped
+      # fetch loop — Capped#walk hands straight back to `super` for an install
+      # that caps nothing, which is a stronger guarantee that the unconfigured
+      # path is unchanged than a second loop kept in step by hand.
+      prepend Capped
 
       # Default BLMOVE block timeout; overridable via config.fetch_poll_interval.
       TIMEOUT = 2
@@ -53,55 +60,6 @@ module Wurk
       # immediately after #quiet, so this pause adds no drain latency.
       QUIET_PAUSE = 0.05
 
-      # Carries the public queue key, the raw (still-JSON) job payload, the
-      # capsule we use to reach Redis, and the fetcher that holds this unit's
-      # ACK until it can ride a pipeline. ACK removes from the private list;
-      # requeue pushes back to the public queue head so the job is next pulled.
-      # LREM count=1 is idempotent for our payloads since each job's JSON
-      # contains a unique `jid`.
-      #
-      # `queue_name` (the queue without its `queue:` prefix) and
-      # `private_queue` come off the fetcher's per-queue cache at build time:
-      # both are pure functions of the queue this unit came from, so deriving
-      # them here would be a per-job cost for a per-queue fact.
-      #
-      # `jid` is filled in by the Processor once it has parsed the payload —
-      # the fetcher never parses. It is only used to retire the job's
-      # poison-pill recovery counter, so an ACK without one is still a
-      # complete ACK.
-      UnitOfWork = Struct.new(:queue, :queue_name, :private_queue, :job, :config, :jid, :fetcher,
-                              keyword_init: true) do
-        # Deferred, never skipped: the LREM goes back to the fetcher, which
-        # pipelines it in front of the next fetch's LMOVE instead of spending a
-        # round trip of its own. Ordering against the job is unchanged — the
-        # LREM still happens only after success or retry handling (Pro §3.2) —
-        # so all that moves is the wall clock. Every path that stops fetching
-        # flushes first; see #flush_pending_acks.
-        def acknowledge
-          fetcher.defer_ack(self)
-        end
-
-        # Queue this unit's ACK into an already-open pipeline. The counter DEL
-        # rides the same round trip rather than taking one of its own: a
-        # per-job call would be a fetch+execute regression for the sake of a
-        # key that exists for roughly no jobs. See Middleware::PoisonPill.
-        #
-        # Sending it only for the jobs that own one is not available to us:
-        # whether a job was reclaimed lives in the counter, and a reclaimed
-        # payload is byte-identical to a first-attempt one (the job JSON is
-        # wire-frozen, so the reaper cannot flag it). Reading the counter to
-        # decide would spend the very round trip the DEL is riding for free.
-        def write_ack(pipe)
-          pipe.call('LREM', private_queue, 1, job)
-          job_jid = jid.to_s
-          Middleware::PoisonPill.clear_in(pipe, job_jid) unless job_jid.empty?
-        end
-
-        def requeue
-          config.redis { |conn| conn.call('RPUSH', queue, job) }
-        end
-      end
-
       # Class-level: the name is a pure function of the public queue and this
       # process's identity, and both the fetcher and the Reaper need it (the
       # fetcher's units carry the string #queue_keys built for them, so nothing
@@ -336,14 +294,10 @@ module Wurk
         end
       end
 
-      # One pipelined RELIABLE_REQUEUE EVALSHA per UoW. Mirrors
-      # Client#push_batched_pipelined: a pipelined EVALSHA surfaces NOSCRIPT
-      # only at finalize (never to eval_cached's inline rescue). Every command
-      # here is the same script, so a flushed cache fails all of them and
-      # applies none — recover by reloading once and replaying the whole
-      # pipeline via source-embedded EVAL.
-      def requeue_pipelined(conn, in_progress, eval_method: :eval_cached)
-        conn.pipelined do |pipe|
+      # One pipelined RELIABLE_REQUEUE EVALSHA per UoW; the flushed-script-cache
+      # replay is Lua::Loader.pipelined_eval's.
+      def requeue_pipelined(conn, in_progress)
+        Wurk::Lua::Loader.pipelined_eval(conn) do |pipe, eval_method|
           in_progress.each do |uow|
             Wurk::Lua::Loader.public_send(
               eval_method, pipe, :reliable_requeue,
@@ -352,11 +306,6 @@ module Wurk
             )
           end
         end
-      rescue RedisClient::CommandError => e
-        raise unless e.message.to_s.start_with?('NOSCRIPT')
-
-        Wurk::Lua::Loader.script_load_all(conn)
-        requeue_pipelined(conn, in_progress, eval_method: :eval_with_source)
       end
 
       # SMEMBERS of the `paused` SET, at most once per PAUSED_TTL per fetcher
@@ -418,7 +367,7 @@ module Wurk
       # fetcher and read this without a lock. A racing writer's entry can be
       # lost, and the next fetch rebuilds it.
       def queue_keys(public_q)
-        pid = ::Process.pid
+        pid = PidCache.pid
         if @queue_keys_pid != pid
           @queue_keys = {}
           @queue_keys_pid = pid
@@ -464,9 +413,12 @@ module Wurk
         raise
       end
 
-      def blmove(public_q)
+      # `timeout` overrides the block window for a caller that has a reason to
+      # come back sooner than a poll interval; Capped#wait_capped shortens it
+      # while a capped queue is sitting at capacity. nil keeps today's default.
+      def blmove(public_q, timeout = nil)
         priv, name = queue_keys(public_q)
-        timeout = poll_interval
+        timeout ||= poll_interval
         # Dedicated fetch pool, not the main one: a parked BLMOVE holds its slot
         # for the whole block window, so routing it here keeps idle fetchers from
         # starving the main pool's background loops (#101). redis-client's
@@ -494,3 +446,7 @@ module Wurk
     end
   end
 end
+
+# At the bottom, not the top: the file reopens the class defined above, so a
+# require before that definition would create it with the wrong superclass.
+require_relative 'unit_of_work'

data/lib/wurk/fetcher/unit_of_work.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative '../queue_slot'
+require_relative '../middleware/poison_pill'
+
+module Wurk
+  class Fetcher
+    # `Fetcher::Reliable` reopened, not a second definition: the fetch loop owns
+    # which job to claim, and this owns what a claimed job still has to write on
+    # its way out — the ACK, the poison-pill counter and the global-concurrency
+    # slot. Required from the bottom of `reliable.rb`, once that class exists,
+    # the way `lua.rb` requires its loader.
+    class Reliable < Fetcher
+      # Carries the public queue key, the raw (still-JSON) job payload, the
+      # capsule we use to reach Redis, and the fetcher that holds this unit's
+      # ACK until it can ride a pipeline. ACK removes from the private list;
+      # requeue pushes back to the public queue head so the job is next pulled.
+      # LREM count=1 is idempotent for our payloads since each job's JSON
+      # contains a unique `jid`.
+      #
+      # `queue_name` (the queue without its `queue:` prefix) and
+      # `private_queue` come off the fetcher's per-queue cache at build time:
+      # both are pure functions of the queue this unit came from, so deriving
+      # them here would be a per-job cost for a per-queue fact.
+      #
+      # `jid` is filled in by the Processor once it has parsed the payload —
+      # the fetcher never parses. It is only used to retire the job's
+      # poison-pill recovery counter, so an ACK without one is still a
+      # complete ACK.
+      #
+      # `slot_key`/`slot_token` name the global-concurrency slot this unit was
+      # admitted under, and are set only by Fetcher::Capped — nil is the whole
+      # uncapped path, which is why the release below is a field test rather
+      # than a question asked of Redis.
+      # LREM's count, pre-stringified. Every argument of every command on the
+      # ACK pipeline is then a String, which is what keeps it on
+      # {Wurk::CommandBuilder}'s allocation-free fast path; an Integer here
+      # would send the whole command back through redis-client's normalizer.
+      LREM_COUNT = '1'
+      private_constant :LREM_COUNT
+
+      UnitOfWork = Struct.new(:queue, :queue_name, :private_queue, :job, :config, :jid, :fetcher,
+                              :slot_key, :slot_token, keyword_init: true) do
+        # Deferred, never skipped: the LREM goes back to the fetcher, which
+        # pipelines it in front of the next fetch's LMOVE instead of spending a
+        # round trip of its own. Ordering against the job is unchanged — the
+        # LREM still happens only after success or retry handling (Pro §3.2) —
+        # so all that moves is the wall clock. Every path that stops fetching
+        # flushes first; see #flush_pending_acks.
+        def acknowledge
+          fetcher.defer_ack(self)
+        end
+
+        # Queue this unit's ACK into an already-open pipeline. The counter DEL
+        # rides the same round trip rather than taking one of its own: a
+        # per-job call would be a fetch+execute regression for the sake of a
+        # key that exists for roughly no jobs. See Middleware::PoisonPill.
+        #
+        # Sending it only for the jobs that own one is not available to us:
+        # whether a job was reclaimed lives in the counter, and a reclaimed
+        # payload is byte-identical to a first-attempt one (the job JSON is
+        # wire-frozen, so the reaper cannot flag it). Reading the counter to
+        # decide would spend the very round trip the DEL is riding for free.
+        def write_ack(pipe)
+          pipe.call('LREM', private_queue, LREM_COUNT, job)
+          job_jid = jid.to_s
+          Middleware::PoisonPill.clear_in(pipe, job_jid) unless job_jid.empty?
+          release_slot_in(pipe) if slot_key
+        end
+
+        # Give a global-concurrency slot back inside a pipeline the caller
+        # already has open — the ACK's, so a capped job costs no round trip more
+        # on the way out than an uncapped one does on the way in. The ZREM lands
+        # ahead of the fetch it is pipelined with, so a thread re-competing for
+        # the queue it just ran releases before it asks, never after.
+        #
+        # Idempotent by construction (QueueSlot#release): it names one member,
+        # so a replayed ACK frees nothing that has since been handed on. The
+        # member names this *claim* rather than this thread, which is what makes
+        # that true of a stale ACK too — one handed back by a failed flush
+        # (Fetcher::Reliable#restore_pending_acks) after the thread has already
+        # claimed its next slot on the same queue. See QueueSlot.claim_token.
+        def release_slot_in(pipe)
+          QueueSlot::HELD.drop(slot_token, slot_key)
+          pipe.call('ZREM', slot_key, slot_token)
+        end
+
+        # The same release for the one path that deliberately does not ACK:
+        # Wurk::Shutdown, where the payload stays in the private list to be
+        # requeued and re-run somewhere else. Whoever re-runs it needs the
+        # capacity, so this cannot wait out the TTL — and there is no ACK left
+        # to ride, hence the round trip. Bounded to the jobs a hard shutdown
+        # kills mid-flight.
+        def release_slot
+          return unless slot_key
+
+          QueueSlot::HELD.drop(slot_token, slot_key)
+          config.redis(idempotent: true) { |conn| conn.call('ZREM', slot_key, slot_token) }
+        end
+
+        def requeue
+          config.redis { |conn| conn.call('RPUSH', queue, job) }
+        end
+      end
+    end
+  end
+end