wurk 1.3.0 → 1.4.0

data/lib/wurk/process_set.rb

@@ -32,7 +32,7 @@ module Wurk
     # absence (process never registered) and expiry (heartbeat lapsed,
     # info field gone) both return nil.
     def self.[](identity)
-      exists, fields = Wurk.redis do |conn|
+      exists, fields = Wurk.redis(idempotent: true) do |conn|
         conn.pipelined do |pipe|
           pipe.call('SISMEMBER', Keys::PROCESSES, identity)
           pipe.call('HMGET', identity, *LOOKUP_FIELDS)
@@ -48,6 +48,11 @@ module Wurk
     # don't dogpile the prune. Returns the number of identities removed
     # (or 0 when the lock was held by someone else).
     #
+    # No apply-safety claim, unlike the reads around it: a replay re-decides
+    # which identities are dead, so it could prune one that registered in
+    # between and hasn't written `info` yet. A blip here just skips one prune —
+    # the next caller a minute later does it.
+    #
     # Spec: docs/target/sidekiq-free.md §31.17.
     def cleanup
       return 0 unless acquired_cleanup_lock?
@@ -84,7 +89,7 @@ module Wurk
     # SCARD over `processes`. Not pruned — may include identities whose
     # heartbeat has lapsed. Use `each` for the accurate count.
     def size
-      Wurk.redis { |conn| conn.call('SCARD', Keys::PROCESSES) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SCARD', Keys::PROCESSES) }
     end
 
     # Sum of `concurrency` across live processes. Iterates `each` so dead
@@ -104,7 +109,7 @@ module Wurk
     # `||=` with empty-string fallback distinguishes "leader is unset" from
     # "memoization not yet computed".
     def leader
-      @leader ||= Wurk.redis { |c| c.call('GET', 'dear-leader') } || ''
+      @leader ||= Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') } || ''
     end
 
     class << self
@@ -134,7 +139,7 @@ module Wurk
     private
 
     def fetch_each_rows
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         procs = conn.call('SMEMBERS', Keys::PROCESSES).sort
         next [] if procs.empty?
 
@@ -223,7 +228,7 @@ module Wurk
     # Compares identity against the `dear-leader` STRING. Ent-only;
     # always false in OSS/free.
     def leader?
-      Wurk.redis { |c| c.call('GET', 'dear-leader') == identity }
+      Wurk.redis(idempotent: true) { |c| c.call('GET', 'dear-leader') == identity }
     end
 
     private

data/lib/wurk/processor.rb

@@ -164,6 +164,12 @@ module Wurk
       job_hash = parse_or_kill(jobstr, uow)
       return if job_hash.nil?
 
+      # The fetcher never parses, so hand it the jid we just read: the ACK
+      # retires this job's poison-pill recovery counter inside the round trip
+      # it already makes. A fetcher plugged in via `config[:fetch_class]` has
+      # no jid slot and simply ACKs — the counter then ages out on its 72h TTL.
+      uow.jid = job_hash['jid'] if uow.respond_to?(:jid=)
+
       ack = false
       begin
         Thread.handle_interrupt(Wurk::Shutdown => :never) do

data/lib/wurk/profiler.rb

@@ -5,6 +5,7 @@ require 'zlib'
 require 'stringio'
 require 'tempfile'
 require_relative 'keys'
+require_relative 'pool_checkout'
 
 module Wurk
   # Job profiling (Sidekiq 8.0+, OSS). When a job is pushed with a `profile`
@@ -113,8 +114,8 @@ module Wurk
         end
       end
 
-      def with_pool(pool, &)
-        pool ? pool.with(&) : Wurk.redis(&)
+      def with_pool(pool, idempotent: false, &)
+        pool ? PoolCheckout.with(pool, idempotent, &) : Wurk.redis(idempotent:, &)
       end
 
       def now

data/lib/wurk/queue.rb

@@ -25,7 +25,7 @@ module Wurk
 
     # @return [Array<Queue>] one per known queue, sorted by name.
     def self.all
-      names = Wurk.redis { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
+      names = Wurk.redis(idempotent: true) { |conn| conn.call('SMEMBERS', Keys::QUEUES_SET) }
       names.sort.map { |n| new(n) }
     end
 
@@ -35,12 +35,12 @@ module Wurk
     end
 
     def size
-      Wurk.redis { |conn| conn.call('LLEN', @rname) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('LLEN', @rname) }
     end
 
     # Seconds since the oldest job (tail of LIST) was enqueued. 0.0 when empty.
     def latency
-      payload = Wurk.redis { |conn| conn.call('LRANGE', @rname, -1, -1).first }
+      payload = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, -1, -1).first }
       return 0.0 if payload.nil?
 
       JobRecord.latency_from(Wurk.load_json(payload)['enqueued_at'])
@@ -51,19 +51,19 @@ module Wurk
     # True iff this queue's name is a member of the `paused` SET. Wurk
     # implements the Pro contract for free; fetchers consult the same set.
     def paused?
-      Wurk.redis { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
+      Wurk.redis(idempotent: true) { |conn| conn.call('SISMEMBER', Keys::PAUSED_SET, @name) } == 1
     end
 
     # Pause new fetches against this queue. Idempotent — `SADD` returns
     # 0 when the name was already present. In-flight jobs are untouched.
     def pause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SADD', Keys::PAUSED_SET, @name) }
       true
     end
 
     # Resume fetches. Idempotent.
     def unpause! # rubocop:disable Naming/PredicateMethod
-      Wurk.redis { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
+      Wurk.redis(idempotent: true) { |conn| conn.call('SREM', Keys::PAUSED_SET, @name) }
       true
     end
 
@@ -74,7 +74,7 @@ module Wurk
       loop do
         start  = page * PAGE_SIZE
         stop   = start + PAGE_SIZE - 1
-        slice  = Wurk.redis { |conn| conn.call('LRANGE', @rname, start, stop) }
+        slice  = Wurk.redis(idempotent: true) { |conn| conn.call('LRANGE', @rname, start, stop) }
         slice.each { |value| yield JobRecord.new(value, @name) }
         break if slice.size < PAGE_SIZE
 
@@ -91,6 +91,9 @@ module Wurk
     # UNLINK the list + drop the queue from the `queues` set. Pipelined
     # so a partial failure leaves at most one of the two ops applied.
     # Method name is Sidekiq wire-compat — `clear?` would break the alias.
+    #
+    # Unlike pause!/unpause!, this one can't claim apply-safety: a replay after
+    # a lost reply would UNLINK whatever a producer enqueued in between.
     def clear # rubocop:disable Naming/PredicateMethod
       Wurk.redis do |conn|
         conn.pipelined do |pipe|

data/lib/wurk/rails_boot.rb

@@ -8,6 +8,11 @@ module Wurk
   # boot policy stays pure and unit-testable without the Railtie DSL.
   # See docs/idea/03-process-model.md for the exact ordering.
   module RailsBoot
+    # What `at_exit` waits for the supervise thread on top of the swarm's own
+    # drain budget: one supervise tick to notice the request, plus slack for
+    # the final reap.
+    DRAIN_JOIN_SLACK = 1
+
     module_function
 
     # Invoked from the `wurk.server_mode` initializer, before config/initializers
@@ -103,36 +108,62 @@ module Wurk
     end
 
     def boot_swarm
-      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology,
-                              shutdown_timeout: Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT)
+      timeout = Wurk.configuration[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT
+      swarm = Wurk::Swarm.new(topology: Wurk.configuration.topology, shutdown_timeout: timeout)
+      supervisor = nil
+      # Registered BEFORE boot, which forks the children one slot at a time: a
+      # fork that raises partway through would otherwise leave the children it
+      # already spawned with nothing to drain them on host exit. Children
+      # inherit the hook — stop_swarm no-ops off the process that forked them.
+      at_exit { stop_swarm(swarm, supervisor, timeout + Swarm::SHUTDOWN_GRACE + DRAIN_JOIN_SLACK) }
       # Co-hosted in the web process (e.g. Puma single mode): the host owns the
       # process-wide TERM/INT traps. Installing the swarm's own would hijack
       # them — a deploy TERM would drain the swarm but never stop the HTTP
       # server. Let the host keep signal ownership and drain the swarm on its
       # graceful exit (same contract as boot_embedded).
       swarm.boot(install_signals: false)
-      at_exit { swarm.shutdown }
       # supervise must still run somewhere or crashed children never respawn and
       # memory checks never fire. A background thread keeps the host's main
       # thread free to serve HTTP.
-      Thread.new do
+      supervisor = Thread.new do
         swarm.supervise
       rescue StandardError => e
         logger.error { "wurk supervisor thread died: #{e.class}: #{e.message}" }
       end
     end
 
+    # at_exit fires on the host's main thread, but the supervise thread owns the
+    # child table and two threads inside `shutdown` race on it — so request the
+    # drain and wait for the supervisor to run it. Draining here is the fallback
+    # for when no live supervisor will: it never started (boot raised) or it
+    # died early; after one that drained it finds no children and no-ops. A
+    # supervisor still alive past the join is wedged mid-drain — leave it be
+    # rather than race it; its children self-terminate (OrphanGuard) once this
+    # process is gone.
+    def stop_swarm(swarm, supervisor, drain_wait)
+      return unless swarm.owner?
+
+      swarm.request_shutdown
+      supervisor&.join(drain_wait)
+      swarm.shutdown unless supervisor&.alive?
+    end
+
     # Sidekiq-embedded parity: a threads-only worker inside the web process, no
     # fork. Redis validation failure keeps the host serving HTTP (log + carry
     # on). at_exit drains on the graceful shutdown the web server runs on TERM.
-    def boot_embedded
-      instance = Wurk::Embedded.new(Wurk.configuration)
-      instance.run
+    def boot_embedded(embedded = Wurk::Embedded)
+      instance = embedded.new(Wurk.configuration)
+      # Registered BEFORE run, for the reason boot_swarm registers before the
+      # fork: run brings the heartbeat, pollers, managers and health listener up
+      # one at a time, and a raise partway through would otherwise leave the ones
+      # already running with nothing to drain them on host exit.
       at_exit { instance.stop }
+      instance.run
       logger.info { 'wurk: running embedded in the web process (config.wurk.embed_in_web) — threads only, no fork' }
       instance
     rescue StandardError => e
       logger.error { "wurk: embedded boot failed: #{e.class}: #{e.message}" }
+      instance&.stop
       nil
     end
 

data/lib/wurk/redis_client_adapter.rb

@@ -21,13 +21,18 @@ module Wurk
 
     DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
 
+    # Every method here dispatches through `call` rather than `@client.call` so
+    # the round-trip odometer below sees method-style commands too — a host
+    # block doing `conn.sadd(...)` then `conn.lpush(...)` has to be as
+    # replay-protected as one written with `conn.call`. On the pipeline
+    # decorator `call` is the same buffered forward it always was.
     module CompatMethods
       def info
-        @client.call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
+        call('INFO') { |i| i.lines(chomp: true).map { |l| l.split(':', 2) }.select { |l| l.size == 2 }.to_h }
       end
 
       def evalsha(sha, keys, argv)
-        @client.call('EVALSHA', sha, keys.size, *keys, *argv)
+        call('EVALSHA', sha, keys.size, *keys, *argv)
       end
 
       # The Redis commands Sidekiq itself uses — defined eagerly so the
@@ -41,7 +46,7 @@ module Wurk
 
       USED_COMMANDS.each do |name|
         define_method(name) do |*args, **kwargs|
-          @client.call(name, *args, **kwargs)
+          call(name, *args, **kwargs)
         end
       end
 
@@ -52,7 +57,7 @@ module Wurk
         if DEPRECATED_COMMANDS.include?(args.first)
           warn("[sidekiq#5788] Redis has deprecated the `#{args.first}` command, called at #{caller(1..1)}")
         end
-        @client.call(*args, &)
+        call(*args, &)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
 
@@ -64,9 +69,48 @@ module Wurk
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
+      # Dispatch methods that put commands on the wire and wait for the reply.
+      # SCAN and friends are left out on purpose: they are pure reads, so
+      # re-running one applies nothing and cannot make a replay unsafe.
+      DISPATCH_METHODS = %i[call call_v call_once call_once_v
+                            blocking_call blocking_call_v pipelined multi].freeze
+
+      # Round trips this connection has completed, monotonic for its whole life.
+      #
+      # {Wurk::RedisPool} replays a failed block only while it can prove nothing
+      # in it applied, and a connect-phase error proves that for the command
+      # that raised — never for the ones before it. redis-client re-dials a
+      # dropped socket mid-block, so `CannotConnectError` surfaces on the second
+      # pipeline of a block whose first one already landed. The pool snapshots
+      # this counter around the block and refuses the replay once it has moved.
+      attr_reader :round_trips
+
+      def initialize(client)
+        super
+        @round_trips = 0
+      end
+
       def config
         @client.config
       end
+
+      # Counted *after* the call returns: a command that never reached a server
+      # leaves the odometer where it was, which is what keeps the pre-apply
+      # backoff alive for blocks that failed on their very first round trip.
+      DISPATCH_METHODS.each do |name|
+        class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
+          # def call(...)
+          #   result = super
+          #   @round_trips += 1
+          #   result
+          # end
+          def #{name}(...)
+            result = super
+            @round_trips += 1
+            result
+          end
+        RUBY
+      end
     end
   end
 end

data/lib/wurk/redis_options.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'redis-client'
+
+module Wurk
+  # Translates a Sidekiq-shaped `config.redis` hash into the exact keyword set
+  # redis-client accepts.
+  #
+  # Sidekiq normalizes the hash itself before handing it over
+  # (`sidekiq/redis_client_adapter.rb#client_opts`), so initializers in the wild
+  # carry keys redis-client has never known. Wurk used to splat the hash straight
+  # into `RedisClient.config`, which surfaced as
+  # `ArgumentError: unknown keyword: :network_timeout` — and only inside the
+  # forked children, which build their own pools (#283). The parent booted fine,
+  # the liveness probe passed, and the swarm respawn loop churned forever
+  # processing zero jobs. Hence `validate!`, called from Configuration#redis= so
+  # a bad hash raises in the parent where someone can actually see it.
+  #
+  # Reference: sidekiq 7.3 / 8.1 `client_opts` — namespace rejected,
+  # size/pool_timeout dropped, `network_timeout` → `timeout`, `master_name` →
+  # `name`, role/driver symbolized, `reconnect_attempts ||= 1`.
+  module RedisOptions
+    # Consumed by the pool layer (RedisPool / Capsule); never a socket concern.
+    POOL_KEYS = %i[size name pool_name pool_timeout on_error].freeze
+
+    # Accepted for Sidekiq parity, then dropped: Sidekiq used `logger` for its
+    # own "connecting to Redis with options ..." line and `cluster_safe` to
+    # unlock `:nodes`. redis-client has no keyword for either.
+    IGNORED_KEYS = %i[logger cluster_safe].freeze
+
+    # Sidekiq-only spellings this module rewrites into redis-client keywords.
+    TRANSLATED_KEYS = %i[network_timeout master_name].freeze
+
+    # The umbrella socket timeout, under both its names. `network_timeout` is
+    # the redis-rb-era spelling every "widen the timeouts for a slow/remote
+    # Redis" snippet still uses; `timeout` is redis-client's own.
+    UMBRELLA_TIMEOUT_KEYS = %i[network_timeout timeout].freeze
+
+    # Wurk splits the socket timeouts (#101) and passes all three explicitly, and
+    # redis-client lets an explicit `read_timeout` win over `timeout` — so
+    # forwarding the umbrella verbatim would silently drop the host's value.
+    # Fan it out instead; a host-supplied split timeout still wins over the fan-out.
+    SPLIT_TIMEOUT_KEYS = %i[connect_timeout read_timeout write_timeout].freeze
+
+    # Symbols in redis-client, strings in plenty of YAML-sourced configs.
+    SYMBOLIZED_KEYS = %i[driver role].freeze
+
+    # Keys that mean something in Sidekiq but have no Wurk equivalent. Raise
+    # naming the key and its replacement — the alternative is an opaque
+    # `unknown keyword:` from three layers down inside a forked child.
+    REJECTED_KEYS = {
+      namespace: 'Redis namespacing was dropped in Sidekiq 7 and Wurk never implemented it ' \
+                 '(docs/migrate-from-sidekiq.md §4). Give Wurk its own Redis database ' \
+                 '(redis://host:6379/1) or its own instance instead.',
+      nodes: 'Wurk does not run on Redis Cluster. Point config.redis at a single server with ' \
+             '`url:`, or at a Sentinel set with `sentinels:`.'
+    }.freeze
+
+    # Keyword parameter kinds in Method#parameters.
+    KEYWORD_PARAMS = %i[key keyreq].freeze
+
+    class << self
+      # The keyword hash for RedisClient.config / RedisClient.sentinel.
+      # `defaults` are Wurk's own socket defaults; everything the host supplied
+      # wins over them.
+      def normalize(options, defaults: {})
+        opts = symbolize(options)
+        validate!(opts)
+        opts = translate(opts)
+
+        # A default `url` is meaningless next to a sentinel set and actively
+        # harmful: SentinelConfig derives the master name and db from it.
+        defaults = defaults.except(:url) if sentinel?(opts)
+
+        defaults.merge(split_timeouts(opts), opts.except(*UMBRELLA_TIMEOUT_KEYS))
+      end
+
+      # Raises for anything redis-client would reject. Cheap and pure, so it runs
+      # in the parent (Configuration#redis=) as well as at pool-build time.
+      def validate!(options)
+        opts = symbolize(options)
+
+        REJECTED_KEYS.each do |key, hint|
+          raise ArgumentError, "config.redis[:#{key}] is not supported. #{hint}" if opts.key?(key)
+        end
+
+        unknown = opts.keys - known_keys
+        return if unknown.empty?
+
+        raise ArgumentError,
+              "config.redis: unknown option#{'s' if unknown.size > 1} " \
+              "#{unknown.map(&:inspect).join(', ')}. Supported keys: #{known_keys.sort.join(', ')}."
+      end
+
+      # Sentinel sets go through RedisClient.sentinel — RedisClient.config
+      # rejects `sentinels:` outright. Same routing Sidekiq does.
+      def sentinel?(client_config)
+        client_config.key?(:sentinels)
+      end
+
+      # Every keyword redis-client itself accepts, read off its own signatures so
+      # the list can't drift from the installed version. Config#initialize takes
+      # a **kwargs rest and forwards to Config::Common, so both have to be walked;
+      # SentinelConfig adds the sentinel-only keys.
+      def known_keys
+        @known_keys ||= [
+          ::RedisClient::Config, ::RedisClient::Config::Common, ::RedisClient::SentinelConfig
+        ].flat_map { |mod| keyword_params(mod) }.union(POOL_KEYS, IGNORED_KEYS, TRANSLATED_KEYS)
+      end
+
+      private
+
+      # Drop what redis-client has no keyword for, then rewrite the Sidekiq
+      # spellings it does have an equivalent for.
+      def translate(opts)
+        opts = opts.except(*POOL_KEYS, *IGNORED_KEYS)
+        opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
+        SYMBOLIZED_KEYS.each { |key| opts[key] = opts[key].to_sym if opts[key] }
+        opts
+      end
+
+      def keyword_params(mod)
+        mod.instance_method(:initialize).parameters.filter_map do |kind, key|
+          key if KEYWORD_PARAMS.include?(kind)
+        end
+      end
+
+      # The umbrella timeout fanned out across the three split ones. Returns {}
+      # when the host set neither, leaving Wurk's defaults in place.
+      def split_timeouts(opts)
+        umbrella = opts.values_at(*UMBRELLA_TIMEOUT_KEYS).compact.first
+        return {} unless umbrella
+
+        SPLIT_TIMEOUT_KEYS.to_h { |key| [key, umbrella] }
+      end
+
+      def symbolize(options)
+        options.transform_keys(&:to_sym)
+      end
+    end
+  end
+end