wurk 1.3.0 → 1.4.0

data/lib/wurk/redis_pool.rb

@@ -3,23 +3,36 @@
 require 'redis-client'
 require 'connection_pool'
 require_relative 'redis_client_adapter'
+require_relative 'redis_options'
 
 module Wurk
   # Per-process pool over redis-client + connection_pool. Never share a socket
   # across forks: the parent closes the pool before fork, each child opens a
   # fresh one (see docs/idea/03-process-model.md, steps 3 and 5).
   #
-  # #with absorbs transient Redis failures (production incident #101):
+  # #with absorbs transient Redis failures (production incident #101), but only
+  # where replaying the caller's block cannot change what the server already
+  # did — the block is arbitrary Ruby, so a replay re-issues every command in it:
   #   * READONLY / NOREPLICAS / UNBLOCKED — a failover happened; close and retry
   #     once immediately so redis-client redials the new primary (spec §26).
-  #   * ConnectionError (incl. CannotConnect / Read- / WriteTimeout) — a blip;
-  #     close and retry with exponential backoff up to CONN_MAX_ATTEMPTS, then
-  #     raise. At-least-once tolerates the rare duplicate and JobRetry re-runs
-  #     the job anyway, so retrying at the pool layer is strictly better.
+  #   * CannotConnect / Failover — raised while dialing, so the command that hit
+  #     one never reached a server and cannot have applied; close and retry with
+  #     exponential backoff up to CONN_MAX_ATTEMPTS, then raise.
+  #   * Read-/WriteTimeout and bare ConnectionError — the command may already
+  #     have applied server-side, so these raise. Replaying would double-push a
+  #     job, double-count a stat, or drop a member ZPOPed by the lost reply.
+  #     Blocks that are safe to re-run (pure reads, an LMOVE the reaper reclaims,
+  #     owner-CAS scripts) opt back into the backoff with `with(idempotent: true)`.
   #   * ConnectionPool::TimeoutError — checkout starved; retry once after a
   #     short jittered pause, then raise (sizing is the fix, not queuing).
-  # Every retry and final give-up is reported through the injected `on_error`
-  # telemetry hook (Wurk::Configuration#on_redis_error).
+  # Those proofs are about the command that raised, not the block around it: a
+  # block is several round trips, and redis-client re-dials mid-block, so a
+  # CannotConnect can surface on the second pipeline of a block whose first one
+  # already landed. So the pool also watches the connection's round-trip
+  # odometer (RedisClientAdapter::CompatClient#round_trips) and refuses to
+  # replay a non-idempotent block that has already completed one.
+  # Every retry, refused replay, and final give-up is reported through the
+  # injected `on_error` telemetry hook (Wurk::Configuration#on_redis_error).
   class RedisPool
     DEFAULT_URL  = ENV.fetch('REDIS_URL', 'redis://localhost:6379/0')
     DEFAULT_NAME = 'default'
@@ -31,18 +44,40 @@ module Wurk
     # wait above. read/write are deliberately wider than connect so a briefly-
     # slow-but-alive Redis (RDB fork pause, a large BLMOVE payload) doesn't
     # spuriously ReadTimeout — the production incident (#101) the single
-    # dual-use timeout caused. reconnect_attempts re-dials a dropped socket once.
+    # dual-use timeout caused. reconnect_attempts re-dials a dropped socket once;
+    # note that redis-client's re-dial also re-sends the one in-flight command,
+    # so the apply-safety split below bounds block replay, not command replay.
     DEFAULT_CONNECT_TIMEOUT    = 1.0
     DEFAULT_READ_TIMEOUT       = 2.5
     DEFAULT_WRITE_TIMEOUT      = 2.5
     DEFAULT_RECONNECT_ATTEMPTS = 1
 
+    # The floor every pool starts from; any key the host passed wins over it.
+    DEFAULT_CLIENT_CONFIG = {
+      url: DEFAULT_URL,
+      connect_timeout: DEFAULT_CONNECT_TIMEOUT,
+      read_timeout: DEFAULT_READ_TIMEOUT,
+      write_timeout: DEFAULT_WRITE_TIMEOUT,
+      reconnect_attempts: DEFAULT_RECONNECT_ATTEMPTS
+    }.freeze
+
     # Server-side messages where the connection is closed and the block retried
     # exactly once. READONLY is itself a RedisClient::ConnectionError subclass,
     # so this message match must be tested BEFORE the generic ConnectionError
     # backoff below (otherwise a failover would sleep instead of redialing).
     RETRYABLE_MSG = /\A(READONLY|NOREPLICAS|UNBLOCKED)/
 
+    # ConnectionErrors that can only be raised while dialing, so the block
+    # provably never applied and replaying it is safe whatever it contains.
+    # CannotConnect covers every connect-phase failure (redis-client converts a
+    # stalled handshake into it); Failover is the Sentinel resolver rejecting a
+    # server whose role changed. Any other ConnectionError — Read-/WriteTimeout
+    # or a reset mid-command — leaves the outcome unknown.
+    PRE_APPLY_ERRORS = [RedisClient::CannotConnectError, RedisClient::FailoverError].freeze
+
+    # The two retry_plan verdicts that re-run the block; the rest raise.
+    REPLAY_PLANS = %i[failover backoff].freeze
+
     # ConnectionError backoff: CONN_MAX_ATTEMPTS total tries, sleeping
     # (BASE * 2**attempt) + rand*JITTER before each retry. The 1.0s + 2.0s pair
     # rides out a sub-4s blip; the jitter de-syncs a fleet reconnecting at once.
@@ -60,9 +95,12 @@ module Wurk
 
     # Takes the standard Sidekiq `config.redis` hash: `pool_timeout` tunes the
     # ConnectionPool checkout; `connect_timeout`/`read_timeout`/`write_timeout`/
-    # `reconnect_attempts` plus any other key (driver, ssl_params, …) forward
-    # verbatim to RedisClient.config. `on_error` is an optional callable fired
-    # per retry / final give-up with { error:, attempt:, retried:, pool: }.
+    # `reconnect_attempts` plus any other redis-client key (driver, ssl_params,
+    # sentinels, …) reach the client. Sidekiq-only spellings (`network_timeout`,
+    # `master_name`, `logger`, …) are translated or dropped by {RedisOptions};
+    # a key redis-client would reject raises there with the key named.
+    # `on_error` is an optional callable fired per retry / final give-up with
+    # { error:, attempt:, retried:, pool: }.
     def initialize(size:, name: DEFAULT_NAME, on_error: nil, **options)
       @size          = size
       @name          = name
@@ -76,10 +114,14 @@ module Wurk
     # Checkout a connection and run the block. ConnectionPool::TimeoutError is
     # raised by @pool.with *before* the block runs, so it is caught out here
     # (the in-block #run rescue never sees it) — one retry, then raise.
-    def with(&block)
+    #
+    # `idempotent: true` asserts the block can be re-run after a command may
+    # already have applied server-side, which buys back the full ConnectionError
+    # backoff. Only claim it for pure reads or writes whose repeat is a no-op.
+    def with(idempotent: false, &block)
       checkout_retried = false
       begin
-        @pool.with { |conn| run(conn, &block) }
+        @pool.with { |conn| run(conn, idempotent, &block) }
       rescue ConnectionPool::TimeoutError => e
         if checkout_retried
           notify_error(e, attempt: 2, retried: false)
@@ -100,7 +142,7 @@ module Wurk
     # slot counts merged in — one call gives a heartbeat both Redis health and
     # local pool saturation. (Real Redis INFO has no `size`/`available` field.)
     def info
-      with { |conn| parse_info(conn.call('INFO')) }
+      with(idempotent: true) { |conn| parse_info(conn.call('INFO')) }
         .merge('size' => @size, 'available' => available)
     end
 
@@ -112,24 +154,29 @@ module Wurk
 
     private
 
-    # Socket config forwarded to RedisClient.config. Host-supplied keys win over
-    # the defaults; `pool_timeout` is dropped (it's a pool concern, not a socket
-    # one) and unknown keys pass straight through.
+    # Socket config forwarded to redis-client. RedisOptions owns the translation
+    # of the Sidekiq-shaped hash (network_timeout, master_name, pool-only keys,
+    # …) so this class stays about pooling; host-supplied keys win over the
+    # defaults.
     def build_client_config(options)
-      {
-        url: DEFAULT_URL,
-        connect_timeout: DEFAULT_CONNECT_TIMEOUT,
-        read_timeout: DEFAULT_READ_TIMEOUT,
-        write_timeout: DEFAULT_WRITE_TIMEOUT,
-        reconnect_attempts: DEFAULT_RECONNECT_ATTEMPTS
-      }.merge(options.except(:pool_timeout)).freeze
+      RedisOptions.normalize(options, defaults: DEFAULT_CLIENT_CONFIG).freeze
     end
 
     # Wrapped in the CompatClient decorator so `Sidekiq.redis { |c| c.smembers }`
     # method-style commands work like Sidekiq 7+ (#204). Wurk's own code paths
     # use #call, which the decorator forwards.
     def build_client
-      RedisClientAdapter::CompatClient.new(RedisClient.config(**@client_config).new_client)
+      RedisClientAdapter::CompatClient.new(redis_client_config.new_client)
+    end
+
+    # A Sentinel set is a different constructor, not a different keyword:
+    # `RedisClient.config(sentinels: [...])` raises. Sidekiq routes the same way.
+    def redis_client_config
+      if RedisOptions.sentinel?(@client_config)
+        RedisClient.sentinel(**@client_config)
+      else
+        RedisClient.config(**@client_config)
+      end
     end
 
     def safe_close(conn)
@@ -141,17 +188,19 @@ module Wurk
     # Runs the block on the checked-out `conn`, retrying transient RedisClient
     # errors in place: the same slot is reused across retries (redis-client
     # redials a closed socket lazily), so a busy fetcher can't leak checkouts.
-    def run(conn)
+    def run(conn, idempotent)
       attempts = 0
       begin
         attempts += 1
+        odometer = conn.round_trips
         yield conn
       rescue RedisClient::Error => e
-        plan = retry_plan(e, attempts)
+        plan = retry_plan(e, attempts, idempotent, conn.round_trips != odometer)
         raise if plan == :propagate
 
-        notify_error(e, attempt: attempts, retried: plan != :exhausted)
-        raise if plan == :exhausted
+        replaying = REPLAY_PLANS.include?(plan)
+        notify_error(e, attempt: attempts, retried: replaying)
+        raise unless replaying
 
         safe_close(conn)
         sleep(backoff_delay(attempts)) if plan == :backoff
@@ -159,19 +208,33 @@ module Wurk
       end
     end
 
-    # Pure classification of a RedisClient error against the attempt count:
+    # Pure classification of a RedisClient error against the attempt count, the
+    # caller's apply-safety claim, and whether this attempt already completed a
+    # round trip (`dirty`):
     #   :failover  → close + immediate retry (a primary swap)
     #   :backoff   → close + sleep + retry (a connection blip)
-    #   :exhausted → ConnectionError past the cap; report and give up
+    #   :unsafe    → something may have applied; report the blip, then raise
+    #   :exhausted → replayable ConnectionError past the cap; report and give up
     #   :propagate → not transient (or a spent failover); raise as-is
-    def retry_plan(err, attempts)
-      if RETRYABLE_MSG.match?(err.message.to_s)
-        attempts > 1 ? :propagate : :failover
-      elsif err.is_a?(RedisClient::ConnectionError)
-        attempts >= CONN_MAX_ATTEMPTS ? :exhausted : :backoff
-      else
-        :propagate
-      end
+    def retry_plan(err, attempts, idempotent, dirty)
+      failover = RETRYABLE_MSG.match?(err.message.to_s)
+      return :propagate unless failover || err.is_a?(RedisClient::ConnectionError)
+      return :unsafe unless replayable?(err, idempotent, dirty, failover)
+      return attempts > 1 ? :propagate : :failover if failover
+
+      attempts >= CONN_MAX_ATTEMPTS ? :exhausted : :backoff
+    end
+
+    # A replay re-issues the whole block, so one completed round trip voids
+    # every pre-apply proof: however provably the *failing* command missed the
+    # server, the ones ahead of it in the block did not. On a still-clean block
+    # a failover reply is proof enough by itself (the command was rejected
+    # outright); a bare ConnectionError needs one of the connect-phase classes.
+    def replayable?(err, idempotent, dirty, failover)
+      return true if idempotent
+      return false if dirty
+
+      failover || PRE_APPLY_ERRORS.any? { |klass| err.is_a?(klass) }
     end
 
     def notify_error(error, attempt:, retried:)

data/lib/wurk/scheduled.rb

@@ -6,6 +6,7 @@ require_relative 'lua'
 require_relative 'lua/loader'
 require_relative 'client'
 require_relative 'process_set'
+require_relative 'timer_loop'
 
 module Wurk
   # Promotes due jobs from the `retry` and `schedule` sorted sets back onto
@@ -57,13 +58,30 @@ module Wurk
         loop do
           break if @done
 
-          jobstr = @config.redis { |conn| Wurk::Lua::Loader.eval_cached(conn, :zpopbyscore, keys: [sset], argv: [now]) }
+          jobstr = pop_due(sset, now)
           break unless jobstr
 
           push_promoted(jobstr, sset)
         end
       end
 
+      # ZPOPBYSCORE is destructive and carries its result in the reply, so this
+      # block never claims apply-safety: a replay discards whatever the lost
+      # reply already removed. The pool therefore raises on a Read-/WriteTimeout,
+      # which leaves the outcome of *this* pop unknown — a due job may or may not
+      # have come off the ZSET. Report it and end this set's drain (the nil makes
+      # #drain_set break) rather than pop again blind; a job caught in that window
+      # falls into the same pop→push loss the default scheduler already documents
+      # on #push_promoted, and `reliable_scheduler!` (ReliableEnq) is the loss-free
+      # fix. Rescuing here rather than around #drain_set keeps the sibling set
+      # draining on this tick.
+      def pop_due(sset, now)
+        @config.redis { |conn| Wurk::Lua::Loader.eval_cached(conn, :zpopbyscore, keys: [sset], argv: [now]) }
+      rescue RedisClient::ConnectionError => e
+        handle_exception(e, { context: 'scheduler_pop', set: sset })
+        nil
+      end
+
       # A raising `@client.push` (bad payload, transient Redis error) must not
       # abort the drain and strand the remaining due jobs until the next poll —
       # rescue per-job, report, continue. The already-popped job IS lost here
@@ -169,13 +187,23 @@ module Wurk
 
       # Idempotent. Wakes the sleeping thread so it observes @done and exits.
       # Also propagates the stop signal to @enq so any in-flight drain loop
-      # short-circuits instead of running to completion.
+      # short-circuits instead of running to completion. Terminal, not a pause:
+      # @enq's stop flag is one-way, so this poller never polls again.
+      #
+      # Joins before returning — the caller (Launcher#quiet, then #stop) clears
+      # the heartbeat right after, and a sweep still in flight would promote
+      # jobs on behalf of a process that no longer exists.
+      #
+      # Cleared only on a confirmed join (Thread#join returns nil on timeout):
+      # a wedged sweep must stay tracked so #start's ||= guard returns it
+      # rather than spawning a second scheduler thread alongside it.
       def terminate
         @mutex.synchronize do
           @done = true
           @enq.terminate
           @sleeper.signal
         end
+        @thread = nil if @thread&.join(TimerLoop::JOIN_TIMEOUT)
       end
 
       # Called on every wake. Any raise inside the Enq is reported and the

data/lib/wurk/stats.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'date'
+require_relative 'pool_checkout'
 
 module Wurk
   # Read-only inspector for cluster state in Redis. The cheap counters are
@@ -38,7 +39,7 @@ module Wurk
     # Sum of the `busy` HASH field across every live process identity.
     # Pipelined but unbounded by process count.
     def workers_size
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         identities = conn.call('SMEMBERS', Keys::PROCESSES)
         next 0 if identities.empty?
 
@@ -55,7 +56,7 @@ module Wurk
     # and dashboards reading this rely on that order. Match it exactly with
     # `sort_by { |_, size| -size }`.
     def queues
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
         next {} if names.empty?
 
@@ -71,7 +72,7 @@ module Wurk
     # (`queue_summaries.sort_by { |qd| -qd.size }`) — this feeds the
     # dashboard's queue table (api_controller#queues).
     def queue_summaries
-      Wurk.redis do |conn|
+      Wurk.redis(idempotent: true) do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
         next [] if names.empty?
 
@@ -89,13 +90,17 @@ module Wurk
     # Latency (secs) of the `default` queue — the most-asked-about gauge.
     def default_queue_latency
       now_ms = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-      payload = Wurk.redis { |c| c.call('LRANGE', Keys.queue('default'), -1, -1) }.first
+      payload = Wurk.redis(idempotent: true) { |c| c.call('LRANGE', Keys.queue('default'), -1, -1) }.first
       compute_latency(payload, now_ms)
     end
 
     # Resets the named global counters. With no args, clears `processed`,
     # `failed`, and `expired`. SET … 0 (not DEL — keeps the key around so
     # reads stay `Integer` not `nil`).
+    #
+    # The only write here, and the only block in this class that can't claim
+    # apply-safety: a replay after a lost reply would re-zero the counters,
+    # discarding whatever the fleet counted in between.
     def reset(*stats)
       all      = %w[failed processed expired]
       to_clear = stats.empty? ? all : all & stats.flatten.map(&:to_s)
@@ -123,7 +128,7 @@ module Wurk
     private_constant :FAST_QUERIES, :FAST_KEYS
 
     def fetch_stats_fast!
-      raw = Wurk.redis do |conn|
+      raw = Wurk.redis(idempotent: true) do |conn|
         conn.pipelined { |pipe| FAST_QUERIES.each { |args| pipe.call(*args) } }
       end
       @stats = FAST_KEYS.zip(raw.map(&:to_i)).to_h
@@ -179,17 +184,17 @@ module Wurk
 
       def date_stat_hash(stat)
         keys = (0...@days_previous).map { |i| (@start_date - i).strftime('%Y-%m-%d') }
-        values = with_redis do |conn|
+        values = with_redis(idempotent: true) do |conn|
           conn.pipelined { |pipe| keys.each { |d| pipe.call('GET', "stat:#{stat}:#{d}") } }
         end
         keys.zip(values.map(&:to_i)).to_h
       end
 
-      def with_redis(&)
+      def with_redis(idempotent: false, &)
         if @pool
-          @pool.with(&)
+          PoolCheckout.with(@pool, idempotent, &)
         else
-          Wurk.redis(&)
+          Wurk.redis(idempotent:, &)
         end
       end
     end

data/lib/wurk/swarm/child_boot.rb

@@ -2,6 +2,7 @@
 
 require_relative '../component'
 require_relative '../launcher'
+require_relative '../client/buffered'
 require_relative '../fetcher/reliable'
 require_relative '../lua'
 require_relative 'orphan_guard'
@@ -106,8 +107,19 @@ module Wurk
 
       def reconnect_after_fork
         @config.reset_redis_pools!
+        # The reliable_push outage buffer, its drainer thread and its mutexes
+        # are process-global and were copied wholesale from the parent. The
+        # Process._fork hook normally beats us to it (making this a no-op) —
+        # the explicit call keeps the swarm path deterministic and ordered
+        # after the pool reset, so a re-armed drainer can only ever see the
+        # child's own pool.
+        Wurk::Client::Buffered.reset_after_fork!
         validate_redis!
         reconnect_active_record
+        # The dogstatsd client is memoized at the class level (Statsd.client),
+        # so without a reset every child would share the parent's UDP socket
+        # and thread-locals instead of building its own after fork.
+        Wurk::Metrics::Statsd.reset!
       end
 
       # Prove the child's fresh Redis socket reaches a live server before it