cogworker 0.4.1 → 0.5.1

data/lib/cogworker/periodic/release_middleware.rb

@@ -23,14 +23,25 @@ module Cogworker
         begin
           yield
         rescue Exception => e # rubocop:disable Lint/RescueException
-          RunningLock.release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
+          release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
           raise e
         end
-        RunningLock.release(pjid, job['jid'])
+        release(pjid, job['jid'])
       end
 
       private
 
+      # Best-effort — a release that fails (Redis away) never turns the
+      # finished run into a failure — and retried later (DeferredReleases)
+      # rather than left to the lock's TTL.
+      def release(pjid, jid)
+        released = BestEffort.call('Periodic lock') do
+          RunningLock.release(pjid, jid)
+          true
+        end
+        DeferredReleases.add(RedisKeys.periodic_running(pjid), jid) unless released
+      end
+
       # Best-effort: failing here must not stop the job from running (it
       # would be routed to retry/dead as if it had failed). If it does fail,
       # the lock just keeps its longer queued TTL until this process's next

data/lib/cogworker/periodic/running_lock.rb

@@ -61,6 +61,18 @@ module Cogworker
         OwnedKey.expire(RedisKeys.periodic_running(pjid), jid, ttl, conn)
       end
 
+      RETAKE_SCRIPT = RETAKE_FUNCTION + <<~LUA
+        retake_running_lock(KEYS[1], ARGV[1], ARGV[2])
+        return 1
+      LUA
+
+      # Extends the lock if still `jid`'s, takes it if it's gone, leaves it
+      # if another run holds it — for a run still going whose lock may have
+      # lapsed (Redis away longer than `active_ttl`).
+      def retake(pjid, jid, ttl, conn)
+        LuaScript.run(conn, RETAKE_SCRIPT, keys: [RedisKeys.periodic_running(pjid)], argv: [jid, ttl.to_f.ceil])
+      end
+
       def release(pjid, jid, conn = nil)
         OwnedKey.delete(RedisKeys.periodic_running(pjid), jid, conn)
       end

data/lib/cogworker/periodic/ticker.rb

@@ -20,12 +20,16 @@ module Cogworker
       # Undoes a won claim whose push then failed, so the slot isn't lost:
       # puts `last_slot` back (only if it still is this slot — a later slot
       # may have been claimed since) and drops the per-slot NX lock.
-      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
-      # previous last_slot ("" if none: then slot - 1, which still lets this
-      # slot fire, unlike deleting the key — with catch_up off, a missing
-      # last_slot would be re-primed to this very slot instead).
+      # KEYS[1] = last_slot, KEYS[2] = slot lock, KEYS[3] = running lock;
+      # ARGV[1] = slot, ARGV[2] = previous last_slot ("" if none: then
+      # slot - 1, which still lets this slot fire, unlike deleting the key —
+      # with catch_up off, a missing last_slot would be re-primed to this
+      # very slot instead), ARGV[3] = the jid the claim took the running lock
+      # under. Releasing that lock happens here too, in the same step: as a
+      # separate call made first, its failure (Redis down — the usual reason
+      # the push failed) skipped the rollback altogether.
       ROLLBACK_SCRIPT = <<~LUA
-        if redis.call("GET", KEYS[1]) == ARGV[1] then
+        if redis.pcall("GET", KEYS[1]) == ARGV[1] then
           if ARGV[2] == "" then
             redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
           else
@@ -33,6 +37,12 @@ module Cogworker
           end
         end
         redis.call("DEL", KEYS[2])
+        -- Best-effort (an entry without until_executed has no lock; one of
+        -- the wrong type mustn't make an applied rollback count as failed
+        -- and be retried forever).
+        if redis.pcall("GET", KEYS[3]) == ARGV[3] then
+          redis.pcall("DEL", KEYS[3])
+        end
         return 1
       LUA
 
@@ -44,10 +54,20 @@ module Cogworker
         @last_checked_slot = {}
         @cron_cache = {}
         @pending_rollbacks = []
+        @blocked_since = {} # pjid => [slot, monotonic time it was first found blocked]
+        @failures = Hash.new(0) # pjid => consecutive failed ticks
+        @retry_at = {}          # pjid => monotonic time it may be tried again
       end
 
+      # Publishing is only for restart survival and the Web UI: its failure
+      # (Redis down at boot, the key holding the wrong type) is logged, and
+      # must not take the process down with it.
       def start!
-        publish_schedule!
+        begin
+          publish_schedule!
+        rescue StandardError => e
+          Cogworker.logger.error { "Periodic schedule not published: #{e.class}: #{e.message}" }
+        end
         @thread = Thread.new { run }
       end
 
@@ -94,72 +114,161 @@ module Cogworker
 
       def tick
         retry_pending_rollbacks
+        # This process's own delayed lock releases first (see BLOCKED_GRACE).
+        BestEffort.call('Deferred lock releases') { Cogworker.config.redis { |c| DeferredReleases.retry_all(c) } }
         now = Time.now
         @entries.each do |entry|
-          slot = cron_for(entry).previous_time(now).to_i
-          next if @last_checked_slot[entry.pjid] == slot
+          next if backing_off?(entry.pjid)
+
+          next if tick_entry(entry, now) == :skipped # (no Redis call made: proves nothing)
 
-          jid = SecureRandom.hex(12)
-          previous = claim(entry, slot, jid) unless disabled?(entry)
-          enqueue(entry, slot, jid, previous) if previous
-          @last_checked_slot[entry.pjid] = slot
+          @failures.delete(entry.pjid)
+          RedisErrors.recovered('Periodic tick')
+        rescue StandardError => e
+          # Per entry: one entry failing (its keys of the wrong type, its
+          # push failing) used to skip every entry after it, on every tick.
+          # Its slot isn't recorded as checked, so it's retried — after an
+          # exponential pause (`back_off`), not on every 5s tick, which made a
+          # deterministically failing entry hammer Redis and the log. Not for
+          # a lost connection, though: that's Redis being away, not the
+          # entry failing — backing off every entry on it left cron idle for
+          # minutes after Redis came back.
+          delay = RedisErrors.unavailable?(e) ? TICK_INTERVAL : back_off(entry.pjid)
+          RedisErrors.report('Periodic tick', e) { "#{entry.pjid} failed (next try in #{delay}s)" }
         end
       end
 
-      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
-      # step entirely, short-circuiting before `claim` even runs, so
-      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
-      # while disabled: the Web UI's own "LastRun" column keeps showing the
-      # last time it *actually* ran, not a due-but-skipped slot, and
-      # re-enabling doesn't trigger a catch-up burst for every slot that
-      # was silently skipped in between. `@last_checked_slot` (this one
-      # Ticker instance's own in-memory dedup, unrelated to the Redis-
-      # persisted last_slot) still advances either way, exactly as it
-      # already did before this entry ever had a disabled state — it only
-      # stops this same tick loop from re-evaluating the same slot twice.
+      BACKOFF_MAX = 300
+
+      def backing_off?(key)
+        (at = @retry_at[key]) && ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < at
+      end
+
+      # 5s, 10s, 20s … up to BACKOFF_MAX; reset by the next success.
+      def back_off(key)
+        @failures[key] += 1
+        delay = [TICK_INTERVAL * (2**(@failures[key] - 1)), BACKOFF_MAX].min
+        @retry_at[key] = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + delay
+        delay
+      end
+
+      def tick_entry(entry, now)
+        slot = cron_for(entry).previous_time(now).to_i
+        return :skipped if @last_checked_slot[entry.pjid] == slot
+
+        jid = SecureRandom.hex(12)
+        previous = claim(entry, slot, jid) unless disabled?(entry)
+        return if previous == :blocked && still_blocked_in_grace?(entry.pjid, slot)
+
+        enqueue(entry, slot, jid, previous) if previous && previous != :blocked
+        @last_checked_slot[entry.pjid] = slot
+        @blocked_since.delete(entry.pjid)
+      end
+
+      # A slot blocked by the entry's running lock is retried for
+      # BLOCKED_GRACE before it counts as skipped. After an outage, the lock
+      # may still be held only because its finished run's release was
+      # deferred (DeferredReleases — retried on the next heartbeat, in
+      # whichever process ran it); marking the slot checked straight away
+      # lost it, in every process, until the entry's next slot — an hour,
+      # for an hourly entry. A run genuinely still going keeps the lock past
+      # the grace, and its slot is skipped as `until_executed` intends.
+      BLOCKED_GRACE = 3 * Heartbeat::INTERVAL
+
+      def still_blocked_in_grace?(pjid, slot)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        since_slot, since = @blocked_since[pjid]
+        @blocked_since[pjid] = [slot, since = now] unless since_slot == slot
+        now - since < BLOCKED_GRACE
+      end
+
       # The usual reason a push fails — Redis unreachable — makes this fail
       # too; it's then kept and retried at the start of every tick until it
       # goes through (before that tick looks at the slot again), rather than
-      # just logged and the slot lost. (A push that timed out *after* Redis
-      # had applied it is the one case this turns into a second run of the
-      # slot: indistinguishable from one that never arrived.)
-      def rollback_claim(entry, slot, previous_slot)
+      # just logged and the slot lost. Returns true, or the error. (A push
+      # that timed out *after* Redis had applied it is the one case this
+      # turns into a second run of the slot: indistinguishable from one that
+      # never arrived.) Pending
+      # rollbacks live in this process's memory only: one still pending when
+      # the process restarts is lost — that slot is skipped, and an
+      # `until_executed` entry stays locked until its lock's TTL.
+      def rollback_claim(entry, slot, previous_slot, jid)
         Cogworker.config.redis do |c|
-          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
-          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot),
+                  RedisKeys.periodic_running(entry.pjid)]
+          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot, jid])
         end
         true
       rescue StandardError => e
-        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
-        @pending_rollbacks << [entry, slot, previous_slot]
-        false
+        RedisErrors.report('Periodic rollback', e) { "#{entry.pjid}: couldn't roll back slot #{slot} (will retry)" }
+        @pending_rollbacks << [entry, slot, previous_slot, jid]
+        e # (truthy, but not `true`: the caller tells a lost connection from a failing rollback)
       end
 
+      # With the same backoff as a failing entry (keyed by the rollback): a
+      # rollback that keeps failing is retried less and less often, but
+      # always before that entry's slot is looked at again.
       def retry_pending_rollbacks
         pending = @pending_rollbacks
         @pending_rollbacks = []
-        pending.each { |args| rollback_claim(*args) }
+        pending.each do |args|
+          key = "rollback:#{args[0].pjid}:#{args[1]}"
+          next @pending_rollbacks << args if backing_off?(key)
+
+          result = rollback_claim(*args)
+          if result == true
+            @failures.delete(key)
+            @retry_at.delete(key)
+            RedisErrors.recovered('Periodic rollback')
+          elsif !RedisErrors.unavailable?(result)
+            back_off(key) # only a rollback failing on its own backs off; see #tick
+          end
+        end
       end
 
+      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
+      # step entirely, short-circuiting before `claim` even runs, so
+      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
+      # while disabled: the Web UI's own "LastRun" column keeps showing the
+      # last time it *actually* ran, not a due-but-skipped slot, and
+      # re-enabling doesn't trigger a catch-up burst for every slot that
+      # was silently skipped in between. `@last_checked_slot` (this one
+      # Ticker instance's own in-memory dedup, unrelated to the Redis-
+      # persisted last_slot) still advances either way, exactly as it
+      # already did before this entry ever had a disabled state — it only
+      # stops this same tick loop from re-evaluating the same slot twice.
+      #
+      # A disabled-set that can't be read (the key holding the wrong type)
+      # counts as "not disabled", logged: it used to fail every entry's tick,
+      # stopping all cron. Losing a Disable is the lesser evil. A dropped
+      # connection still raises (the tick is then retried as a whole).
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
+      rescue StandardError => e
+        raise if RedisErrors.unavailable?(e)
+
+        Cogworker.logger.error { "Periodic: can't read #{RedisKeys::PERIODIC_DISABLED} (#{e.class}: #{e.message}); treating #{entry.pjid} as enabled" }
+        false
       end
 
       def cron_for(entry)
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      # nil if this process didn't win the slot; otherwise the previous
-      # `last_slot` value ("" if none), for #rollback_claim.
+      # nil if this process didn't win the slot, :blocked if the entry's
+      # running lock stopped it; otherwise the previous `last_slot` value
+      # ("" if none), for #rollback_claim.
       def claim(entry, slot, jid)
         return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           LuaScript.run(c, CLAIM_SCRIPT,
-                 keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
-                        RedisKeys.periodic_lock(entry.pjid, slot)],
-                 argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
+                        keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
+                               RedisKeys.periodic_lock(entry.pjid, slot)],
+                        argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
+        return :blocked if result == 2
+
         result.is_a?(Array) ? result[1].to_s : nil
       end
 
@@ -201,8 +310,7 @@ module Cogworker
         pushed = begin
           Client.push(job)
         rescue StandardError
-          RunningLock.release(entry.pjid, jid) if entry.until_executed?
-          rollback_claim(entry, slot, previous_slot)
+          rollback_claim(entry, slot, previous_slot, jid)
           raise
         end
         return unless entry.until_executed?

data/lib/cogworker/process_set.rb

@@ -3,6 +3,9 @@
 require 'json'
 
 module Cogworker
+  # Every live worker process, as published by its Heartbeat — one
+  # Cogworker::Process per `cogworker:processes` member whose presence key
+  # still exists (stale members are pruned while iterating).
   class ProcessSet
     include Enumerable
 

data/lib/cogworker/processor.rb

@@ -56,9 +56,9 @@ module Cogworker
     def run
       until @manager.stopping?
         begin
-          process_one
+          RedisErrors.recovered('Processor') unless process_one == :idle
         rescue StandardError => e
-          Cogworker.logger.error { "Processor error: #{e.class}: #{e.message}" }
+          RedisErrors.report('Processor', e)
           sleep(ERROR_BACKOFF)
         end
       end
@@ -67,7 +67,7 @@ module Cogworker
     def process_one
       if @manager.quiet?
         sleep(0.5)
-        return
+        return :idle # (no Redis call made)
       end
 
       work = begin
@@ -86,14 +86,24 @@ module Cogworker
       return fetcher.give_back(work) if @manager.stopping?
 
       @manager.processor_busy!
+      @manager.job_started(work.raw_job)
       @busy = true
-      disposition = :give_back
+      @ran = nil
+      @performed = false
+      disposition = nil
       begin
         disposition = execute(work)
       ensure
         @busy = false
+        @manager.job_performed(work.raw_job) unless @performed
         @manager.processor_idle!
-        settle(work, disposition)
+        # `execute` raising leaves `disposition` unset: before the job ran
+        # that's a hand-back; once it ran, whatever its outcome dictates
+        # (`@ran`, set the moment `run_job` returns) — so an unexpected
+        # exception later in the bookkeeping (a bug, a misbehaving logger)
+        # can't re-queue a job that already ran.
+        settle(work, disposition || @ran || :give_back)
+        @manager.job_finished(work.raw_job)
       end
     end
 
@@ -123,25 +133,65 @@ module Cogworker
     def interrupt(work, job, error)
       payload = JSON.parse(work.raw_job)
       count = payload['interrupted_count'].to_i + 1
-      payload.merge!('interrupted_count' => count, 'retry_count' => job['retry_count'],
+      # This attempt's number, from the payload as fetched — not from the
+      # in-memory job, whose count `route_failure` may not have got to.
+      attempt = payload['retry_count'].to_i + 1
+      payload.merge!('interrupted_count' => count, 'retry_count' => attempt,
                      'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error),
                      'failed_at' => Time.now.to_f)
-      set, score = if count > MAX_INTERRUPTS
+      # No retries left (`retry: false`/`0`, or the last one used up) means
+      # no extra attempt either — straight to dead, like any terminal failure.
+      out_of_retries = attempt > JobUtil.max_retries(payload)
+      delay = INTERRUPT_DELAY * count
+      set, score = if out_of_retries || count > MAX_INTERRUPTS
                      [RedisKeys::DEAD, Time.now.to_f]
                    else
-                     [RedisKeys::RETRY, Time.now.to_f + (INTERRUPT_DELAY * count)]
+                     [RedisKeys::RETRY, Time.now.to_f + delay]
                    end
-      fetcher.interrupt(work, set, score, JSON.generate(payload))
-      Cogworker.logger.error do
-        "couldn't record the failure of jid=#{payload['jid']} (#{error.class}); " \
-          "moved to #{set.delete_prefix('cogworker:')} (interruption #{count}/#{MAX_INTERRUPTS})"
+      encoded = JSON.generate(payload)
+      begin
+        fetcher.interrupt(work, set, score, encoded)
+      rescue StandardError
+        # Remembered with its payload: this process's reconcile (and its
+        # shutdown) keep trying to file it — never re-run it as a stray.
+        @manager.settle_later(work.raw_job, [set, score, encoded]) if fetcher.is_a?(ReliableFetch)
+        raise
       end
+      if set == RedisKeys::RETRY
+        best_effort('lock extension') { extend_locks_for_retry(job, delay) }
+      else
+        best_effort('lock release') { release_locks(payload) }
+      end
+      reason = if set == RedisKeys::RETRY
+                 "retrying in #{delay}s (interruption #{count} of at most #{MAX_INTERRUPTS})"
+               elsif out_of_retries
+                 'moved to dead: no retries left'
+               else
+                 "moved to dead: interrupted more than #{MAX_INTERRUPTS} times"
+               end
+      Cogworker.logger.error { "couldn't record the failure of jid=#{payload['jid']} (#{error.class}); #{reason}" }
     rescue StandardError => e
+      # Whatever failed (even before anything was written), remember the job
+      # with the simplest possible dead entry, so reconcile files it — not
+      # leave it a stray to be re-run. (Already remembered if it got as far
+      # as the write.)
+      remember_for_dead(work, error) if fetcher.is_a?(ReliableFetch) && !@manager.pending_settlements.key?(work.raw_job)
       Cogworker.logger.error do
         "couldn't record or interrupt jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
       end
     end
 
+    def remember_for_dead(work, error)
+      # The original job's own fields where readable (so filing it releases
+      # its locks); just enough to show in Dead otherwise.
+      base = ReliableFetch.parse_hash(work.raw_job) || { 'raw_payload' => JobUtil.safe_string(work.raw_job, 100_000) }
+      entry = JSON.generate(base.merge('jid' => jid_of(work), 'error_class' => error.class.name.to_s,
+                                       'error_message' => JobUtil.error_message(error), 'failed_at' => Time.now.to_f))
+      @manager.settle_later(work.raw_job, [RedisKeys::DEAD, Time.now.to_f, entry])
+    rescue StandardError
+      nil
+    end
+
     # A failed ack leaves an already-finished job on this process's own
     # in-progress list, where `Manager#stop!` would later requeue it — a
     # silent second run. Retried briefly, then logged with the jid so that
@@ -156,9 +206,12 @@ module Cogworker
           sleep(0.1 * attempts)
           retry
         end
+        # Remembered, so this process's own reconcile finishes the ack later
+        # instead of mistaking the entry for a stray and re-running it.
+        @manager.settle_later(work.raw_job, :ack)
         Cogworker.logger.error do
           "couldn't acknowledge finished job jid=#{jid_of(work)} after #{attempts} attempts " \
-            "(it may run again on shutdown): #{e.class}: #{e.message}"
+            "(will retry; it may run again if this process stops first): #{e.class}: #{e.message}"
         end
       end
     end
@@ -169,17 +222,25 @@ module Cogworker
       '?'
     end
 
-    # Returns how `settle` should dispose of the fetched job; raises only
-    # before the job has run (then `process_one`'s default, :give_back,
-    # applies).
+    # Returns how `settle` should dispose of the fetched job. If it raises
+    # instead, `process_one` falls back on `@ran` — set the moment the job
+    # has run — or, before that, on :give_back.
     def execute(work)
       job = parse_job(work)
       return :acknowledge unless job
 
-      register_in_workers(work.queue, job)
+      # Best-effort: for the reliable fetch the in-progress list is what
+      # keeps the job safe; `cogworker:workers` only feeds the Workers tab
+      # (and BasicFetch's shutdown requeue). A failure here — even a
+      # deterministic one, the key holding the wrong type — used to hand
+      # the job back before it ever ran, over and over.
+      best_effort('register') { register_in_workers(work.queue, job) }
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
       begin
         error = run_job(work, job)
+        @manager.job_performed(work.raw_job)
+        @performed = true
+        @ran = error ? [:interrupt, job, error] : :acknowledge
         error ? finish_failure(job, error) : finish_success(job)
       ensure
         best_effort('deregister') { deregister_from_workers }
@@ -235,7 +296,9 @@ module Cogworker
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
         Throughput.record('failed')
       end
-      Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}" }
+      Cogworker.logger.warn do
+        "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}"
+      end
       route_failure(job, error) ? :acknowledge : [:interrupt, job, error]
     end
 
@@ -299,7 +362,12 @@ module Cogworker
       begin
         payload = JSON.generate(job)
         Cogworker.config.redis do |c|
-          retrying ? c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, payload) : c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+          if retrying
+            c.zadd(RedisKeys::RETRY, Time.now.to_f + delay,
+                   payload)
+          else
+            c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+          end
         end
       rescue StandardError => e
         Cogworker.logger.error { "couldn't record failure of jid=#{job['jid']}: #{e.class}: #{e.message}" }
@@ -329,6 +397,13 @@ module Cogworker
       end
     end
 
+    # Into dead by way of `interrupt` — possibly with retries still left,
+    # so the middleware that normally releases a terminal failure's locks
+    # didn't: release them here (owner-checked).
+    def release_locks(job)
+      Cogworker.config.redis { |c| JobUtil.release_terminal_locks(c, job) }
+    end
+
     # Grows with the retry count, with jitter to avoid a thundering herd of
     # retries all landing on the same second.
     def retry_delay(count)

data/lib/cogworker/redis_errors.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Telling "Redis is unavailable right now" from every other error, and
+  # keeping the log readable while it is.
+  #
+  # Unavailable means a lost connection, or one of the replies a server
+  # gives while it can't serve writes for a while — mid-failover (READONLY,
+  # MASTERDOWN), still loading its data (LOADING), or busy with a script
+  # (BUSY, TRYAGAIN). None of those say anything about the job, entry or key
+  # being worked on, so nothing should back off, count a failure, or give up
+  # on them; they just mean "try again shortly".
+  #
+  # `report` logs such errors once per `site` (a short name of the place
+  # they come from) as they start, then a one-line summary at most every
+  # SUMMARY_INTERVAL, and `recovered(site)` logs once that it's over. A 6
+  # minute outage used to produce thousands of identical lines. Any other
+  # error is logged in full, every time.
+  module RedisErrors
+    UNAVAILABLE_REPLIES = %w[READONLY LOADING MASTERDOWN TRYAGAIN BUSY].freeze
+    SUMMARY_INTERVAL = 60
+    SCRIPT_WRAPPED = /-(?:#{UNAVAILABLE_REPLIES.join('|')})(?=\s|\z)/
+
+    @mutex = Mutex.new
+    @outages = {} # site => { count:, since:, summarized_at:, error: }
+
+    class << self
+      def unavailable?(error)
+        return true if error.is_a?(Redis::BaseConnectionError)
+        # redis-rb 5's pub/sub path can surface redis-client's own error class.
+        return true if defined?(RedisClient::ConnectionError) && error.is_a?(RedisClient::ConnectionError)
+
+        return false unless error.is_a?(Redis::CommandError)
+
+        message = error.message.to_s
+        # The reply's code is its whole first word — BUSYKEY or BUSYGROUP
+        # aren't BUSY. Inside a script, Redis 6.x wraps the code instead:
+        # "ERR Error running script (...): ... -READONLY You can't ...".
+        UNAVAILABLE_REPLIES.include?(message[/\A\S+/]) ||
+          (message.include?('Error running script') && message.match?(SCRIPT_WRAPPED))
+      end
+
+      # Logs `error` at `site`; the block, if given, adds context to the
+      # first line (e.g. what was being done).
+      def report(site, error, &context)
+        detail = "#{error.class}: #{error.message}"
+        detail = "#{context.call} — #{detail}" if context
+        return Cogworker.logger.error { "#{site}: #{detail}" } unless unavailable?(error)
+
+        line = @mutex.synchronize { track(site, error, detail) }
+        Cogworker.logger.error { line } if line
+      end
+
+      # Call wherever `site` has just done its job: ends an outage there.
+      def recovered(site)
+        outage = @mutex.synchronize { @outages.delete(site) }
+        return unless outage
+
+        Cogworker.logger.warn do
+          "#{site}: Redis available again after #{outage[:count]} error(s) over #{(now - outage[:since]).round}s"
+        end
+      end
+
+      def reset!
+        @mutex.synchronize { @outages.clear }
+      end
+
+      private
+
+      def track(site, error, detail)
+        outage = @outages[site]
+        unless outage
+          @outages[site] = { count: 1, since: now, summarized_at: now, suppressed: 0 }
+          return "#{site}: Redis unavailable — #{detail} (repeats summarized every #{SUMMARY_INTERVAL}s)"
+        end
+
+        outage[:count] += 1
+        outage[:suppressed] += 1
+        return unless now - outage[:summarized_at] >= SUMMARY_INTERVAL
+
+        suppressed = outage[:suppressed]
+        outage[:summarized_at] = now
+        outage[:suppressed] = 0
+        "#{site}: Redis still unavailable — #{suppressed} more error(s) in the last #{SUMMARY_INTERVAL}s " \
+          "(latest: #{error.class}: #{error.message})"
+      end
+
+      def now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/cogworker/redis_keys.rb

@@ -17,6 +17,19 @@ module Cogworker
     # No TTL, unlike `process(identity)`: it's what tells "dead for a
     # while" apart from "missed a few beats" (Config#orphan_threshold).
     LAST_BEAT = 'cogworker:last_beat'
+    # Jobs recovered from dead processes more than `config.max_orphanings` times,
+    # parked by REQUEUE_SCRIPT for Ruby to file in `dead` (see
+    # ReliableFetch.bury_repeat_orphans).
+    REPEAT_ORPHANS = 'cogworker:repeat_orphans'
+    ORPHANINGS_PREFIX = 'cogworker:orphanings:'
+    # Jobs that ran and failed but couldn't be filed in retry/dead, handed
+    # over by a process shutting down (JSON {set, score, payload} each) —
+    # see ReliableFetch.park_unsettled / file_unsettled.
+    UNSETTLED = 'cogworker:unsettled'
+    # queue => Redis time a live process last said it reads it (every
+    # heartbeat). Unlike QUEUES (the Web UI's listing, never pruned), this
+    # tells "nobody reads this queue any more" apart from "somebody does".
+    LIVE_QUEUES = 'cogworker:live_queues'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'