cogworker 0.5.0 → 0.5.1

data/lib/cogworker/client.rb

@@ -43,16 +43,29 @@ module Cogworker
         end
       end
 
+      # The queue listing only feeds the Web UI/stats; failing to update it
+      # is logged, never raised — whatever the error, Redis being away
+      # included: the job is already queued by then, and raising would make
+      # the caller push it a second time (see raw_push).
+      def register_queue(conn, queue)
+        conn.sadd?(RedisKeys::QUEUES, queue)
+      rescue StandardError => e
+        RedisErrors.report("#{RedisKeys::QUEUES} not updated", e) { queue }
+      end
+
       def raw_push(job)
         Cogworker.config.redis do |conn|
           if job['at']
             conn.zadd(RedisKeys::SCHEDULE, job['at'].to_f, JSON.generate(job))
           else
             job['enqueued_at'] = Time.now.to_f
-            conn.multi do |pipeline|
-              pipeline.sadd?(RedisKeys::QUEUES, job['queue'])
-              pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
-            end
+            # The push is the one write that matters, and goes first, on its
+            # own. They used to share a MULTI — which doesn't roll back: with
+            # `cogworker:queues` unwritable the job was queued anyway but the
+            # whole push raised, so callers retried it (the cron ticker did,
+            # every tick) and queued it again and again.
+            conn.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
+            register_queue(conn, job['queue'])
           end
         end
       end

data/lib/cogworker/config.rb

@@ -61,7 +61,10 @@ module Cogworker
     end
 
     def max_orphanings=(count)
-      raise ArgumentError, "max_orphanings must be a positive Integer, got #{count.inspect}" unless count.is_a?(Integer) && count.positive?
+      unless count.is_a?(Integer) && count.positive?
+        raise ArgumentError,
+              "max_orphanings must be a positive Integer, got #{count.inspect}"
+      end
 
       @max_orphanings = count
     end

data/lib/cogworker/deferred_releases.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Lock releases that failed (Redis away when a job finished) — kept here
+  # and retried on every heartbeat beat (Heartbeat#beat) until they go
+  # through, owner-checked like the original release. A failed release is
+  # best-effort so it can't fail the job, but left alone it held the lock
+  # until its TTL: an `until_executed` cron entry skipped slots for up to
+  # `RunningLock.active_ttl`, a unique job blocked its duplicates for up to
+  # `unique_lock_ttl`. In this process's memory only: one still pending when
+  # the process exits falls back to the lock's TTL.
+  module DeferredReleases
+    @mutex = Mutex.new
+    @pending = {} # lock key => owner jid
+
+    class << self
+      def add(key, owner)
+        @mutex.synchronize { @pending[key] = owner }
+      end
+
+      # Retries every pending release; stops at the first one Redis can't
+      # serve (no point hammering it), keeps whatever's left.
+      def retry_all(conn)
+        pending.each do |key, owner|
+          OwnedKey.delete(key, owner, conn) # (a no-op if it lapsed or changed hands meanwhile)
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+        rescue StandardError => e
+          raise if RedisErrors.unavailable?(e)
+
+          # Anything else (the key holding the wrong type, say) won't get
+          # better by retrying every 5s: dropped, logged once.
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+          Cogworker.logger.error { "deferred release of #{key} dropped: #{e.class}: #{e.message}" }
+        end
+      end
+
+      def pending
+        @mutex.synchronize { @pending.dup }
+      end
+
+      def reset!
+        @mutex.synchronize { @pending.clear }
+      end
+    end
+  end
+end

data/lib/cogworker/heartbeat.rb

@@ -99,9 +99,12 @@ module Cogworker
 
     def beat_safely
       beat
+      @beat_failed = false
+      RedisErrors.recovered('Heartbeat')
       true
     rescue StandardError => e
-      Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      @beat_failed = true
+      RedisErrors.report('Heartbeat', e)
       false
     end
 
@@ -128,21 +131,43 @@ module Cogworker
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
-        secondary('process registry') { c.sadd?(RedisKeys::PROCESSES, identity) }
-        secondary('workers expiry') { c.expire(RedisKeys.workers(identity), TTL) }
-        if reliable
-          secondary('in-progress registry') { c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
-          secondary('last-beat stamp') { c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c)) }
-        end
+        write_registries(c, identity, reliable)
         touch_running_periodic_locks(c, identity)
+        # Not via #secondary: Redis failing here, after the presence key went
+        # through, doesn't make this beat a failed one.
+        BestEffort.call('Deferred lock releases') { DeferredReleases.retry_all(c) }
       end
     end
 
+    # Everything a beat writes besides the presence key — each on its own,
+    # best-effort (see #secondary).
+    def write_registries(conn, identity, reliable)
+      secondary('process registry') { conn.sadd?(RedisKeys::PROCESSES, identity) }
+      # The queues this process reads are listed even while empty: the Web
+      # UI shows them, and recovery can tell a queue nobody reads from one
+      # that's merely had nothing pushed yet.
+      secondary('queue registry') { conn.sadd(RedisKeys::QUEUES, @manager.queues.uniq) }
+      secondary('live queue registry') { mark_queues_live(conn) }
+      secondary('workers expiry') { conn.expire(RedisKeys.workers(identity), TTL) }
+      return unless reliable
+
+      secondary('in-progress registry') { conn.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
+      secondary('last-beat stamp') { conn.hset(RedisKeys::LAST_BEAT, identity, redis_now(conn)) }
+    end
+
+    # Stamps this process's queues as read right now, and drops ones no
+    # process has stamped for a day (only ever a listing, never data).
+    def mark_queues_live(conn)
+      now = redis_now(conn)
+      conn.zadd(RedisKeys::LIVE_QUEUES, @manager.queues.uniq.map { |q| [now, q] })
+      conn.zremrangebyscore(RedisKeys::LIVE_QUEUES, '-inf', now - 86_400)
+    end
+
     def secondary(what)
       yield
-    rescue Redis::BaseConnectionError
-      raise # Redis itself is gone: the beat as a whole failed
     rescue StandardError => e
+      raise if RedisErrors.unavailable?(e) # Redis itself is away: the beat as a whole failed
+
       Cogworker.logger.error { "Heartbeat: #{what} not written: #{e.class}: #{e.message}" }
     end
 
@@ -161,12 +186,30 @@ module Cogworker
     # see Periodic::RunningLock. From the Manager's own in-memory record of
     # what's running here, not `cogworker:workers` in Redis: a failover that
     # lost that Hash would otherwise let a long run's lock lapse mid-run.
+    # (One narrow race remains: a release landing between this beat's
+    # snapshot of `performing_jobs` and its Lua call, on the first beat
+    # after a failed one, can be undone — the lock then lapses on
+    # `active_ttl`. Not worth a cross-thread lock on every beat.)
+    # Only jobs still inside `perform` (`performing_jobs`): once it returns,
+    # Periodic::ReleaseMiddleware has released the lock, and refreshing it
+    # again would bring it back. A missing lock is only *re-taken* on the
+    # first beat after a failed one — i.e. after Redis was away, possibly
+    # long enough for the lock to lapse; otherwise a missing lock means it
+    # was released, and stays released.
     def touch_running_periodic_locks(conn, _identity)
-      @manager.running_jobs.each do |raw|
+      recovering = @beat_failed
+      @manager.performing_jobs.each do |raw|
         job = JSON.parse(raw)
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
-        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        if recovering && job['periodic_until_executed']
+          # Re-taken, not just extended: after Redis was away longer than
+          # `active_ttl`, the lock is gone while the run is still going, and
+          # the ticker could start the next slot alongside it.
+          Periodic::RunningLock.retake(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        else
+          Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        end
       rescue StandardError => e
         # Per entry: one bad entry mustn't leave the others' locks to expire.
         Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -199,6 +242,7 @@ module Cogworker
     def subscribe_loop
       @subscribe_client = ::Redis.new(RedisConnection.client_options(Cogworker.config.redis_options))
       @subscribe_client.subscribe(RedisKeys.signal(Cogworker.identity)) do |on|
+        on.subscribe { |*| RedisErrors.recovered('Signal subscriber') }
         on.message do |_channel, message|
           dispatch(message)
         end
@@ -206,7 +250,7 @@ module Cogworker
     rescue StandardError => e
       return if @stopping
 
-      Cogworker.logger.error { "Signal subscriber died: #{e.class}: #{e.message}" }
+      RedisErrors.report('Signal subscriber', e)
       sleep(1)
       retry
     end

data/lib/cogworker/history/middleware.rb

@@ -5,13 +5,17 @@ module Cogworker
     # Server middleware: times every run and records it via Storage,
     # success or failure, regardless of what other middleware/worker code does.
     class Middleware
+      # Recording is best-effort (BestEffort): only the job's own exception
+      # is re-raised, never one from writing its history.
       def call(_worker, job, queue)
         started_at = Time.now.to_f
-        yield
-        Storage.record(job, queue, started_at, Time.now.to_f, 'success')
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e)
-        raise e
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e) }
+          raise e
+        end
+        BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'success') }
       end
     end
   end

data/lib/cogworker/job_util.rb

@@ -142,6 +142,23 @@ module Cogworker
       "#<#{error.class}>"
     end
 
+    # A job on its way to dead outside the usual path (no server middleware
+    # around it to do this): releases its `until_executed` locks, owner-
+    # checked, each on its own — so the entry / the unique key isn't blocked
+    # until the locks' TTL. Best-effort.
+    def release_terminal_locks(conn, job)
+      keys = []
+      keys << RedisKeys.unique_lock(UniqueJobs.digest(job)) if UniqueJobs.until_executed?(job)
+      keys << RedisKeys.periodic_running(job['periodic_pjid']) if job['periodic_pjid']
+      keys.each do |key|
+        released = BestEffort.call('Lock release') do
+          OwnedKey.delete(key, job['jid'], conn)
+          true
+        end
+        DeferredReleases.add(key, job['jid']) unless released
+      end
+    end
+
     UNPARSEABLE_CLASS = '(unparseable)'
 
     # A payload that isn't a JSON object can't be run, retried or even

data/lib/cogworker/key_guard.rb

@@ -22,6 +22,7 @@ module Cogworker
       RedisKeys::QUEUES => 'set', RedisKeys::PROCESSES => 'set', RedisKeys::PAUSED_QUEUES => 'set',
       RedisKeys::IN_PROGRESS_IDENTITIES => 'set', RedisKeys::LAST_BEAT => 'hash',
       RedisKeys::REPEAT_ORPHANS => 'list', RedisKeys::UNSETTLED => 'list', RedisKeys::PERIODIC_SCHEDULE => 'hash',
+      RedisKeys::LIVE_QUEUES => 'zset',
       RedisKeys::PERIODIC_DISABLED => 'set', RedisKeys::STATS_PROCESSED => 'string',
       RedisKeys::STATS_FAILED => 'string'
     }.freeze
@@ -65,9 +66,41 @@ module Cogworker
         types[RedisKeys.process(id)] = 'hash'
         types[RedisKeys.workers(id)] = 'hash'
       end
+      readable_fields(RedisKeys::PERIODIC_SCHEDULE).each do |pjid|
+        types[RedisKeys.periodic_running(pjid)] = 'string'
+        types[RedisKeys.periodic_last_slot(pjid)] = 'string'
+      end
       types
     end
 
+    def readable_fields(hash)
+      Cogworker.config.redis { |c| c.hkeys(hash) }
+    rescue Redis::CommandError
+      []
+    end
+
+    MAX_COPIES = 5
+
+    # Keeps the newest MAX_COPIES quarantined copies of one key: something
+    # that keeps writing the wrong type would otherwise leave one more every
+    # CHECK_INTERVAL, for QUARANTINE_TTL. (A SCAN, but only on the rare
+    # occasion a key actually gets quarantined.)
+    def prune_copies(key)
+      prefix = "#{QUARANTINE_PREFIX}#{key}:"
+      exact = /\A#{Regexp.escape(prefix)}(\d+):\h{8}\z/ # this key's copies only — not `<key>:<more>:...`'s
+      Cogworker.config.redis do |c|
+        copies = c.scan_each(match: "#{glob_escape(prefix)}*").select { |name| name.match?(exact) }
+        stale = copies.sort_by { |name| name[exact, 1].to_i }.reverse.drop(MAX_COPIES)
+        c.del(*stale) unless stale.empty?
+      end
+    rescue StandardError => e
+      Cogworker.logger.error { "pruning quarantined copies of #{key} failed: #{e.class}: #{e.message}" }
+    end
+
+    def glob_escape(text)
+      text.gsub(/[*?\[\]\\]/) { |ch| "\\#{ch}" }
+    end
+
     def readable_members(set)
       Cogworker.config.redis { |c| c.smembers(set) }
     rescue Redis::CommandError
@@ -77,12 +110,14 @@ module Cogworker
     def quarantine(key, expected, actual)
       # Unique per move: a second quarantine of the same key within the same
       # second must not overwrite (RENAME) the first one's copy.
-      target = "#{QUARANTINE_PREFIX}#{key}:#{Time.now.to_i}:#{SecureRandom.hex(4)}"
+      # Milliseconds, so copies made within one second still sort by age.
+      target = "#{QUARANTINE_PREFIX}#{key}:#{(Time.now.to_f * 1000).to_i}:#{SecureRandom.hex(4)}"
       moved = Cogworker.config.redis do |c|
         LuaScript.run(c, QUARANTINE_SCRIPT, keys: [key, target], argv: [expected, QUARANTINE_TTL])
       end
       return false unless moved == 1
 
+      prune_copies(key)
       Cogworker.logger.error do
         "#{key} held a #{actual}, not a #{expected}: moved to #{target} (kept #{QUARANTINE_TTL / 86_400} days)"
       end

data/lib/cogworker/launcher.rb

@@ -45,6 +45,9 @@ module Cogworker
 
     def stop!
       @manager.stop!
+      # Last chance for releases deferred by a recent outage — they live in
+      # this process's memory and would otherwise wait out their locks' TTL.
+      BestEffort.call('Deferred lock releases') { @config.redis { |c| DeferredReleases.retry_all(c) } }
       @scheduled.stop!
       @ticker.stop!
       @heartbeat.stop!

data/lib/cogworker/lua_script.rb

@@ -9,8 +9,8 @@ module Cogworker
   # restarted or failed-over server, SCRIPT FLUSH). Every script in this
   # gem goes through here.
   module LuaScript
-    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
-    SHAS_MUTEX = Mutex.new
+    @shas = {} # source => sha1; memoized, the sources are frozen constants
+    @shas_mutex = Mutex.new
 
     module_function
 
@@ -24,7 +24,7 @@ module Cogworker
     end
 
     def sha(source)
-      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+      @shas_mutex.synchronize { @shas[source] ||= Digest::SHA1.hexdigest(source) }
     end
   end
 end

data/lib/cogworker/manager.rb

@@ -16,6 +16,7 @@ module Cogworker
       # be what's failing (or what lost its last writes in a failover).
       @jobs_mutex = Mutex.new
       @running_jobs = Hash.new(0)
+      @performing_jobs = Hash.new(0) # the subset still inside `perform`
       @pending_settlements = {}
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
@@ -74,7 +75,24 @@ module Cogworker
     end
 
     def job_started(raw)
-      @jobs_mutex.synchronize { @running_jobs[raw] += 1 }
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] += 1
+        @performing_jobs[raw] += 1
+      end
+    end
+
+    # `perform` (with its middleware) has returned: the job is still this
+    # process's until settled, but its locks are no longer its to keep
+    # alive — Heartbeat stops refreshing them (see `performing_jobs`).
+    def job_performed(raw)
+      @jobs_mutex.synchronize do
+        @performing_jobs[raw] -= 1
+        @performing_jobs.delete(raw) unless @performing_jobs[raw].positive?
+      end
+    end
+
+    def performing_jobs
+      @jobs_mutex.synchronize { @performing_jobs.keys }
     end
 
     def job_finished(raw)

data/lib/cogworker/periodic/claim.lua

@@ -10,7 +10,7 @@
 -- Returns {1, <previous last_slot, or "" if none>} if this call won the
 -- claim for this slot (the caller should enqueue the job — and, if that
 -- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
--- value), 0 otherwise. Composes two guards: `last_slot` stops a
+-- value), 2 if blocked by the entry's running lock, 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
@@ -18,8 +18,11 @@
 -- For `until_executed`, the running lock is written here, in the same
 -- atomic step as the claim and before the job exists anywhere — see
 -- Periodic::RunningLock for why it must never be written after the push.
+-- Blocked by a run still holding the entry's lock: a distinct 2, so the
+-- caller can retry the slot briefly — the lock may only be waiting for a
+-- release delayed by an outage (see Ticker::BLOCKED_GRACE).
 if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
-  return 0
+  return 2
 end
 
 local previous = redis.call("GET", KEYS[2])

data/lib/cogworker/periodic/release_middleware.rb

@@ -23,14 +23,25 @@ module Cogworker
         begin
           yield
         rescue Exception => e # rubocop:disable Lint/RescueException
-          RunningLock.release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
+          release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
           raise e
         end
-        RunningLock.release(pjid, job['jid'])
+        release(pjid, job['jid'])
       end
 
       private
 
+      # Best-effort — a release that fails (Redis away) never turns the
+      # finished run into a failure — and retried later (DeferredReleases)
+      # rather than left to the lock's TTL.
+      def release(pjid, jid)
+        released = BestEffort.call('Periodic lock') do
+          RunningLock.release(pjid, jid)
+          true
+        end
+        DeferredReleases.add(RedisKeys.periodic_running(pjid), jid) unless released
+      end
+
       # Best-effort: failing here must not stop the job from running (it
       # would be routed to retry/dead as if it had failed). If it does fail,
       # the lock just keeps its longer queued TTL until this process's next

data/lib/cogworker/periodic/running_lock.rb

@@ -61,6 +61,18 @@ module Cogworker
         OwnedKey.expire(RedisKeys.periodic_running(pjid), jid, ttl, conn)
       end
 
+      RETAKE_SCRIPT = RETAKE_FUNCTION + <<~LUA
+        retake_running_lock(KEYS[1], ARGV[1], ARGV[2])
+        return 1
+      LUA
+
+      # Extends the lock if still `jid`'s, takes it if it's gone, leaves it
+      # if another run holds it — for a run still going whose lock may have
+      # lapsed (Redis away longer than `active_ttl`).
+      def retake(pjid, jid, ttl, conn)
+        LuaScript.run(conn, RETAKE_SCRIPT, keys: [RedisKeys.periodic_running(pjid)], argv: [jid, ttl.to_f.ceil])
+      end
+
       def release(pjid, jid, conn = nil)
         OwnedKey.delete(RedisKeys.periodic_running(pjid), jid, conn)
       end

data/lib/cogworker/periodic/ticker.rb

@@ -29,7 +29,7 @@ module Cogworker
       # separate call made first, its failure (Redis down — the usual reason
       # the push failed) skipped the rollback altogether.
       ROLLBACK_SCRIPT = <<~LUA
-        if redis.call("GET", KEYS[1]) == ARGV[1] then
+        if redis.pcall("GET", KEYS[1]) == ARGV[1] then
           if ARGV[2] == "" then
             redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
           else
@@ -54,6 +54,7 @@ module Cogworker
         @last_checked_slot = {}
         @cron_cache = {}
         @pending_rollbacks = []
+        @blocked_since = {} # pjid => [slot, monotonic time it was first found blocked]
         @failures = Hash.new(0) # pjid => consecutive failed ticks
         @retry_at = {}          # pjid => monotonic time it may be tried again
       end
@@ -113,12 +114,16 @@ module Cogworker
 
       def tick
         retry_pending_rollbacks
+        # This process's own delayed lock releases first (see BLOCKED_GRACE).
+        BestEffort.call('Deferred lock releases') { Cogworker.config.redis { |c| DeferredReleases.retry_all(c) } }
         now = Time.now
         @entries.each do |entry|
           next if backing_off?(entry.pjid)
 
-          tick_entry(entry, now)
+          next if tick_entry(entry, now) == :skipped # (no Redis call made: proves nothing)
+
           @failures.delete(entry.pjid)
+          RedisErrors.recovered('Periodic tick')
         rescue StandardError => e
           # Per entry: one entry failing (its keys of the wrong type, its
           # push failing) used to skip every entry after it, on every tick.
@@ -128,8 +133,8 @@ module Cogworker
           # a lost connection, though: that's Redis being away, not the
           # entry failing — backing off every entry on it left cron idle for
           # minutes after Redis came back.
-          delay = e.is_a?(Redis::BaseConnectionError) ? TICK_INTERVAL : back_off(entry.pjid)
-          Cogworker.logger.error { "Periodic tick failed for #{entry.pjid} (next try in #{delay}s): #{e.class}: #{e.message}" }
+          delay = RedisErrors.unavailable?(e) ? TICK_INTERVAL : back_off(entry.pjid)
+          RedisErrors.report('Periodic tick', e) { "#{entry.pjid} failed (next try in #{delay}s)" }
         end
       end
 
@@ -149,12 +154,32 @@ module Cogworker
 
       def tick_entry(entry, now)
         slot = cron_for(entry).previous_time(now).to_i
-        return if @last_checked_slot[entry.pjid] == slot
+        return :skipped if @last_checked_slot[entry.pjid] == slot
 
         jid = SecureRandom.hex(12)
         previous = claim(entry, slot, jid) unless disabled?(entry)
-        enqueue(entry, slot, jid, previous) if previous
+        return if previous == :blocked && still_blocked_in_grace?(entry.pjid, slot)
+
+        enqueue(entry, slot, jid, previous) if previous && previous != :blocked
         @last_checked_slot[entry.pjid] = slot
+        @blocked_since.delete(entry.pjid)
+      end
+
+      # A slot blocked by the entry's running lock is retried for
+      # BLOCKED_GRACE before it counts as skipped. After an outage, the lock
+      # may still be held only because its finished run's release was
+      # deferred (DeferredReleases — retried on the next heartbeat, in
+      # whichever process ran it); marking the slot checked straight away
+      # lost it, in every process, until the entry's next slot — an hour,
+      # for an hourly entry. A run genuinely still going keeps the lock past
+      # the grace, and its slot is skipped as `until_executed` intends.
+      BLOCKED_GRACE = 3 * Heartbeat::INTERVAL
+
+      def still_blocked_in_grace?(pjid, slot)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        since_slot, since = @blocked_since[pjid]
+        @blocked_since[pjid] = [slot, since = now] unless since_slot == slot
+        now - since < BLOCKED_GRACE
       end
 
       # The usual reason a push fails — Redis unreachable — makes this fail
@@ -175,7 +200,7 @@ module Cogworker
         end
         true
       rescue StandardError => e
-        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
+        RedisErrors.report('Periodic rollback', e) { "#{entry.pjid}: couldn't roll back slot #{slot} (will retry)" }
         @pending_rollbacks << [entry, slot, previous_slot, jid]
         e # (truthy, but not `true`: the caller tells a lost connection from a failing rollback)
       end
@@ -194,7 +219,8 @@ module Cogworker
           if result == true
             @failures.delete(key)
             @retry_at.delete(key)
-          elsif !result.is_a?(Redis::BaseConnectionError)
+            RedisErrors.recovered('Periodic rollback')
+          elsif !RedisErrors.unavailable?(result)
             back_off(key) # only a rollback failing on its own backs off; see #tick
           end
         end
@@ -218,9 +244,9 @@ module Cogworker
       # connection still raises (the tick is then retried as a whole).
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
-      rescue Redis::BaseConnectionError
-        raise
       rescue StandardError => e
+        raise if RedisErrors.unavailable?(e)
+
         Cogworker.logger.error { "Periodic: can't read #{RedisKeys::PERIODIC_DISABLED} (#{e.class}: #{e.message}); treating #{entry.pjid} as enabled" }
         false
       end
@@ -229,17 +255,20 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      # nil if this process didn't win the slot; otherwise the previous
-      # `last_slot` value ("" if none), for #rollback_claim.
+      # nil if this process didn't win the slot, :blocked if the entry's
+      # running lock stopped it; otherwise the previous `last_slot` value
+      # ("" if none), for #rollback_claim.
       def claim(entry, slot, jid)
         return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           LuaScript.run(c, CLAIM_SCRIPT,
-                 keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
-                        RedisKeys.periodic_lock(entry.pjid, slot)],
-                 argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
+                        keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
+                               RedisKeys.periodic_lock(entry.pjid, slot)],
+                        argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
+        return :blocked if result == 2
+
         result.is_a?(Array) ? result[1].to_s : nil
       end
 

data/lib/cogworker/process_set.rb

@@ -3,6 +3,9 @@
 require 'json'
 
 module Cogworker
+  # Every live worker process, as published by its Heartbeat — one
+  # Cogworker::Process per `cogworker:processes` member whose presence key
+  # still exists (stale members are pruned while iterating).
   class ProcessSet
     include Enumerable