cogworker 0.4.1 → 0.5.1

data/lib/cogworker/client.rb

@@ -43,16 +43,29 @@ module Cogworker
         end
       end
 
+      # The queue listing only feeds the Web UI/stats; failing to update it
+      # is logged, never raised — whatever the error, Redis being away
+      # included: the job is already queued by then, and raising would make
+      # the caller push it a second time (see raw_push).
+      def register_queue(conn, queue)
+        conn.sadd?(RedisKeys::QUEUES, queue)
+      rescue StandardError => e
+        RedisErrors.report("#{RedisKeys::QUEUES} not updated", e) { queue }
+      end
+
       def raw_push(job)
         Cogworker.config.redis do |conn|
           if job['at']
             conn.zadd(RedisKeys::SCHEDULE, job['at'].to_f, JSON.generate(job))
           else
             job['enqueued_at'] = Time.now.to_f
-            conn.multi do |pipeline|
-              pipeline.sadd?(RedisKeys::QUEUES, job['queue'])
-              pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
-            end
+            # The push is the one write that matters, and goes first, on its
+            # own. They used to share a MULTI — which doesn't roll back: with
+            # `cogworker:queues` unwritable the job was queued anyway but the
+            # whole push raised, so callers retried it (the cron ticker did,
+            # every tick) and queued it again and again.
+            conn.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
+            register_queue(conn, job['queue'])
           end
         end
       end

data/lib/cogworker/config.rb

@@ -9,7 +9,7 @@ module Cogworker
     FETCH_MODES = %i[reliable basic].freeze
 
     attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
-                :orphan_threshold
+                :orphan_threshold, :max_orphanings
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -51,9 +51,24 @@ module Cogworker
       # failover, a long GVL-holding call) — each of which runs its jobs
       # twice; lower: a really crashed process's jobs come back sooner.
       @orphan_threshold = 300
+      # How many times a job may be recovered from a process that died
+      # while running it before it goes to dead instead (ReliableFetch). The
+      # count can't single out the culprit: every job running in a process
+      # that crashed is counted, so keep this above the number of crashes an
+      # innocent job might plausibly sit through.
+      @max_orphanings = 3
       register_default_middleware
     end
 
+    def max_orphanings=(count)
+      unless count.is_a?(Integer) && count.positive?
+        raise ArgumentError,
+              "max_orphanings must be a positive Integer, got #{count.inspect}"
+      end
+
+      @max_orphanings = count
+    end
+
     def orphan_threshold=(seconds)
       unless seconds.is_a?(Numeric) && seconds.positive?
         raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"

data/lib/cogworker/deferred_releases.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Lock releases that failed (Redis away when a job finished) — kept here
+  # and retried on every heartbeat beat (Heartbeat#beat) until they go
+  # through, owner-checked like the original release. A failed release is
+  # best-effort so it can't fail the job, but left alone it held the lock
+  # until its TTL: an `until_executed` cron entry skipped slots for up to
+  # `RunningLock.active_ttl`, a unique job blocked its duplicates for up to
+  # `unique_lock_ttl`. In this process's memory only: one still pending when
+  # the process exits falls back to the lock's TTL.
+  module DeferredReleases
+    @mutex = Mutex.new
+    @pending = {} # lock key => owner jid
+
+    class << self
+      def add(key, owner)
+        @mutex.synchronize { @pending[key] = owner }
+      end
+
+      # Retries every pending release; stops at the first one Redis can't
+      # serve (no point hammering it), keeps whatever's left.
+      def retry_all(conn)
+        pending.each do |key, owner|
+          OwnedKey.delete(key, owner, conn) # (a no-op if it lapsed or changed hands meanwhile)
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+        rescue StandardError => e
+          raise if RedisErrors.unavailable?(e)
+
+          # Anything else (the key holding the wrong type, say) won't get
+          # better by retrying every 5s: dropped, logged once.
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+          Cogworker.logger.error { "deferred release of #{key} dropped: #{e.class}: #{e.message}" }
+        end
+      end
+
+      def pending
+        @mutex.synchronize { @pending.dup }
+      end
+
+      def reset!
+        @mutex.synchronize { @pending.clear }
+      end
+    end
+  end
+end

data/lib/cogworker/heartbeat.rb

@@ -99,9 +99,12 @@ module Cogworker
 
     def beat_safely
       beat
+      @beat_failed = false
+      RedisErrors.recovered('Heartbeat')
       true
     rescue StandardError => e
-      Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      @beat_failed = true
+      RedisErrors.report('Heartbeat', e)
       false
     end
 
@@ -117,22 +120,57 @@ module Cogworker
         'rss_kb' => current_rss_kb
       }
 
+      # The presence key is what makes this process alive to everyone else
+      # (and what Heartbeat#start! waits for) — the one write a beat can't
+      # do without. Everything else is written on its own, best-effort: one
+      # shared key of the wrong type used to fail every beat of every
+      # process, so no new process could start anywhere in the fleet.
       Cogworker.config.redis do |c|
-        c.sadd?(RedisKeys::PROCESSES, identity)
         c.hset(RedisKeys.process(identity),
                'info', JSON.generate(info),
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
-        c.expire(RedisKeys.workers(identity), TTL)
-        if reliable
-          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
-          c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c))
-        end
+        write_registries(c, identity, reliable)
         touch_running_periodic_locks(c, identity)
+        # Not via #secondary: Redis failing here, after the presence key went
+        # through, doesn't make this beat a failed one.
+        BestEffort.call('Deferred lock releases') { DeferredReleases.retry_all(c) }
       end
     end
 
+    # Everything a beat writes besides the presence key — each on its own,
+    # best-effort (see #secondary).
+    def write_registries(conn, identity, reliable)
+      secondary('process registry') { conn.sadd?(RedisKeys::PROCESSES, identity) }
+      # The queues this process reads are listed even while empty: the Web
+      # UI shows them, and recovery can tell a queue nobody reads from one
+      # that's merely had nothing pushed yet.
+      secondary('queue registry') { conn.sadd(RedisKeys::QUEUES, @manager.queues.uniq) }
+      secondary('live queue registry') { mark_queues_live(conn) }
+      secondary('workers expiry') { conn.expire(RedisKeys.workers(identity), TTL) }
+      return unless reliable
+
+      secondary('in-progress registry') { conn.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
+      secondary('last-beat stamp') { conn.hset(RedisKeys::LAST_BEAT, identity, redis_now(conn)) }
+    end
+
+    # Stamps this process's queues as read right now, and drops ones no
+    # process has stamped for a day (only ever a listing, never data).
+    def mark_queues_live(conn)
+      now = redis_now(conn)
+      conn.zadd(RedisKeys::LIVE_QUEUES, @manager.queues.uniq.map { |q| [now, q] })
+      conn.zremrangebyscore(RedisKeys::LIVE_QUEUES, '-inf', now - 86_400)
+    end
+
+    def secondary(what)
+      yield
+    rescue StandardError => e
+      raise if RedisErrors.unavailable?(e) # Redis itself is away: the beat as a whole failed
+
+      Cogworker.logger.error { "Heartbeat: #{what} not written: #{e.class}: #{e.message}" }
+    end
+
     # Redis's clock, not this host's (so host clock skew doesn't matter to
     # orphan detection) — falling back to this host's where `TIME` isn't
     # allowed (some proxies/managed services): better a skewed stamp than a
@@ -145,15 +183,33 @@ module Cogworker
 
     # Keeps every in-flight `until_executed` periodic run's
     # `periodic:running:<pjid>` lock alive for as long as this process is —
-    # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
-    # Hash the Workers tab does, so it covers exactly the jobs actually
-    # executing here right now.
-    def touch_running_periodic_locks(conn, identity)
-      conn.hvals(RedisKeys.workers(identity)).each do |raw|
-        job = JSON.parse(raw)['payload']
+    # see Periodic::RunningLock. From the Manager's own in-memory record of
+    # what's running here, not `cogworker:workers` in Redis: a failover that
+    # lost that Hash would otherwise let a long run's lock lapse mid-run.
+    # (One narrow race remains: a release landing between this beat's
+    # snapshot of `performing_jobs` and its Lua call, on the first beat
+    # after a failed one, can be undone — the lock then lapses on
+    # `active_ttl`. Not worth a cross-thread lock on every beat.)
+    # Only jobs still inside `perform` (`performing_jobs`): once it returns,
+    # Periodic::ReleaseMiddleware has released the lock, and refreshing it
+    # again would bring it back. A missing lock is only *re-taken* on the
+    # first beat after a failed one — i.e. after Redis was away, possibly
+    # long enough for the lock to lapse; otherwise a missing lock means it
+    # was released, and stays released.
+    def touch_running_periodic_locks(conn, _identity)
+      recovering = @beat_failed
+      @manager.performing_jobs.each do |raw|
+        job = JSON.parse(raw)
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
-        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        if recovering && job['periodic_until_executed']
+          # Re-taken, not just extended: after Redis was away longer than
+          # `active_ttl`, the lock is gone while the run is still going, and
+          # the ticker could start the next slot alongside it.
+          Periodic::RunningLock.retake(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        else
+          Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        end
       rescue StandardError => e
         # Per entry: one bad entry mustn't leave the others' locks to expire.
         Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -186,6 +242,7 @@ module Cogworker
     def subscribe_loop
       @subscribe_client = ::Redis.new(RedisConnection.client_options(Cogworker.config.redis_options))
       @subscribe_client.subscribe(RedisKeys.signal(Cogworker.identity)) do |on|
+        on.subscribe { |*| RedisErrors.recovered('Signal subscriber') }
         on.message do |_channel, message|
           dispatch(message)
         end
@@ -193,7 +250,7 @@ module Cogworker
     rescue StandardError => e
       return if @stopping
 
-      Cogworker.logger.error { "Signal subscriber died: #{e.class}: #{e.message}" }
+      RedisErrors.report('Signal subscriber', e)
       sleep(1)
       retry
     end

data/lib/cogworker/history/middleware.rb

@@ -5,13 +5,17 @@ module Cogworker
     # Server middleware: times every run and records it via Storage,
     # success or failure, regardless of what other middleware/worker code does.
     class Middleware
+      # Recording is best-effort (BestEffort): only the job's own exception
+      # is re-raised, never one from writing its history.
       def call(_worker, job, queue)
         started_at = Time.now.to_f
-        yield
-        Storage.record(job, queue, started_at, Time.now.to_f, 'success')
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e)
-        raise e
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e) }
+          raise e
+        end
+        BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'success') }
       end
     end
   end

data/lib/cogworker/job_util.rb

@@ -37,11 +37,18 @@ module Cogworker
     # define `#to_i` (`nil.to_i` happens to be `0` — coincidentally correct
     # for absent — but `false.to_i` raises `NoMethodError` outright, which is
     # exactly the bug this method exists to not have).
+    #
+    # Anything else (a Hash, a non-numeric String…) means no retries — the
+    # cautious reading for a job with side effects — rather than raising
+    # from the middle of recording its failure.
     def max_retries(job)
-      case job['retry']
+      case (value = job['retry'])
       when false then 0
       when true, nil then DEFAULT_MAX_RETRY_ATTEMPTS
-      else job['retry'].to_i
+      when Integer then value
+      when Float then value.to_i
+      when String then Integer(value, exception: false) || 0
+      else 0
       end
     end
 
@@ -75,12 +82,17 @@ module Cogworker
     # commands, a crash or dropped connection between the ZREM and the
     # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
     # by Lua's cjson (which would round large integers in args).
+    # Present? Push, then remove — never remove first: a script stopped by
+    # an error keeps the writes made before it, so a ZREM followed by a
+    # failing LPUSH (the queue key holding the wrong type) lost the entry.
+    # The `cogworker:queues` listing is best-effort.
     CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
-      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+      if not redis.call("ZSCORE", KEYS[1], ARGV[1]) then
         return 0
       end
-      redis.call("SADD", KEYS[2], ARGV[2])
       redis.call("LPUSH", KEYS[3], ARGV[3])
+      redis.call("ZREM", KEYS[1], ARGV[1])
+      redis.pcall("SADD", KEYS[2], ARGV[2])
       return 1
     LUA
 
@@ -120,8 +132,31 @@ module Cogworker
       limit ? str[0, limit] : str
     end
 
+    # Never raises: an exception whose own `#message` raises (a broken
+    # custom exception class) used to escape from the middle of recording
+    # its failure — and, the job then counting as not-yet-run, it was handed
+    # back and re-run about once a second, forever.
     def error_message(error)
       safe_string(error.message, ERROR_MESSAGE_LIMIT)
+    rescue Exception # rubocop:disable Lint/RescueException
+      "#<#{error.class}>"
+    end
+
+    # A job on its way to dead outside the usual path (no server middleware
+    # around it to do this): releases its `until_executed` locks, owner-
+    # checked, each on its own — so the entry / the unique key isn't blocked
+    # until the locks' TTL. Best-effort.
+    def release_terminal_locks(conn, job)
+      keys = []
+      keys << RedisKeys.unique_lock(UniqueJobs.digest(job)) if UniqueJobs.until_executed?(job)
+      keys << RedisKeys.periodic_running(job['periodic_pjid']) if job['periodic_pjid']
+      keys.each do |key|
+        released = BestEffort.call('Lock release') do
+          OwnedKey.delete(key, job['jid'], conn)
+          true
+        end
+        DeferredReleases.add(key, job['jid']) unless released
+      end
     end
 
     UNPARSEABLE_CLASS = '(unparseable)'

data/lib/cogworker/key_guard.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Cogworker
+  # Finds this gem's Redis keys holding the wrong type of value — which
+  # only a manual write or data corruption produces, but which then breaks
+  # every command on that key, deterministically: a queue that can't be
+  # pushed to, a `dead` set nothing can be filed in, a registry that fails
+  # every beat — and moves each such key out of the way, to
+  # `cogworker:quarantine:<key>:<time>:<random>` (kept QUARANTINE_TTL for a human to
+  # look at), with an error in the log. Everything then carries on with a
+  # fresh, empty key of the right type. Run by every process's Scheduled
+  # poller at boot and every CHECK_INTERVAL.
+  module KeyGuard
+    CHECK_INTERVAL = 60
+    QUARANTINE_TTL = 30 * 24 * 60 * 60
+    QUARANTINE_PREFIX = 'cogworker:quarantine:'
+
+    FIXED = {
+      RedisKeys::SCHEDULE => 'zset', RedisKeys::RETRY => 'zset', RedisKeys::DEAD => 'zset',
+      RedisKeys::QUEUES => 'set', RedisKeys::PROCESSES => 'set', RedisKeys::PAUSED_QUEUES => 'set',
+      RedisKeys::IN_PROGRESS_IDENTITIES => 'set', RedisKeys::LAST_BEAT => 'hash',
+      RedisKeys::REPEAT_ORPHANS => 'list', RedisKeys::UNSETTLED => 'list', RedisKeys::PERIODIC_SCHEDULE => 'hash',
+      RedisKeys::LIVE_QUEUES => 'zset',
+      RedisKeys::PERIODIC_DISABLED => 'set', RedisKeys::STATS_PROCESSED => 'string',
+      RedisKeys::STATS_FAILED => 'string'
+    }.freeze
+
+    # KEYS[1] = the key, KEYS[2] = its quarantine name; ARGV[1] = expected
+    # type, ARGV[2] = quarantine TTL. Re-checked here, atomically with the
+    # move, so a key fixed (or recreated) in between is left alone.
+    QUARANTINE_SCRIPT = <<~LUA
+      local actual = redis.call("TYPE", KEYS[1])["ok"]
+      if actual == "none" or actual == ARGV[1] then
+        return 0
+      end
+      redis.call("RENAME", KEYS[1], KEYS[2])
+      redis.call("EXPIRE", KEYS[2], ARGV[2])
+      return 1
+    LUA
+
+    module_function
+
+    # Returns the keys quarantined.
+    def check(queues: Cogworker.config.queues)
+      expected = expected_types(queues)
+      actual = Cogworker.config.redis do |c|
+        c.pipelined { |p| expected.each_key { |key| p.type(key) } }
+      end
+      expected.keys.zip(actual).filter_map do |key, type|
+        next if type == 'none' || type == expected[key]
+
+        key if quarantine(key, expected[key], type)
+      end
+    end
+
+    # The fixed keys, plus the per-queue and per-process ones currently
+    # listed in them (only where the listing itself is readable).
+    def expected_types(queues)
+      types = FIXED.dup
+      queue_names = Array(queues) | readable_members(RedisKeys::QUEUES)
+      queue_names.each { |q| types[RedisKeys.queue(q)] = 'list' }
+      readable_members(RedisKeys::IN_PROGRESS_IDENTITIES).each { |id| types[RedisKeys.in_progress(id)] = 'list' }
+      readable_members(RedisKeys::PROCESSES).each do |id|
+        types[RedisKeys.process(id)] = 'hash'
+        types[RedisKeys.workers(id)] = 'hash'
+      end
+      readable_fields(RedisKeys::PERIODIC_SCHEDULE).each do |pjid|
+        types[RedisKeys.periodic_running(pjid)] = 'string'
+        types[RedisKeys.periodic_last_slot(pjid)] = 'string'
+      end
+      types
+    end
+
+    def readable_fields(hash)
+      Cogworker.config.redis { |c| c.hkeys(hash) }
+    rescue Redis::CommandError
+      []
+    end
+
+    MAX_COPIES = 5
+
+    # Keeps the newest MAX_COPIES quarantined copies of one key: something
+    # that keeps writing the wrong type would otherwise leave one more every
+    # CHECK_INTERVAL, for QUARANTINE_TTL. (A SCAN, but only on the rare
+    # occasion a key actually gets quarantined.)
+    def prune_copies(key)
+      prefix = "#{QUARANTINE_PREFIX}#{key}:"
+      exact = /\A#{Regexp.escape(prefix)}(\d+):\h{8}\z/ # this key's copies only — not `<key>:<more>:...`'s
+      Cogworker.config.redis do |c|
+        copies = c.scan_each(match: "#{glob_escape(prefix)}*").select { |name| name.match?(exact) }
+        stale = copies.sort_by { |name| name[exact, 1].to_i }.reverse.drop(MAX_COPIES)
+        c.del(*stale) unless stale.empty?
+      end
+    rescue StandardError => e
+      Cogworker.logger.error { "pruning quarantined copies of #{key} failed: #{e.class}: #{e.message}" }
+    end
+
+    def glob_escape(text)
+      text.gsub(/[*?\[\]\\]/) { |ch| "\\#{ch}" }
+    end
+
+    def readable_members(set)
+      Cogworker.config.redis { |c| c.smembers(set) }
+    rescue Redis::CommandError
+      [] # itself of the wrong type: quarantined this round, listed again next
+    end
+
+    def quarantine(key, expected, actual)
+      # Unique per move: a second quarantine of the same key within the same
+      # second must not overwrite (RENAME) the first one's copy.
+      # Milliseconds, so copies made within one second still sort by age.
+      target = "#{QUARANTINE_PREFIX}#{key}:#{(Time.now.to_f * 1000).to_i}:#{SecureRandom.hex(4)}"
+      moved = Cogworker.config.redis do |c|
+        LuaScript.run(c, QUARANTINE_SCRIPT, keys: [key, target], argv: [expected, QUARANTINE_TTL])
+      end
+      return false unless moved == 1
+
+      prune_copies(key)
+      Cogworker.logger.error do
+        "#{key} held a #{actual}, not a #{expected}: moved to #{target} (kept #{QUARANTINE_TTL / 86_400} days)"
+      end
+      true
+    end
+  end
+end

data/lib/cogworker/launcher.rb

@@ -13,14 +13,15 @@ module Cogworker
       @manager = Manager.new(config)
       @scheduled = Scheduled.new(@manager)
       @heartbeat = Heartbeat.new(@manager)
-      # The first tick waits for the first orphan check: after the whole
+      # The first tick waits for the first orphan check that really
+      # completed (Scheduled#ready_for_ticker?): after the whole
       # fleet was down longer than a running `until_executed` job's lock
       # lives, the ticker would otherwise claim that entry's next slot before
       # recovery had requeued the interrupted run (and re-taken its lock) —
       # two runs at once.
       @ticker = Periodic::Ticker.new(
         @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
-                                                   ready: -> { @scheduled.recovered_once? }
+                                                   ready: -> { @scheduled.ready_for_ticker? }
       )
       @signal_queue = ::Queue.new
     end
@@ -44,6 +45,9 @@ module Cogworker
 
     def stop!
       @manager.stop!
+      # Last chance for releases deferred by a recent outage — they live in
+      # this process's memory and would otherwise wait out their locks' TTL.
+      BestEffort.call('Deferred lock releases') { @config.redis { |c| DeferredReleases.retry_all(c) } }
       @scheduled.stop!
       @ticker.stop!
       @heartbeat.stop!

data/lib/cogworker/lua_script.rb

@@ -9,8 +9,8 @@ module Cogworker
   # restarted or failed-over server, SCRIPT FLUSH). Every script in this
   # gem goes through here.
   module LuaScript
-    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
-    SHAS_MUTEX = Mutex.new
+    @shas = {} # source => sha1; memoized, the sources are frozen constants
+    @shas_mutex = Mutex.new
 
     module_function
 
@@ -24,7 +24,7 @@ module Cogworker
     end
 
     def sha(source)
-      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+      @shas_mutex.synchronize { @shas[source] ||= Digest::SHA1.hexdigest(source) }
     end
   end
 end

data/lib/cogworker/manager.rb

@@ -7,6 +7,17 @@ module Cogworker
     def initialize(config = Cogworker.config)
       @config = config
       @fetch_mutex = Mutex.new
+      # This process's own record of its jobs (raw payloads, as fetched):
+      # running right now (a count — two copies of one payload can run at
+      # once), and finished but not yet settled in Redis — `:ack` (the ack
+      # failed) or `[set, score, payload]` (the job ran and failed, and
+      # filing it in retry/dead failed too). In memory on purpose:
+      # ReliableFetch.reconcile trusts it over anything in Redis, which may
+      # be what's failing (or what lost its last writes in a failover).
+      @jobs_mutex = Mutex.new
+      @running_jobs = Hash.new(0)
+      @performing_jobs = Hash.new(0) # the subset still inside `perform`
+      @pending_settlements = {}
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -63,6 +74,51 @@ module Cogworker
       @stopping
     end
 
+    def job_started(raw)
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] += 1
+        @performing_jobs[raw] += 1
+      end
+    end
+
+    # `perform` (with its middleware) has returned: the job is still this
+    # process's until settled, but its locks are no longer its to keep
+    # alive — Heartbeat stops refreshing them (see `performing_jobs`).
+    def job_performed(raw)
+      @jobs_mutex.synchronize do
+        @performing_jobs[raw] -= 1
+        @performing_jobs.delete(raw) unless @performing_jobs[raw].positive?
+      end
+    end
+
+    def performing_jobs
+      @jobs_mutex.synchronize { @performing_jobs.keys }
+    end
+
+    def job_finished(raw)
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] -= 1
+        @running_jobs.delete(raw) unless @running_jobs[raw].positive?
+      end
+    end
+
+    def running_jobs
+      @jobs_mutex.synchronize { @running_jobs.keys }
+    end
+
+    # `how`: :ack, or [set, score, payload] — see ReliableFetch.settle.
+    def settle_later(raw, how)
+      @jobs_mutex.synchronize { @pending_settlements[raw] = how }
+    end
+
+    def settled(raw)
+      @jobs_mutex.synchronize { @pending_settlements.delete(raw) }
+    end
+
+    def pending_settlements
+      @jobs_mutex.synchronize { @pending_settlements.dup }
+    end
+
     def busy_count
       @busy_mutex.synchronize { @busy_count }
     end
@@ -109,8 +165,27 @@ module Cogworker
       end
     end
 
+    # Jobs that finished but aren't settled in Redis yet are still on the
+    # in-progress list. One more try at settling them first; then `except:`
+    # drops the ones only waiting for their ack (which is what dropping
+    # them is), in the same atomic step as requeueing the rest — a separate
+    # ack pass that failed used to leave them to be requeued and run again.
+    # Ones that couldn't be filed in retry/dead even now are handed over to
+    # UNSETTLED for any live process to file later (ReliableFetch.park_unsettled)
+    # — never requeued: they already ran. Any that can't even be handed
+    # over are left on the list (`keep:`): orphan recovery requeues them
+    # once this process is gone, at most `max_orphanings` times.
     def requeue_unfinished
-      moved = fetch_class.requeue_in_progress(Cogworker.identity)
+      keep = []
+      if fetch_class == ReliableFetch
+        ReliableFetch.settle_pending(Cogworker.identity, pending_settlements) { |raw| settled(raw) }
+        unfiled = pending_settlements.reject { |_raw, how| how == :ack }
+        keep = ReliableFetch.park_unsettled(Cogworker.identity, unfiled)
+        (unfiled.keys - keep).each { |raw| settled(raw) }
+      end
+      to_ack = pending_settlements.select { |_raw, how| how == :ack }.keys
+      moved = fetch_class.requeue_in_progress(Cogworker.identity, except: to_ack, keep: keep)
+      to_ack.each { |raw| settled(raw) }
       Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
     rescue StandardError => e
       Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }

data/lib/cogworker/periodic/claim.lua

@@ -10,7 +10,7 @@
 -- Returns {1, <previous last_slot, or "" if none>} if this call won the
 -- claim for this slot (the caller should enqueue the job — and, if that
 -- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
--- value), 0 otherwise. Composes two guards: `last_slot` stops a
+-- value), 2 if blocked by the entry's running lock, 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
@@ -18,8 +18,11 @@
 -- For `until_executed`, the running lock is written here, in the same
 -- atomic step as the claim and before the job exists anywhere — see
 -- Periodic::RunningLock for why it must never be written after the push.
+-- Blocked by a run still holding the entry's lock: a distinct 2, so the
+-- caller can retry the slot briefly — the lock may only be waiting for a
+-- release delayed by an outage (see Ticker::BLOCKED_GRACE).
 if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
-  return 0
+  return 2
 end
 
 local previous = redis.call("GET", KEYS[2])