cogworker 0.2.2 → 0.4.0

data/lib/cogworker/periodic/claim.lua

@@ -4,17 +4,26 @@
 -- ARGV[1] = slot (epoch int)
 -- ARGV[2] = unique mode ("until_executed" or "")
 -- ARGV[3] = lock TTL (seconds)
+-- ARGV[4] = jid the caller will push this slot's job under
+-- ARGV[5] = running-lock TTL (seconds)
 --
--- Returns 1 if this call won the claim for this slot (the caller should
--- enqueue the job), 0 otherwise. Composes two guards: `last_slot` stops a
+-- Returns {1, <previous last_slot, or "" if none>} if this call won the
+-- claim for this slot (the caller should enqueue the job — and, if that
+-- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
+-- value), 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
+--
+-- For `until_executed`, the running lock is written here, in the same
+-- atomic step as the claim and before the job exists anywhere — see
+-- Periodic::RunningLock for why it must never be written after the push.
 if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
   return 0
 end
 
-local last = tonumber(redis.call("GET", KEYS[2]) or "0")
+local previous = redis.call("GET", KEYS[2])
+local last = tonumber(previous or "0")
 if tonumber(ARGV[1]) <= last then
   return 0
 end
@@ -24,4 +33,7 @@ if not redis.call("SET", KEYS[3], "1", "NX", "EX", ARGV[3]) then
 end
 
 redis.call("SET", KEYS[2], ARGV[1])
-return 1
+if ARGV[2] == "until_executed" then
+  redis.call("SET", KEYS[1], ARGV[4], "EX", ARGV[5])
+end
+return {1, previous or ""}

data/lib/cogworker/periodic/release_middleware.rb

@@ -2,25 +2,43 @@
 
 module Cogworker
   module Periodic
-    # Releases the `periodic:running:<pjid>` lock (used only for
-    # `unique: :until_executed` entries) on success, or on the terminal
-    # failed attempt. A no-op for any job that isn't periodic-scheduled
-    # (`job['periodic_pjid']` absent). Registered unconditionally in every
-    # Config, not gated on whether `config.periodic` is actually used, so it
-    # never becomes a hidden dependency of the (separate) job status layer.
+    # Drives the server side of the `periodic:running:<pjid>` lock (used
+    # only for `unique: :until_executed` entries; see RunningLock for the
+    # whole lifecycle): shortens it to `RunningLock.active_ttl` when the run
+    # starts, and releases it on success or on the terminal failed attempt
+    # (a failure that will be retried is `Processor#route_failure`'s to
+    # extend — only it knows the retry delay).
+    # A no-op for any job that isn't periodic-scheduled (`job['periodic_pjid']`
+    # absent) — and every call is a no-op on a lock owned by another jid, so
+    # it's harmless for a non-`until_executed` entry too. Registered
+    # unconditionally in every Config, not gated on whether
+    # `config.periodic` is actually used, so it never becomes a hidden
+    # dependency of the (separate) job status layer.
     class ReleaseMiddleware
       def call(_worker, job, _queue)
-        yield
-        release(job) if job['periodic_pjid']
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        release(job) if job['periodic_pjid'] && JobUtil.terminal_failure?(job)
-        raise e
+        pjid = job['periodic_pjid']
+        return yield unless pjid
+
+        shorten_lock(pjid, job['jid'])
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          RunningLock.release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
+          raise e
+        end
+        RunningLock.release(pjid, job['jid'])
       end
 
       private
 
-      def release(job)
-        Cogworker.config.redis { |c| c.del(RedisKeys.periodic_running(job['periodic_pjid'])) }
+      # Best-effort: failing here must not stop the job from running (it
+      # would be routed to retry/dead as if it had failed). If it does fail,
+      # the lock just keeps its longer queued TTL until this process's next
+      # heartbeat shortens it — within Heartbeat::INTERVAL.
+      def shorten_lock(pjid, jid)
+        RunningLock.touch(pjid, jid, RunningLock.active_ttl)
+      rescue StandardError => e
+        Cogworker.logger.warn { "couldn't shorten periodic lock for #{pjid}: #{e.class}: #{e.message}" }
       end
     end
   end

data/lib/cogworker/periodic/running_lock.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Periodic
+    # `periodic:running:<pjid>` — the "an earlier run of this entry hasn't
+    # finished yet" lock behind `unique: :until_executed` periodic entries.
+    #
+    # Lifecycle (every write keyed on the owning jid, so a stray older run
+    # can never shorten, extend or release a newer run's lock):
+    # - set by `claim.lua`, atomically with the slot claim itself, *before*
+    #   the job is pushed — setting it after the push let a fast job finish
+    #   (and release a lock that didn't exist yet) before the lock was
+    #   written, wedging the entry forever. TTL = `queued_ttl`, covering
+    #   however long the job may wait in its queue.
+    # - shortened to `active_ttl` once a processor starts it, then kept alive
+    #   by that process's Heartbeat every beat — so a run whose process dies
+    #   (OOM, SIGKILL, shutdown past the drain timeout) can't hold the entry
+    #   forever. `active_ttl` deliberately outlives the time it takes for
+    #   that dead process's job to be recovered (`config.orphan_threshold`
+    #   plus one orphan check), so the lock is still there, still owned by
+    #   that job, when the job is requeued — at which point `retake` (or ReliableFetch's
+    #   REQUEUE_SCRIPT) extends it again for the wait in the queue. A shorter
+    #   TTL let the lock lapse first: the ticker would enqueue the next slot
+    #   while the recovered run was still to come, i.e. two runs at once.
+    # - extended on a non-terminal failure to cover the retry backoff plus
+    #   `queued_ttl` (`Processor#extend_locks_for_retry` — the job now waits
+    #   in `cogworker:retry`, not on a process).
+    # - deleted on success or on the terminal failure.
+    module RunningLock
+      # Lua, prepended to every script that puts a job back on its queue
+      # (ReliableFetch::REQUEUE_SCRIPT, BasicFetch::REQUEUE_ENTRY_SCRIPT), so
+      # the requeue and the lock move together in one atomic step. Extends
+      # the lock if still that job's, takes it if it lapsed, leaves it if a
+      # newer run holds it.
+      RETAKE_FUNCTION = <<~LUA
+        local function retake_running_lock(lock, jid, ttl)
+          local owner = redis.call("GET", lock)
+          if not owner then
+            redis.call("SET", lock, jid, "EX", ttl)
+          elseif owner == jid then
+            redis.call("EXPIRE", lock, ttl)
+          end
+        end
+      LUA
+
+      module_function
+
+      # Whole seconds: it goes straight into `SET ... EX` in claim.lua, which
+      # rejects a non-integer (a fractional `unique_lock_ttl` used to make
+      # every claim — and so every tick — fail).
+      # A method, not a constant: it follows `config.orphan_threshold`.
+      def active_ttl
+        (Cogworker.config.orphan_threshold + Scheduled::ORPHAN_CHECK_INTERVAL + 60).to_f.ceil
+      end
+
+      def queued_ttl
+        Cogworker.config.unique_lock_ttl.to_f.ceil
+      end
+
+      def touch(pjid, jid, ttl, conn = nil)
+        OwnedKey.expire(RedisKeys.periodic_running(pjid), jid, ttl, conn)
+      end
+
+      def release(pjid, jid, conn = nil)
+        OwnedKey.delete(RedisKeys.periodic_running(pjid), jid, conn)
+      end
+    end
+  end
+end

data/lib/cogworker/periodic/ticker.rb

@@ -2,6 +2,7 @@
 
 require 'fugit'
 require 'json'
+require 'securerandom'
 
 module Cogworker
   module Periodic
@@ -16,6 +17,25 @@ module Cogworker
 
       CLAIM_SCRIPT = File.read(File.join(__dir__, 'claim.lua'))
 
+      # Undoes a won claim whose push then failed, so the slot isn't lost:
+      # puts `last_slot` back (only if it still is this slot — a later slot
+      # may have been claimed since) and drops the per-slot NX lock.
+      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
+      # previous last_slot ("" if none: then slot - 1, which still lets this
+      # slot fire, unlike deleting the key — with catch_up off, a missing
+      # last_slot would be re-primed to this very slot instead).
+      ROLLBACK_SCRIPT = <<~LUA
+        if redis.call("GET", KEYS[1]) == ARGV[1] then
+          if ARGV[2] == "" then
+            redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
+          else
+            redis.call("SET", KEYS[1], ARGV[2])
+          end
+        end
+        redis.call("DEL", KEYS[2])
+        return 1
+      LUA
+
       def initialize(manager, entries, catch_up: true)
         @manager = manager
         @entries = entries
@@ -54,13 +74,20 @@ module Cogworker
         Cogworker.config.redis { |c| c.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) }
       end
 
+      # A failed tick is logged and retried next interval rather than ending
+      # the thread — a single Redis blip used to silently stop every
+      # periodic entry in this process for good. A slot whose claim or push
+      # raised isn't recorded in `@last_checked_slot` (and a failed push
+      # rolls its claim back — see #enqueue), so the next tick retries it.
       def run
         until @manager.stopping?
-          tick unless @manager.quiet?
+          begin
+            tick unless @manager.quiet?
+          rescue StandardError => e
+            Cogworker.logger.error { "Periodic tick failed: #{e.class}: #{e.message}" }
+          end
           sleep(TICK_INTERVAL)
         end
-      rescue StandardError => e
-        Cogworker.logger.error { "Periodic ticker died: #{e.class}: #{e.message}" }
       end
 
       def tick
@@ -69,13 +96,15 @@ module Cogworker
           slot = cron_for(entry).previous_time(now).to_i
           next if @last_checked_slot[entry.pjid] == slot
 
-          enqueue(entry, slot) if !disabled?(entry) && claim?(entry, slot)
+          jid = SecureRandom.hex(12)
+          previous = claim(entry, slot, jid) unless disabled?(entry)
+          enqueue(entry, slot, jid, previous) if previous
           @last_checked_slot[entry.pjid] = slot
         end
       end
 
       # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
-      # step entirely, short-circuiting before `claim?` even runs, so
+      # step entirely, short-circuiting before `claim` even runs, so
       # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
       # while disabled: the Web UI's own "LastRun" column keeps showing the
       # last time it *actually* ran, not a due-but-skipped slot, and
@@ -85,6 +114,15 @@ module Cogworker
       # persisted last_slot) still advances either way, exactly as it
       # already did before this entry ever had a disabled state — it only
       # stops this same tick loop from re-evaluating the same slot twice.
+      def rollback_claim(entry, slot, previous_slot)
+        Cogworker.config.redis do |c|
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
+          c.eval(ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot}: #{e.class}: #{e.message}" }
+      end
+
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
       end
@@ -93,16 +131,18 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      def claim?(entry, slot)
-        return false if !@catch_up && priming_first_slot?(entry, slot)
+      # nil if this process didn't win the slot; otherwise the previous
+      # `last_slot` value ("" if none), for #rollback_claim.
+      def claim(entry, slot, jid)
+        return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           c.eval(CLAIM_SCRIPT,
                  keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
                         RedisKeys.periodic_lock(entry.pjid, slot)],
-                 argv: [slot, entry.unique.to_s, LOCK_TTL])
+                 argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
-        result == 1
+        result.is_a?(Array) ? result[1].to_s : nil
       end
 
       # With catch-up disabled, an entry's very first tick — ever, across
@@ -123,15 +163,40 @@ module Cogworker
         Cogworker.config.redis { |c| c.set(RedisKeys.periodic_last_slot(entry.pjid), slot, nx: true) }
       end
 
-      def enqueue(entry, slot)
+      # The running lock (for `until_executed`) is already in place, set by
+      # the claim under this same `jid` — nothing to write after the push.
+      # That relies on the job keeping that jid: `JobUtil.normalize_item`
+      # only fills `jid` in when absent, but a custom client middleware that
+      # overwrites it would leave the lock owned by a jid no job carries —
+      # the entry then stays closed for `RunningLock.queued_ttl` (logged).
+      # Only undone here if the push didn't actually produce a job (raised,
+      # or a client middleware swallowed it), so a job that never existed
+      # can't hold the entry for the whole `RunningLock.queued_ttl`.
+      def enqueue(entry, slot, jid, previous_slot)
         job = {
           'class' => entry.class_name, 'args' => entry.args, 'retry' => entry.retry,
-          'periodic_pjid' => entry.pjid, 'periodic_slot' => slot
+          'periodic_pjid' => entry.pjid, 'periodic_slot' => slot, 'jid' => jid
         }
-        jid = Client.push(job)
+        # Lets a requeue after a crash/shutdown re-take this run's lock
+        # (Periodic::RunningLock::RETAKE_FUNCTION, in both fetches' requeue scripts).
+        job['periodic_until_executed'] = true if entry.until_executed?
+        pushed = begin
+          Client.push(job)
+        rescue StandardError
+          RunningLock.release(entry.pjid, jid) if entry.until_executed?
+          rollback_claim(entry, slot, previous_slot)
+          raise
+        end
         return unless entry.until_executed?
 
-        Cogworker.config.redis { |c| c.set(RedisKeys.periodic_running(entry.pjid), jid) }
+        if pushed.nil?
+          RunningLock.release(entry.pjid, jid)
+        elsif pushed != jid
+          Cogworker.logger.warn do
+            "periodic #{entry.pjid}: a client middleware changed the job's jid (#{jid} -> #{pushed}); its running " \
+              "lock won't be released by the job and keeps the entry closed for up to #{RunningLock.queued_ttl}s"
+          end
+        end
       end
     end
   end

data/lib/cogworker/process.rb

@@ -6,8 +6,8 @@ module Cogworker
   # that's this same process — e.g. WorkerKiller finding itself via
   # ProcessSet — or a genuinely remote one) is subscribed to it and reacts.
   # `#quiet!`/`#stop!` mirror a real `kill -TSTP`/`TERM`; `#resume!` has no
-  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet in real
-  # Sidekiq-alike tooling) — it's Cogworker-specific, made possible by
+  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet signal) —
+  # it's Cogworker-specific, made possible by
   # `Manager#quiet` being a plain in-memory flag rather than a one-way state
   # transition like `stopping?`.
   #

data/lib/cogworker/process_set.rb

@@ -11,7 +11,7 @@ module Cogworker
       identities.each do |identity|
         info = Cogworker.config.redis { |c| c.hgetall(RedisKeys.process(identity)) }
         if info.empty?
-          Cogworker.config.redis { |c| c.srem(RedisKeys::PROCESSES, identity) }
+          Cogworker.config.redis { |c| c.srem?(RedisKeys::PROCESSES, identity) }
           next
         end
 

data/lib/cogworker/processor.rb

@@ -7,43 +7,126 @@ module Cogworker
   # One thread of a Manager's pool: fetch -> register in-flight -> run the
   # server middleware chain around perform -> stats/retry -> deregister.
   class Processor
+    ERROR_BACKOFF = 1 # seconds
+    ACK_ATTEMPTS = 3
+
     attr_reader :thread, :tid
 
     def initialize(manager)
       @manager = manager
+      @busy = false
       @tid = SecureRandom.hex(6)
-      @fetcher = BasicFetch.new(manager.queues)
     end
 
     def start!
       @thread = Thread.new { run }
     end
 
+    # True while executing a job (as opposed to fetching or idle) — what
+    # `Manager#stop!` uses to tell a thread that will exit on its own within
+    # one fetch timeout from one stuck in a long job.
+    def busy?
+      @busy
+    end
+
     private
 
+    # An error escaping one iteration (a Redis blip in `retrieve_work` or
+    # the in-flight bookkeeping) is logged and the loop carries on after
+    # ERROR_BACKOFF — it used to end the thread for good, with nothing ever
+    # restarting it, so the process silently lost capacity one slot at a
+    # time.
     def run
       until @manager.stopping?
-        if @manager.quiet?
-          sleep(0.5)
-          next
+        begin
+          process_one
+        rescue StandardError => e
+          Cogworker.logger.error { "Processor error: #{e.class}: #{e.message}" }
+          sleep(ERROR_BACKOFF)
         end
+      end
+    end
 
-        work = @fetcher.retrieve_work
-        next unless work
+    def process_one
+      if @manager.quiet?
+        sleep(0.5)
+        return
+      end
 
-        @manager.processor_busy!
-        begin
-          execute(work)
-        ensure
-          @manager.processor_idle!
-        end
+      work = begin
+        fetcher.retrieve_work
+      rescue ReliableFetch::UnsupportedError => e
+        @manager.fall_back_to_basic_fetch!(e)
+        @fetcher = nil
+        return
+      end
+      return unless work
+
+      # Fetched after shutdown began (e.g. still blocked in BRPOP when
+      # `Manager#stop!` requeued unfinished work, which it then popped right
+      # back): hand it back rather than start it, since the process is about
+      # to exit underneath it.
+      return fetcher.give_back(work) if @manager.stopping?
+
+      @manager.processor_busy!
+      @busy = true
+      finished = false
+      begin
+        execute(work)
+        finished = true
+      ensure
+        @busy = false
+        @manager.processor_idle!
+        finished ? acknowledge(work) : give_back(work)
       end
+    end
+
+    # `execute` raising means bookkeeping failed (Redis, mid-way through
+    # registering the job or routing its failure to retry/dead) — not the
+    # job itself, whose own errors `execute` always handles. Acknowledging
+    # here would drop a job that never made it to retry/dead; instead it
+    # goes straight back on its queue (it may run again: at-least-once). If
+    # even that fails, it simply stays on the in-progress list, which
+    # `Manager#stop!`/orphan recovery requeue later.
+    def give_back(work)
+      fetcher.give_back(work)
     rescue StandardError => e
-      Cogworker.logger.error { "Processor thread died: #{e.class}: #{e.message}" }
+      Cogworker.logger.error do
+        "couldn't hand back jid=#{jid_of(work)} after a failed run (left in progress): #{e.class}: #{e.message}"
+      end
+    end
+
+    # A failed ack leaves an already-finished job on this process's own
+    # in-progress list, where `Manager#stop!` would later requeue it — a
+    # silent second run. Retried briefly, then logged with the jid so that
+    # duplicate can at least be traced.
+    def acknowledge(work)
+      attempts = 0
+      begin
+        attempts += 1
+        fetcher.acknowledge(work)
+      rescue StandardError => e
+        if attempts < ACK_ATTEMPTS
+          sleep(0.1 * attempts)
+          retry
+        end
+        Cogworker.logger.error do
+          "couldn't acknowledge finished job jid=#{jid_of(work)} after #{attempts} attempts " \
+            "(it may run again on shutdown): #{e.class}: #{e.message}"
+        end
+      end
+    end
+
+    def jid_of(work)
+      JSON.parse(work.raw_job)['jid']
+    rescue StandardError
+      '?'
     end
 
     def execute(work)
-      job = JSON.parse(work.raw_job)
+      job = parse_job(work)
+      return unless job
+
       register_in_workers(work.queue, job)
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
 
@@ -89,6 +172,36 @@ module Cogworker
       deregister_from_workers
     end
 
+    # A payload that isn't a JSON object goes straight to `dead`
+    # (`JobUtil.unparseable_job`), and `nil` tells `execute` to skip it.
+    # Before this, `JSON.parse` raised outside any rescue: the job was lost
+    # and the processor thread died with it.
+    def parse_job(work)
+      job = JSON.parse(work.raw_job)
+      raise JSON::ParserError, "expected a JSON object, got #{job.class}" unless job.is_a?(Hash)
+
+      job
+    rescue JSON::ParserError => e
+      bury_unparseable(work, e)
+      nil
+    end
+
+    def bury_unparseable(work, error)
+      job = JobUtil.unparseable_job(work.raw_job, queue: work.queue, error: error)
+      Cogworker.config.redis do |c|
+        c.zadd(RedisKeys::DEAD, job['failed_at'], JSON.generate(job))
+        c.incr(RedisKeys::STATS_FAILED)
+      end
+      Throughput.record('failed')
+      Cogworker.logger.error { "unparseable job on queue #{work.queue} moved to dead: #{error.message}" }
+    end
+
+    # Built on first use, not in the constructor (see Manager#fetch_class),
+    # and rebuilt after a fall-back to BasicFetch.
+    def fetcher
+      @fetcher ||= @manager.fetch_class.new(@manager.queues)
+    end
+
     def build_worker(job)
       klass = Object.const_get(job['class'])
       worker = klass.new
@@ -108,6 +221,7 @@ module Cogworker
       if new_count <= max_retries
         delay = retry_delay(new_count)
         Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
+        extend_locks_for_retry(job, delay)
         Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
       else
         Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
@@ -115,6 +229,22 @@ module Cogworker
       end
     end
 
+    # A job waiting out its retry backoff still holds its `until_executed`
+    # lock(s) — extended here to cover that whole wait plus the usual
+    # `unique_lock_ttl`, since a fixed TTL counted from the original push
+    # ran out during a long retry series (the delay grows as `count**4`)
+    # and let a duplicate in. Owner-checked, so a lock that has since
+    # passed to another jid is left alone.
+    def extend_locks_for_retry(job, delay)
+      ttl = delay + Cogworker.config.unique_lock_ttl
+      Cogworker.config.redis do |c|
+        if UniqueJobs.until_executed?(job)
+          OwnedKey.expire(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'], ttl, c)
+        end
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], ttl, c) if job['periodic_pjid']
+      end
+    end
+
     # Grows with the retry count, with jitter to avoid a thundering herd of
     # retries all landing on the same second.
     def retry_delay(count)

data/lib/cogworker/queue.rb

@@ -33,8 +33,8 @@ module Cogworker
     end
 
     # Removes every occurrence of this exact raw job entry (matched by full
-    # JSON string, same "raw" identity `Routes::Dead`/`Routes::Retries`
-    # already key their own delete/retry actions off).
+    # JSON string, same "raw" identity `Routes::Jobs`' dead/retrying
+    # actions already key their own delete/retry off).
     def delete(raw)
       Cogworker.config.redis { |c| c.lrem(@key, 0, raw) }
     end
@@ -49,11 +49,11 @@ module Cogworker
     # are untouched and keep arriving via `Client.push`, they just stop
     # being picked up until `resume!`.
     def pause!
-      Cogworker.config.redis { |c| c.sadd(RedisKeys::PAUSED_QUEUES, name) }
+      Cogworker.config.redis { |c| c.sadd?(RedisKeys::PAUSED_QUEUES, name) }
     end
 
     def resume!
-      Cogworker.config.redis { |c| c.srem(RedisKeys::PAUSED_QUEUES, name) }
+      Cogworker.config.redis { |c| c.srem?(RedisKeys::PAUSED_QUEUES, name) }
     end
 
     def paused?

data/lib/cogworker/redis_keys.rb

@@ -9,6 +9,14 @@ module Cogworker
   # that nothing else reads.
   module RedisKeys
     QUEUE_PREFIX = 'cogworker:queue:'
+    IN_PROGRESS_PREFIX = 'cogworker:inprogress:'
+    # Every identity that has (or may have) an in-progress list — what
+    # orphan recovery walks instead of SCANning the whole keyspace.
+    IN_PROGRESS_IDENTITIES = 'cogworker:inprogress_identities'
+    # identity => Redis server time (epoch seconds) of its last heartbeat.
+    # No TTL, unlike `process(identity)`: it's what tells "dead for a
+    # while" apart from "missed a few beats" (Config#orphan_threshold).
+    LAST_BEAT = 'cogworker:last_beat'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'
@@ -26,6 +34,7 @@ module Cogworker
     def job_attempts(jid) = "cogworker:job_attempts:#{jid}"
     def process(identity) = "cogworker:process:#{identity}"
     def workers(identity) = "cogworker:workers:#{identity}"
+    def in_progress(identity) = "#{IN_PROGRESS_PREFIX}#{identity}"
     def signal(identity) = "cogworker:signal:#{identity}"
     def periodic_running(pjid) = "periodic:running:#{pjid}"
     def periodic_last_slot(pjid) = "periodic:last_slot:#{pjid}"

data/lib/cogworker/reliable_fetch/unsupported_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Cogworker
+  class ReliableFetch
+    # Raised by ReliableFetch#retrieve_work when the server turns out not to
+    # know LMOVE (Redis < 6.2) even though the version check at boot let
+    # `:reliable` through (e.g. Redis wasn't reachable then). The Processor
+    # reacts by switching the whole Manager over to BasicFetch.
+    class UnsupportedError < StandardError; end
+  end
+end