cogworker 0.4.0 → 0.5.0

data/lib/cogworker/periodic/ticker.rb

@@ -20,10 +20,14 @@ module Cogworker
       # Undoes a won claim whose push then failed, so the slot isn't lost:
       # puts `last_slot` back (only if it still is this slot — a later slot
       # may have been claimed since) and drops the per-slot NX lock.
-      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
-      # previous last_slot ("" if none: then slot - 1, which still lets this
-      # slot fire, unlike deleting the key — with catch_up off, a missing
-      # last_slot would be re-primed to this very slot instead).
+      # KEYS[1] = last_slot, KEYS[2] = slot lock, KEYS[3] = running lock;
+      # ARGV[1] = slot, ARGV[2] = previous last_slot ("" if none: then
+      # slot - 1, which still lets this slot fire, unlike deleting the key —
+      # with catch_up off, a missing last_slot would be re-primed to this
+      # very slot instead), ARGV[3] = the jid the claim took the running lock
+      # under. Releasing that lock happens here too, in the same step: as a
+      # separate call made first, its failure (Redis down — the usual reason
+      # the push failed) skipped the rollback altogether.
       ROLLBACK_SCRIPT = <<~LUA
         if redis.call("GET", KEYS[1]) == ARGV[1] then
           if ARGV[2] == "" then
@@ -33,19 +37,36 @@ module Cogworker
           end
         end
         redis.call("DEL", KEYS[2])
+        -- Best-effort (an entry without until_executed has no lock; one of
+        -- the wrong type mustn't make an applied rollback count as failed
+        -- and be retried forever).
+        if redis.pcall("GET", KEYS[3]) == ARGV[3] then
+          redis.pcall("DEL", KEYS[3])
+        end
         return 1
       LUA
 
-      def initialize(manager, entries, catch_up: true)
+      def initialize(manager, entries, catch_up: true, ready: -> { true })
         @manager = manager
         @entries = entries
         @catch_up = catch_up
+        @ready = ready
         @last_checked_slot = {}
         @cron_cache = {}
+        @pending_rollbacks = []
+        @failures = Hash.new(0) # pjid => consecutive failed ticks
+        @retry_at = {}          # pjid => monotonic time it may be tried again
       end
 
+      # Publishing is only for restart survival and the Web UI: its failure
+      # (Redis down at boot, the key holding the wrong type) is logged, and
+      # must not take the process down with it.
       def start!
-        publish_schedule!
+        begin
+          publish_schedule!
+        rescue StandardError => e
+          Cogworker.logger.error { "Periodic schedule not published: #{e.class}: #{e.message}" }
+        end
         @thread = Thread.new { run }
       end
 
@@ -82,7 +103,7 @@ module Cogworker
       def run
         until @manager.stopping?
           begin
-            tick unless @manager.quiet?
+            tick unless @manager.quiet? || !@ready.call
           rescue StandardError => e
             Cogworker.logger.error { "Periodic tick failed: #{e.class}: #{e.message}" }
           end
@@ -91,15 +112,91 @@ module Cogworker
       end
 
       def tick
+        retry_pending_rollbacks
         now = Time.now
         @entries.each do |entry|
-          slot = cron_for(entry).previous_time(now).to_i
-          next if @last_checked_slot[entry.pjid] == slot
+          next if backing_off?(entry.pjid)
+
+          tick_entry(entry, now)
+          @failures.delete(entry.pjid)
+        rescue StandardError => e
+          # Per entry: one entry failing (its keys of the wrong type, its
+          # push failing) used to skip every entry after it, on every tick.
+          # Its slot isn't recorded as checked, so it's retried — after an
+          # exponential pause (`back_off`), not on every 5s tick, which made a
+          # deterministically failing entry hammer Redis and the log. Not for
+          # a lost connection, though: that's Redis being away, not the
+          # entry failing — backing off every entry on it left cron idle for
+          # minutes after Redis came back.
+          delay = e.is_a?(Redis::BaseConnectionError) ? TICK_INTERVAL : back_off(entry.pjid)
+          Cogworker.logger.error { "Periodic tick failed for #{entry.pjid} (next try in #{delay}s): #{e.class}: #{e.message}" }
+        end
+      end
+
+      BACKOFF_MAX = 300
+
+      def backing_off?(key)
+        (at = @retry_at[key]) && ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < at
+      end
+
+      # 5s, 10s, 20s … up to BACKOFF_MAX; reset by the next success.
+      def back_off(key)
+        @failures[key] += 1
+        delay = [TICK_INTERVAL * (2**(@failures[key] - 1)), BACKOFF_MAX].min
+        @retry_at[key] = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + delay
+        delay
+      end
+
+      def tick_entry(entry, now)
+        slot = cron_for(entry).previous_time(now).to_i
+        return if @last_checked_slot[entry.pjid] == slot
+
+        jid = SecureRandom.hex(12)
+        previous = claim(entry, slot, jid) unless disabled?(entry)
+        enqueue(entry, slot, jid, previous) if previous
+        @last_checked_slot[entry.pjid] = slot
+      end
 
-          jid = SecureRandom.hex(12)
-          previous = claim(entry, slot, jid) unless disabled?(entry)
-          enqueue(entry, slot, jid, previous) if previous
-          @last_checked_slot[entry.pjid] = slot
+      # The usual reason a push fails — Redis unreachable — makes this fail
+      # too; it's then kept and retried at the start of every tick until it
+      # goes through (before that tick looks at the slot again), rather than
+      # just logged and the slot lost. Returns true, or the error. (A push
+      # that timed out *after* Redis had applied it is the one case this
+      # turns into a second run of the slot: indistinguishable from one that
+      # never arrived.) Pending
+      # rollbacks live in this process's memory only: one still pending when
+      # the process restarts is lost — that slot is skipped, and an
+      # `until_executed` entry stays locked until its lock's TTL.
+      def rollback_claim(entry, slot, previous_slot, jid)
+        Cogworker.config.redis do |c|
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot),
+                  RedisKeys.periodic_running(entry.pjid)]
+          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot, jid])
+        end
+        true
+      rescue StandardError => e
+        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
+        @pending_rollbacks << [entry, slot, previous_slot, jid]
+        e # (truthy, but not `true`: the caller tells a lost connection from a failing rollback)
+      end
+
+      # With the same backoff as a failing entry (keyed by the rollback): a
+      # rollback that keeps failing is retried less and less often, but
+      # always before that entry's slot is looked at again.
+      def retry_pending_rollbacks
+        pending = @pending_rollbacks
+        @pending_rollbacks = []
+        pending.each do |args|
+          key = "rollback:#{args[0].pjid}:#{args[1]}"
+          next @pending_rollbacks << args if backing_off?(key)
+
+          result = rollback_claim(*args)
+          if result == true
+            @failures.delete(key)
+            @retry_at.delete(key)
+          elsif !result.is_a?(Redis::BaseConnectionError)
+            back_off(key) # only a rollback failing on its own backs off; see #tick
+          end
         end
       end
 
@@ -114,17 +211,18 @@ module Cogworker
       # persisted last_slot) still advances either way, exactly as it
       # already did before this entry ever had a disabled state — it only
       # stops this same tick loop from re-evaluating the same slot twice.
-      def rollback_claim(entry, slot, previous_slot)
-        Cogworker.config.redis do |c|
-          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
-          c.eval(ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
-        end
-      rescue StandardError => e
-        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot}: #{e.class}: #{e.message}" }
-      end
-
+      #
+      # A disabled-set that can't be read (the key holding the wrong type)
+      # counts as "not disabled", logged: it used to fail every entry's tick,
+      # stopping all cron. Losing a Disable is the lesser evil. A dropped
+      # connection still raises (the tick is then retried as a whole).
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
+      rescue Redis::BaseConnectionError
+        raise
+      rescue StandardError => e
+        Cogworker.logger.error { "Periodic: can't read #{RedisKeys::PERIODIC_DISABLED} (#{e.class}: #{e.message}); treating #{entry.pjid} as enabled" }
+        false
       end
 
       def cron_for(entry)
@@ -137,7 +235,7 @@ module Cogworker
         return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
-          c.eval(CLAIM_SCRIPT,
+          LuaScript.run(c, CLAIM_SCRIPT,
                  keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
                         RedisKeys.periodic_lock(entry.pjid, slot)],
                  argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
@@ -183,8 +281,7 @@ module Cogworker
         pushed = begin
           Client.push(job)
         rescue StandardError
-          RunningLock.release(entry.pjid, jid) if entry.until_executed?
-          rollback_claim(entry, slot, previous_slot)
+          rollback_claim(entry, slot, previous_slot, jid)
           raise
         end
         return unless entry.until_executed?

data/lib/cogworker/processor.rb

@@ -6,9 +6,26 @@ require 'securerandom'
 module Cogworker
   # One thread of a Manager's pool: fetch -> register in-flight -> run the
   # server middleware chain around perform -> stats/retry -> deregister.
+  #
+  # What happens to the fetched job afterwards depends on how far it got —
+  # the one rule being that a job is never re-run because of a failure that
+  # happened *after* it ran:
+  # - failed before `perform` (registering it in `cogworker:workers`):
+  #   `give_back` — straight back on its queue, it hasn't run yet;
+  # - ran (succeeded, or failed and was recorded in retry/dead): acknowledge.
+  #   Everything after `perform` other than that one retry/dead write —
+  #   stats, throughput, deregistering, lock extension, the attempts log —
+  #   is best-effort: logged, never raised;
+  # - ran and failed, but the retry/dead write itself failed: `interrupt` —
+  #   into `retry` after INTERRUPT_DELAY, counting `interrupted_count`, and
+  #   into `dead` once that passes MAX_INTERRUPTS. Never straight back on
+  #   the queue: a write that fails deterministically used to make that
+  #   loop forever, re-running the job every second.
   class Processor
     ERROR_BACKOFF = 1 # seconds
     ACK_ATTEMPTS = 3
+    MAX_INTERRUPTS = 3
+    INTERRUPT_DELAY = 30 # seconds, times the interruption count
 
     attr_reader :thread, :tid
 
@@ -69,33 +86,104 @@ module Cogworker
       return fetcher.give_back(work) if @manager.stopping?
 
       @manager.processor_busy!
+      @manager.job_started(work.raw_job)
       @busy = true
-      finished = false
+      @ran = nil
+      disposition = nil
       begin
-        execute(work)
-        finished = true
+        disposition = execute(work)
       ensure
         @busy = false
         @manager.processor_idle!
-        finished ? acknowledge(work) : give_back(work)
+        # `execute` raising leaves `disposition` unset: before the job ran
+        # that's a hand-back; once it ran, whatever its outcome dictates
+        # (`@ran`, set the moment `run_job` returns) — so an unexpected
+        # exception later in the bookkeeping (a bug, a misbehaving logger)
+        # can't re-queue a job that already ran.
+        settle(work, disposition || @ran || :give_back)
+        @manager.job_finished(work.raw_job)
       end
     end
 
-    # `execute` raising means bookkeeping failed (Redis, mid-way through
-    # registering the job or routing its failure to retry/dead) — not the
-    # job itself, whose own errors `execute` always handles. Acknowledging
-    # here would drop a job that never made it to retry/dead; instead it
-    # goes straight back on its queue (it may run again: at-least-once). If
-    # even that fails, it simply stays on the in-progress list, which
-    # `Manager#stop!`/orphan recovery requeue later.
+    def settle(work, disposition)
+      case disposition
+      when :acknowledge then acknowledge(work)
+      when :give_back then give_back(work)
+      else interrupt(work, *disposition.drop(1))
+      end
+    end
+
+    # Only for a job that hasn't run yet (see the class comment). If even
+    # this fails, it stays on the in-progress list, which `Manager#stop!` /
+    # orphan recovery / `ReliableFetch.reconcile` requeue later.
     def give_back(work)
       fetcher.give_back(work)
     rescue StandardError => e
       Cogworker.logger.error do
-        "couldn't hand back jid=#{jid_of(work)} after a failed run (left in progress): #{e.class}: #{e.message}"
+        "couldn't hand back jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
+      end
+    end
+
+    # The job ran and failed, but recording that in retry/dead failed. Built
+    # from the original payload (always serializable — it came from JSON)
+    # plus the failure fields, never from the in-memory job hash: whatever
+    # made that one fail to serialize must not do it again here.
+    def interrupt(work, job, error)
+      payload = JSON.parse(work.raw_job)
+      count = payload['interrupted_count'].to_i + 1
+      # This attempt's number, from the payload as fetched — not from the
+      # in-memory job, whose count `route_failure` may not have got to.
+      attempt = payload['retry_count'].to_i + 1
+      payload.merge!('interrupted_count' => count, 'retry_count' => attempt,
+                     'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error),
+                     'failed_at' => Time.now.to_f)
+      # No retries left (`retry: false`/`0`, or the last one used up) means
+      # no extra attempt either — straight to dead, like any terminal failure.
+      out_of_retries = attempt > JobUtil.max_retries(payload)
+      delay = INTERRUPT_DELAY * count
+      set, score = if out_of_retries || count > MAX_INTERRUPTS
+                     [RedisKeys::DEAD, Time.now.to_f]
+                   else
+                     [RedisKeys::RETRY, Time.now.to_f + delay]
+                   end
+      encoded = JSON.generate(payload)
+      begin
+        fetcher.interrupt(work, set, score, encoded)
+      rescue StandardError
+        # Remembered with its payload: this process's reconcile (and its
+        # shutdown) keep trying to file it — never re-run it as a stray.
+        @manager.settle_later(work.raw_job, [set, score, encoded]) if fetcher.is_a?(ReliableFetch)
+        raise
+      end
+      best_effort('lock extension') { extend_locks_for_retry(job, delay) } if set == RedisKeys::RETRY
+      reason = if set == RedisKeys::RETRY
+                 "retrying in #{delay}s (interruption #{count} of at most #{MAX_INTERRUPTS})"
+               elsif out_of_retries
+                 'moved to dead: no retries left'
+               else
+                 "moved to dead: interrupted more than #{MAX_INTERRUPTS} times"
+               end
+      Cogworker.logger.error { "couldn't record the failure of jid=#{payload['jid']} (#{error.class}); #{reason}" }
+    rescue StandardError => e
+      # Whatever failed (even before anything was written), remember the job
+      # with the simplest possible dead entry, so reconcile files it — not
+      # leave it a stray to be re-run. (Already remembered if it got as far
+      # as the write.)
+      remember_for_dead(work, error) if fetcher.is_a?(ReliableFetch) && !@manager.pending_settlements.key?(work.raw_job)
+      Cogworker.logger.error do
+        "couldn't record or interrupt jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
       end
     end
 
+    def remember_for_dead(work, error)
+      entry = JSON.generate('jid' => jid_of(work), 'error_class' => error.class.name.to_s,
+                            'error_message' => JobUtil.error_message(error), 'failed_at' => Time.now.to_f,
+                            'raw_payload' => JobUtil.safe_string(work.raw_job, 100_000))
+      @manager.settle_later(work.raw_job, [RedisKeys::DEAD, Time.now.to_f, entry])
+    rescue StandardError
+      nil
+    end
+
     # A failed ack leaves an already-finished job on this process's own
     # in-progress list, where `Manager#stop!` would later requeue it — a
     # silent second run. Retried briefly, then logged with the jid so that
@@ -110,9 +198,12 @@ module Cogworker
           sleep(0.1 * attempts)
           retry
         end
+        # Remembered, so this process's own reconcile finishes the ack later
+        # instead of mistaking the entry for a stray and re-running it.
+        @manager.settle_later(work.raw_job, :ack)
         Cogworker.logger.error do
           "couldn't acknowledge finished job jid=#{jid_of(work)} after #{attempts} attempts " \
-            "(it may run again on shutdown): #{e.class}: #{e.message}"
+            "(will retry; it may run again if this process stops first): #{e.class}: #{e.message}"
         end
       end
     end
@@ -123,53 +214,86 @@ module Cogworker
       '?'
     end
 
+    # Returns how `settle` should dispose of the fetched job. If it raises
+    # instead, `process_one` falls back on `@ran` — set the moment the job
+    # has run — or, before that, on :give_back.
     def execute(work)
       job = parse_job(work)
-      return unless job
-
-      register_in_workers(work.queue, job)
+      return :acknowledge unless job
+
+      # Best-effort: for the reliable fetch the in-progress list is what
+      # keeps the job safe; `cogworker:workers` only feeds the Workers tab
+      # (and BasicFetch's shutdown requeue). A failure here — even a
+      # deterministic one, the key holding the wrong type — used to hand
+      # the job back before it ever ran, over and over.
+      best_effort('register') { register_in_workers(work.queue, job) }
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
-
       begin
-        # `build_worker` itself can raise (most commonly `NameError`, e.g. a
-        # class the caller registered/enqueued but never actually defined —
-        # deliberately still routed through the *same* middleware chain
-        # below rather than caught here directly: `History::Middleware`/
-        # `Status::ServerMiddleware` (and any custom server middleware)
-        # only ever see a job by wrapping this `chain.invoke` call, so a
-        # resolution failure caught out here, before `invoke` ever runs,
-        # would never reach them — this job would fail/retry/die with no
-        # History entry and no status update at all, silently. Catching it
-        # here and re-raising it as the chain's own "final block" gives
-        # every middleware the same crack at observing this failure as any
-        # perform-time one, with `worker` as `nil` in that case (every
-        # built-in server middleware ignores the `worker` arg entirely
-        # already; a custom one that needs a real instance simply can't act
-        # on this failure either way — there's no worker to act on).
-        resolution_error = nil
-        worker = begin
-          build_worker(job)
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          resolution_error = e
-          nil
-        end
+        error = run_job(work, job)
+        @ran = error ? [:interrupt, job, error] : :acknowledge
+        error ? finish_failure(job, error) : finish_success(job)
+      ensure
+        best_effort('deregister') { deregister_from_workers }
+      end
+    end
 
-        Cogworker.config.server_chain.invoke(worker, job, work.queue) do
-          raise resolution_error if resolution_error
+    # The job's own outcome: nil, or what it (or its middleware) raised.
+    def run_job(work, job)
+      # `build_worker` itself can raise (most commonly `NameError`, e.g. a
+      # class the caller registered/enqueued but never actually defined —
+      # deliberately still routed through the *same* middleware chain
+      # below rather than caught here directly: `History::Middleware`/
+      # `Status::ServerMiddleware` (and any custom server middleware)
+      # only ever see a job by wrapping this `chain.invoke` call, so a
+      # resolution failure caught out here, before `invoke` ever runs,
+      # would never reach them — this job would fail/retry/die with no
+      # History entry and no status update at all, silently. Catching it
+      # here and re-raising it as the chain's own "final block" gives
+      # every middleware the same crack at observing this failure as any
+      # perform-time one, with `worker` as `nil` in that case (every
+      # built-in server middleware ignores the `worker` arg entirely
+      # already; a custom one that needs a real instance simply can't act
+      # on this failure either way — there's no worker to act on).
+      resolution_error = nil
+      worker = begin
+        build_worker(job)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        resolution_error = e
+        nil
+      end
 
-          worker.perform(*job['args'])
-        end
+      Cogworker.config.server_chain.invoke(worker, job, work.queue) do
+        raise resolution_error if resolution_error
+
+        worker.perform(*job['args'])
+      end
+      nil
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      e
+    end
+
+    def finish_success(job)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
         Throughput.record('processed')
-        Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
-      rescue Exception => e # rubocop:disable Lint/RescueException
+      end
+      Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
+      :acknowledge
+    end
+
+    def finish_failure(job, error)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
         Throughput.record('failed')
-        route_failure(job, e)
-        Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
       end
-    ensure
-      deregister_from_workers
+      Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}" }
+      route_failure(job, error) ? :acknowledge : [:interrupt, job, error]
+    end
+
+    def best_effort(what)
+      yield
+    rescue StandardError => e
+      Cogworker.logger.error { "#{what} failed (ignored): #{e.class}: #{e.message}" }
     end
 
     # A payload that isn't a JSON object goes straight to `dead`
@@ -199,7 +323,7 @@ module Cogworker
     # Built on first use, not in the constructor (see Manager#fetch_class),
     # and rebuilt after a fall-back to BasicFetch.
     def fetcher
-      @fetcher ||= @manager.fetch_class.new(@manager.queues)
+      @fetcher ||= @manager.fetch_class.new(@manager.queues, stopping: -> { @manager.stopping? })
     end
 
     def build_worker(job)
@@ -209,24 +333,35 @@ module Cogworker
       worker
     end
 
+    # True once the failure is recorded in retry/dead — the one write that
+    # must happen; everything after it is best-effort. False if that write
+    # failed (`finish_failure` then interrupts the job instead).
     def route_failure(job, error)
       job['error_class'] = error.class.name
-      job['error_message'] = error.message.to_s[0, 10_000]
+      job['error_message'] = JobUtil.error_message(error)
       job['failed_at'] ||= Time.now.to_f
 
       max_retries = JobUtil.max_retries(job)
       new_count = job['retry_count'].to_i + 1
       job['retry_count'] = new_count
+      retrying = new_count <= max_retries
+      delay = retry_delay(new_count) if retrying
+
+      begin
+        payload = JSON.generate(job)
+        Cogworker.config.redis do |c|
+          retrying ? c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, payload) : c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "couldn't record failure of jid=#{job['jid']}: #{e.class}: #{e.message}" }
+        return false
+      end
 
-      if new_count <= max_retries
-        delay = retry_delay(new_count)
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
-        extend_locks_for_retry(job, delay)
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
-      else
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'dead')
+      best_effort('lock extension') { extend_locks_for_retry(job, delay) } if retrying
+      best_effort('attempts log') do
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: retrying ? 'retrying' : 'dead')
       end
+      true
     end
 
     # A job waiting out its retry backoff still holds its `until_executed`

data/lib/cogworker/redis_keys.rb

@@ -17,6 +17,15 @@ module Cogworker
     # No TTL, unlike `process(identity)`: it's what tells "dead for a
     # while" apart from "missed a few beats" (Config#orphan_threshold).
     LAST_BEAT = 'cogworker:last_beat'
+    # Jobs recovered from dead processes more than `config.max_orphanings` times,
+    # parked by REQUEUE_SCRIPT for Ruby to file in `dead` (see
+    # ReliableFetch.bury_repeat_orphans).
+    REPEAT_ORPHANS = 'cogworker:repeat_orphans'
+    ORPHANINGS_PREFIX = 'cogworker:orphanings:'
+    # Jobs that ran and failed but couldn't be filed in retry/dead, handed
+    # over by a process shutting down (JSON {set, score, payload} each) —
+    # see ReliableFetch.park_unsettled / file_unsettled.
+    UNSETTLED = 'cogworker:unsettled'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'