cogworker 0.3.0 → 0.4.1

data/lib/cogworker/periodic/ticker.rb

@@ -17,12 +17,33 @@ module Cogworker
 
       CLAIM_SCRIPT = File.read(File.join(__dir__, 'claim.lua'))
 
-      def initialize(manager, entries, catch_up: true)
+      # Undoes a won claim whose push then failed, so the slot isn't lost:
+      # puts `last_slot` back (only if it still is this slot — a later slot
+      # may have been claimed since) and drops the per-slot NX lock.
+      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
+      # previous last_slot ("" if none: then slot - 1, which still lets this
+      # slot fire, unlike deleting the key — with catch_up off, a missing
+      # last_slot would be re-primed to this very slot instead).
+      ROLLBACK_SCRIPT = <<~LUA
+        if redis.call("GET", KEYS[1]) == ARGV[1] then
+          if ARGV[2] == "" then
+            redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
+          else
+            redis.call("SET", KEYS[1], ARGV[2])
+          end
+        end
+        redis.call("DEL", KEYS[2])
+        return 1
+      LUA
+
+      def initialize(manager, entries, catch_up: true, ready: -> { true })
         @manager = manager
         @entries = entries
         @catch_up = catch_up
+        @ready = ready
         @last_checked_slot = {}
         @cron_cache = {}
+        @pending_rollbacks = []
       end
 
       def start!
@@ -57,12 +78,13 @@ module Cogworker
 
       # A failed tick is logged and retried next interval rather than ending
       # the thread — a single Redis blip used to silently stop every
-      # periodic entry in this process for good. A slot whose claim raised
-      # isn't recorded in `@last_checked_slot`, so the next tick retries it.
+      # periodic entry in this process for good. A slot whose claim or push
+      # raised isn't recorded in `@last_checked_slot` (and a failed push
+      # rolls its claim back — see #enqueue), so the next tick retries it.
       def run
         until @manager.stopping?
           begin
-            tick unless @manager.quiet?
+            tick unless @manager.quiet? || !@ready.call
           rescue StandardError => e
             Cogworker.logger.error { "Periodic tick failed: #{e.class}: #{e.message}" }
           end
@@ -71,19 +93,21 @@ module Cogworker
       end
 
       def tick
+        retry_pending_rollbacks
         now = Time.now
         @entries.each do |entry|
           slot = cron_for(entry).previous_time(now).to_i
           next if @last_checked_slot[entry.pjid] == slot
 
           jid = SecureRandom.hex(12)
-          enqueue(entry, slot, jid) if !disabled?(entry) && claim?(entry, slot, jid)
+          previous = claim(entry, slot, jid) unless disabled?(entry)
+          enqueue(entry, slot, jid, previous) if previous
           @last_checked_slot[entry.pjid] = slot
         end
       end
 
       # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
-      # step entirely, short-circuiting before `claim?` even runs, so
+      # step entirely, short-circuiting before `claim` even runs, so
       # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
       # while disabled: the Web UI's own "LastRun" column keeps showing the
       # last time it *actually* ran, not a due-but-skipped slot, and
@@ -93,6 +117,30 @@ module Cogworker
       # persisted last_slot) still advances either way, exactly as it
       # already did before this entry ever had a disabled state — it only
       # stops this same tick loop from re-evaluating the same slot twice.
+      # The usual reason a push fails — Redis unreachable — makes this fail
+      # too; it's then kept and retried at the start of every tick until it
+      # goes through (before that tick looks at the slot again), rather than
+      # just logged and the slot lost. (A push that timed out *after* Redis
+      # had applied it is the one case this turns into a second run of the
+      # slot: indistinguishable from one that never arrived.)
+      def rollback_claim(entry, slot, previous_slot)
+        Cogworker.config.redis do |c|
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
+          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+        end
+        true
+      rescue StandardError => e
+        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
+        @pending_rollbacks << [entry, slot, previous_slot]
+        false
+      end
+
+      def retry_pending_rollbacks
+        pending = @pending_rollbacks
+        @pending_rollbacks = []
+        pending.each { |args| rollback_claim(*args) }
+      end
+
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
       end
@@ -101,16 +149,18 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      def claim?(entry, slot, jid)
-        return false if !@catch_up && priming_first_slot?(entry, slot)
+      # nil if this process didn't win the slot; otherwise the previous
+      # `last_slot` value ("" if none), for #rollback_claim.
+      def claim(entry, slot, jid)
+        return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
-          c.eval(CLAIM_SCRIPT,
+          LuaScript.run(c, CLAIM_SCRIPT,
                  keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
                         RedisKeys.periodic_lock(entry.pjid, slot)],
                  argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
-        result == 1
+        result.is_a?(Array) ? result[1].to_s : nil
       end
 
       # With catch-up disabled, an entry's very first tick — ever, across
@@ -140,15 +190,19 @@ module Cogworker
       # Only undone here if the push didn't actually produce a job (raised,
       # or a client middleware swallowed it), so a job that never existed
       # can't hold the entry for the whole `RunningLock.queued_ttl`.
-      def enqueue(entry, slot, jid)
+      def enqueue(entry, slot, jid, previous_slot)
         job = {
           'class' => entry.class_name, 'args' => entry.args, 'retry' => entry.retry,
           'periodic_pjid' => entry.pjid, 'periodic_slot' => slot, 'jid' => jid
         }
+        # Lets a requeue after a crash/shutdown re-take this run's lock
+        # (Periodic::RunningLock::RETAKE_FUNCTION, in both fetches' requeue scripts).
+        job['periodic_until_executed'] = true if entry.until_executed?
         pushed = begin
           Client.push(job)
         rescue StandardError
           RunningLock.release(entry.pjid, jid) if entry.until_executed?
+          rollback_claim(entry, slot, previous_slot)
           raise
         end
         return unless entry.until_executed?

data/lib/cogworker/processor.rb

@@ -6,9 +6,26 @@ require 'securerandom'
 module Cogworker
   # One thread of a Manager's pool: fetch -> register in-flight -> run the
   # server middleware chain around perform -> stats/retry -> deregister.
+  #
+  # What happens to the fetched job afterwards depends on how far it got —
+  # the one rule being that a job is never re-run because of a failure that
+  # happened *after* it ran:
+  # - failed before `perform` (registering it in `cogworker:workers`):
+  #   `give_back` — straight back on its queue, it hasn't run yet;
+  # - ran (succeeded, or failed and was recorded in retry/dead): acknowledge.
+  #   Everything after `perform` other than that one retry/dead write —
+  #   stats, throughput, deregistering, lock extension, the attempts log —
+  #   is best-effort: logged, never raised;
+  # - ran and failed, but the retry/dead write itself failed: `interrupt` —
+  #   into `retry` after INTERRUPT_DELAY, counting `interrupted_count`, and
+  #   into `dead` once that passes MAX_INTERRUPTS. Never straight back on
+  #   the queue: a write that fails deterministically used to make that
+  #   loop forever, re-running the job every second.
   class Processor
     ERROR_BACKOFF = 1 # seconds
     ACK_ATTEMPTS = 3
+    MAX_INTERRUPTS = 3
+    INTERRUPT_DELAY = 30 # seconds, times the interruption count
 
     attr_reader :thread, :tid
 
@@ -16,7 +33,6 @@ module Cogworker
       @manager = manager
       @busy = false
       @tid = SecureRandom.hex(6)
-      @fetcher = manager.fetch_class.new(manager.queues)
     end
 
     def start!
@@ -54,23 +70,75 @@ module Cogworker
         return
       end
 
-      work = @fetcher.retrieve_work
+      work = begin
+        fetcher.retrieve_work
+      rescue ReliableFetch::UnsupportedError => e
+        @manager.fall_back_to_basic_fetch!(e)
+        @fetcher = nil
+        return
+      end
       return unless work
 
       # Fetched after shutdown began (e.g. still blocked in BRPOP when
       # `Manager#stop!` requeued unfinished work, which it then popped right
       # back): hand it back rather than start it, since the process is about
       # to exit underneath it.
-      return @fetcher.give_back(work) if @manager.stopping?
+      return fetcher.give_back(work) if @manager.stopping?
 
       @manager.processor_busy!
       @busy = true
+      disposition = :give_back
       begin
-        execute(work)
+        disposition = execute(work)
       ensure
         @busy = false
         @manager.processor_idle!
-        acknowledge(work)
+        settle(work, disposition)
+      end
+    end
+
+    def settle(work, disposition)
+      case disposition
+      when :acknowledge then acknowledge(work)
+      when :give_back then give_back(work)
+      else interrupt(work, *disposition.drop(1))
+      end
+    end
+
+    # Only for a job that hasn't run yet (see the class comment). If even
+    # this fails, it stays on the in-progress list, which `Manager#stop!` /
+    # orphan recovery / `ReliableFetch.reconcile` requeue later.
+    def give_back(work)
+      fetcher.give_back(work)
+    rescue StandardError => e
+      Cogworker.logger.error do
+        "couldn't hand back jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
+      end
+    end
+
+    # The job ran and failed, but recording that in retry/dead failed. Built
+    # from the original payload (always serializable — it came from JSON)
+    # plus the failure fields, never from the in-memory job hash: whatever
+    # made that one fail to serialize must not do it again here.
+    def interrupt(work, job, error)
+      payload = JSON.parse(work.raw_job)
+      count = payload['interrupted_count'].to_i + 1
+      payload.merge!('interrupted_count' => count, 'retry_count' => job['retry_count'],
+                     'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error),
+                     'failed_at' => Time.now.to_f)
+      set, score = if count > MAX_INTERRUPTS
+                     [RedisKeys::DEAD, Time.now.to_f]
+                   else
+                     [RedisKeys::RETRY, Time.now.to_f + (INTERRUPT_DELAY * count)]
+                   end
+      fetcher.interrupt(work, set, score, JSON.generate(payload))
+      Cogworker.logger.error do
+        "couldn't record the failure of jid=#{payload['jid']} (#{error.class}); " \
+          "moved to #{set.delete_prefix('cogworker:')} (interruption #{count}/#{MAX_INTERRUPTS})"
+      end
+    rescue StandardError => e
+      Cogworker.logger.error do
+        "couldn't record or interrupt jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
       end
     end
 
@@ -82,7 +150,7 @@ module Cogworker
       attempts = 0
       begin
         attempts += 1
-        @fetcher.acknowledge(work)
+        fetcher.acknowledge(work)
       rescue StandardError => e
         if attempts < ACK_ATTEMPTS
           sleep(0.1 * attempts)
@@ -101,53 +169,80 @@ module Cogworker
       '?'
     end
 
+    # Returns how `settle` should dispose of the fetched job; raises only
+    # before the job has run (then `process_one`'s default, :give_back,
+    # applies).
     def execute(work)
       job = parse_job(work)
-      return unless job
+      return :acknowledge unless job
 
       register_in_workers(work.queue, job)
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
-
       begin
-        # `build_worker` itself can raise (most commonly `NameError`, e.g. a
-        # class the caller registered/enqueued but never actually defined —
-        # deliberately still routed through the *same* middleware chain
-        # below rather than caught here directly: `History::Middleware`/
-        # `Status::ServerMiddleware` (and any custom server middleware)
-        # only ever see a job by wrapping this `chain.invoke` call, so a
-        # resolution failure caught out here, before `invoke` ever runs,
-        # would never reach them — this job would fail/retry/die with no
-        # History entry and no status update at all, silently. Catching it
-        # here and re-raising it as the chain's own "final block" gives
-        # every middleware the same crack at observing this failure as any
-        # perform-time one, with `worker` as `nil` in that case (every
-        # built-in server middleware ignores the `worker` arg entirely
-        # already; a custom one that needs a real instance simply can't act
-        # on this failure either way — there's no worker to act on).
-        resolution_error = nil
-        worker = begin
-          build_worker(job)
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          resolution_error = e
-          nil
-        end
+        error = run_job(work, job)
+        error ? finish_failure(job, error) : finish_success(job)
+      ensure
+        best_effort('deregister') { deregister_from_workers }
+      end
+    end
+
+    # The job's own outcome: nil, or what it (or its middleware) raised.
+    def run_job(work, job)
+      # `build_worker` itself can raise (most commonly `NameError`, e.g. a
+      # class the caller registered/enqueued but never actually defined —
+      # deliberately still routed through the *same* middleware chain
+      # below rather than caught here directly: `History::Middleware`/
+      # `Status::ServerMiddleware` (and any custom server middleware)
+      # only ever see a job by wrapping this `chain.invoke` call, so a
+      # resolution failure caught out here, before `invoke` ever runs,
+      # would never reach them — this job would fail/retry/die with no
+      # History entry and no status update at all, silently. Catching it
+      # here and re-raising it as the chain's own "final block" gives
+      # every middleware the same crack at observing this failure as any
+      # perform-time one, with `worker` as `nil` in that case (every
+      # built-in server middleware ignores the `worker` arg entirely
+      # already; a custom one that needs a real instance simply can't act
+      # on this failure either way — there's no worker to act on).
+      resolution_error = nil
+      worker = begin
+        build_worker(job)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        resolution_error = e
+        nil
+      end
 
-        Cogworker.config.server_chain.invoke(worker, job, work.queue) do
-          raise resolution_error if resolution_error
+      Cogworker.config.server_chain.invoke(worker, job, work.queue) do
+        raise resolution_error if resolution_error
 
-          worker.perform(*job['args'])
-        end
+        worker.perform(*job['args'])
+      end
+      nil
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      e
+    end
+
+    def finish_success(job)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
         Throughput.record('processed')
-        Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
-      rescue Exception => e # rubocop:disable Lint/RescueException
+      end
+      Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
+      :acknowledge
+    end
+
+    def finish_failure(job, error)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
         Throughput.record('failed')
-        route_failure(job, e)
-        Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
       end
-    ensure
-      deregister_from_workers
+      Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}" }
+      route_failure(job, error) ? :acknowledge : [:interrupt, job, error]
+    end
+
+    def best_effort(what)
+      yield
+    rescue StandardError => e
+      Cogworker.logger.error { "#{what} failed (ignored): #{e.class}: #{e.message}" }
     end
 
     # A payload that isn't a JSON object goes straight to `dead`
@@ -174,6 +269,12 @@ module Cogworker
       Cogworker.logger.error { "unparseable job on queue #{work.queue} moved to dead: #{error.message}" }
     end
 
+    # Built on first use, not in the constructor (see Manager#fetch_class),
+    # and rebuilt after a fall-back to BasicFetch.
+    def fetcher
+      @fetcher ||= @manager.fetch_class.new(@manager.queues, stopping: -> { @manager.stopping? })
+    end
+
     def build_worker(job)
       klass = Object.const_get(job['class'])
       worker = klass.new
@@ -181,24 +282,35 @@ module Cogworker
       worker
     end
 
+    # True once the failure is recorded in retry/dead — the one write that
+    # must happen; everything after it is best-effort. False if that write
+    # failed (`finish_failure` then interrupts the job instead).
     def route_failure(job, error)
       job['error_class'] = error.class.name
-      job['error_message'] = error.message.to_s[0, 10_000]
+      job['error_message'] = JobUtil.error_message(error)
       job['failed_at'] ||= Time.now.to_f
 
       max_retries = JobUtil.max_retries(job)
       new_count = job['retry_count'].to_i + 1
       job['retry_count'] = new_count
+      retrying = new_count <= max_retries
+      delay = retry_delay(new_count) if retrying
+
+      begin
+        payload = JSON.generate(job)
+        Cogworker.config.redis do |c|
+          retrying ? c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, payload) : c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "couldn't record failure of jid=#{job['jid']}: #{e.class}: #{e.message}" }
+        return false
+      end
 
-      if new_count <= max_retries
-        delay = retry_delay(new_count)
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
-        extend_locks_for_retry(job, delay)
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
-      else
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'dead')
+      best_effort('lock extension') { extend_locks_for_retry(job, delay) } if retrying
+      best_effort('attempts log') do
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: retrying ? 'retrying' : 'dead')
       end
+      true
     end
 
     # A job waiting out its retry backoff still holds its `until_executed`

data/lib/cogworker/redis_keys.rb

@@ -10,6 +10,13 @@ module Cogworker
   module RedisKeys
     QUEUE_PREFIX = 'cogworker:queue:'
     IN_PROGRESS_PREFIX = 'cogworker:inprogress:'
+    # Every identity that has (or may have) an in-progress list — what
+    # orphan recovery walks instead of SCANning the whole keyspace.
+    IN_PROGRESS_IDENTITIES = 'cogworker:inprogress_identities'
+    # identity => Redis server time (epoch seconds) of its last heartbeat.
+    # No TTL, unlike `process(identity)`: it's what tells "dead for a
+    # while" apart from "missed a few beats" (Config#orphan_threshold).
+    LAST_BEAT = 'cogworker:last_beat'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'

data/lib/cogworker/reliable_fetch/unsupported_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Cogworker
+  class ReliableFetch
+    # Raised by ReliableFetch#retrieve_work when the server turns out not to
+    # know LMOVE (Redis < 6.2) even though the version check at boot let
+    # `:reliable` through (e.g. Redis wasn't reachable then). The Processor
+    # reacts by switching the whole Manager over to BasicFetch.
+    class UnsupportedError < StandardError; end
+  end
+end