cogworker 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfb90fe3a477f31658c58fd09a41e4404a0c09999b74614789bd343ee704fad2
-  data.tar.gz: 407aa20293538fa35c8219f88f21fd8b1c15150d96e77d2b495d017ef21f924e
+  metadata.gz: 680a06059bb5853e9d7843ff6d5313d7cd9431b222b846b496c9ca965d6b8844
+  data.tar.gz: fd688cdc23fa56e86e6d4c50a7f319e98698d4c6166d8213359c38eb0f94aa71
 SHA512:
-  metadata.gz: c7df3eba977499a45acca66e8d25fd87cfe4669b9098c1e99be798f587de2ddf76edc75adb01953e8fe311ac324d7e4eb44cce66a33be264efe4f266e67d8f3e
-  data.tar.gz: 7fa4d7ba18f7be8ae8e80bd7b260be8bd8e9387122a623802812abd166652859f48669bbb42c16f72f15df1104d855934f4cfdb6cf6aede2737d740540d27278
+  metadata.gz: fc5d3f1a358e0c826b2801ddd73f2b039b31b1e29616a38947da6df0cd85edfdf3b9b8c877e7123fc6283617f7f34317b9503c76db4eeb89a881677e335c5cfb
+  data.tar.gz: 486aff800ab67a15ce9c815f659a70ee95abcfd28218e7c862329a26a0f3643dce3a3c24dc79c539531a9b29b50f745e81482e617ba908c7d80ea131fe7b6d60

data/lib/cogworker/attempts.rb

@@ -24,7 +24,7 @@ module Cogworker
     def record(jid, attempt:, error:, outcome:)
       entry = JSON.generate(
         'attempt' => attempt, 'failed_at' => Time.now.to_f, 'outcome' => outcome,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000]
+        'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error)
       )
       key = RedisKeys.job_attempts(jid)
       Cogworker.config.redis do |c|

data/lib/cogworker/basic_fetch.rb

@@ -17,8 +17,12 @@ module Cogworker
   class BasicFetch
     TIMEOUT = 2 # seconds; also how often a stopped/quieted processor notices and exits its fetch loop.
 
-    def initialize(queues)
+    # `stopping:` lets a fetcher cut its own waits short once shutdown has
+    # begun (Processor passes the Manager's `stopping?`), so a long idle
+    # pause can't hold up `Manager#stop!`.
+    def initialize(queues, stopping: -> { false })
       @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
+      @stopping = stopping
     end
 
     # Re-reads `cogworker:paused_queues` on every single fetch (not once at
@@ -33,7 +37,7 @@ module Cogworker
     def retrieve_work
       keys = active_keys
       if keys.empty?
-        sleep(TIMEOUT)
+        interruptible_sleep(TIMEOUT)
         return nil
       end
 
@@ -47,6 +51,12 @@ module Cogworker
     # Nothing to do: BRPOP already removed the job from Redis.
     def acknowledge(_work); end
 
+    # `Processor#interrupt`: the job ran, recording its failure failed —
+    # file it under `set` (retry/dead) at `score` instead.
+    def interrupt(_work, set, score, payload)
+      Cogworker.config.redis { |c| c.zadd(set, score, payload) }
+    end
+
     # Returns a fetched-but-unstarted job to the end of its queue that's
     # popped next.
     def give_back(work)
@@ -75,14 +85,24 @@ module Cogworker
       queue_key = RedisKeys.queue(queue)
       retake = job['periodic_until_executed'] && job['periodic_pjid'] && job['jid']
       lock_key = retake ? RedisKeys.periodic_running(job['periodic_pjid']) : queue_key
-      conn.eval(REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
-                                      argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
+      LuaScript.run(conn, REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
+                                               argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
                                              Periodic::RunningLock.queued_ttl])
     end
     private_class_method :requeue_entry
 
     private
 
+    def interruptible_sleep(seconds)
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + seconds
+      until @stopping.call
+        left = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        break if left <= 0
+
+        sleep([left, 0.25].min)
+      end
+    end
+
     def active_keys
       paused = Cogworker.config.redis { |c| c.smembers(RedisKeys::PAUSED_QUEUES) }
       return @queue_keys if paused.empty?

data/lib/cogworker/heartbeat.rb

@@ -127,12 +127,22 @@ module Cogworker
         c.expire(RedisKeys.workers(identity), TTL)
         if reliable
           c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
-          c.hset(RedisKeys::LAST_BEAT, identity, c.time.first) # Redis's clock, not this host's
+          c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c))
         end
         touch_running_periodic_locks(c, identity)
       end
     end
 
+    # Redis's clock, not this host's (so host clock skew doesn't matter to
+    # orphan detection) — falling back to this host's where `TIME` isn't
+    # allowed (some proxies/managed services): better a skewed stamp than a
+    # process that can never complete its first beat.
+    def redis_now(conn)
+      conn.time.first
+    rescue Redis::CommandError
+      Time.now.to_i
+    end
+
     # Keeps every in-flight `until_executed` periodic run's
     # `periodic:running:<pjid>` lock alive for as long as this process is —
     # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`

data/lib/cogworker/history/storage.rb

@@ -30,8 +30,8 @@ module Cogworker
         }
         if error
           entry['error_class'] = error.class.name
-          entry['error_message'] = error.message.to_s[0, 10_000]
-          entry['backtrace'] = (error.backtrace || []).first(200)
+          entry['error_message'] = JobUtil.error_message(error)
+          entry['backtrace'] = (error.backtrace || []).first(200).map { |line| JobUtil.safe_string(line) }
         end
 
         raw = JSON.generate(entry)

data/lib/cogworker/job_util.rb

@@ -96,12 +96,34 @@ module Cogworker
       return :invalid unless job
 
       job['enqueued_at'] = Time.now.to_f
-      won = conn.eval(CLAIM_AND_REQUEUE_SCRIPT,
-                      keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
-                      argv: [raw, job['queue'], JSON.generate(job)])
+      won = LuaScript.run(conn, CLAIM_AND_REQUEUE_SCRIPT,
+                          keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
+                          argv: [raw, job['queue'], JSON.generate(job)])
       won == 1 ? :requeued : :gone
     end
 
+    ERROR_MESSAGE_LIMIT = 10_000
+
+    # Valid UTF-8, whatever came in — every string this gem stores from
+    # outside its control (an exception's message, a raw payload) goes
+    # through here before `JSON.generate`, which raises on invalid UTF-8.
+    # An unchecked `raise "bad \xff".b` used to make every attempt to record
+    # the failure itself raise, deterministically. Strings in another
+    # (valid) encoding are transcoded; binary/broken ones are scrubbed.
+    def safe_string(value, limit = nil)
+      str = value.to_s
+      str = if [Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII].include?(str.encoding)
+              str.dup.force_encoding(Encoding::UTF_8).scrub
+            else
+              str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+            end
+      limit ? str[0, limit] : str
+    end
+
+    def error_message(error)
+      safe_string(error.message, ERROR_MESSAGE_LIMIT)
+    end
+
     UNPARSEABLE_CLASS = '(unparseable)'
 
     # A payload that isn't a JSON object can't be run, retried or even
@@ -113,8 +135,8 @@ module Cogworker
       {
         'class' => UNPARSEABLE_CLASS, 'args' => [], 'queue' => queue, 'jid' => SecureRandom.hex(12),
         'retry' => false, 'retry_count' => 0, 'enqueued_at' => now, 'failed_at' => now,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000],
-        'raw_payload' => raw.to_s[0, 100_000]
+        'error_class' => error.class.name, 'error_message' => error_message(error),
+        'raw_payload' => safe_string(raw, 100_000)
       }
     end
   end

data/lib/cogworker/launcher.rb

@@ -13,8 +13,14 @@ module Cogworker
       @manager = Manager.new(config)
       @scheduled = Scheduled.new(@manager)
       @heartbeat = Heartbeat.new(@manager)
+      # The first tick waits for the first orphan check: after the whole
+      # fleet was down longer than a running `until_executed` job's lock
+      # lives, the ticker would otherwise claim that entry's next slot before
+      # recovery had requeued the interrupted run (and re-taken its lock) —
+      # two runs at once.
       @ticker = Periodic::Ticker.new(
-        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up
+        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
+                                                   ready: -> { @scheduled.recovered_once? }
       )
       @signal_queue = ::Queue.new
     end

data/lib/cogworker/lua_script.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+
+module Cogworker
+  # Runs a Lua script by its SHA1 (EVALSHA) instead of shipping the whole
+  # source on every call, falling back to EVAL — which also loads it into
+  # the server's script cache for next time — on NOSCRIPT (first use, a
+  # restarted or failed-over server, SCRIPT FLUSH). Every script in this
+  # gem goes through here.
+  module LuaScript
+    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
+    SHAS_MUTEX = Mutex.new
+
+    module_function
+
+    def run(conn, source, keys: [], argv: [])
+      begin
+        return conn.evalsha(sha(source), keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+      end
+      conn.eval(source, keys: keys, argv: argv)
+    end
+
+    def sha(source)
+      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+    end
+  end
+end

data/lib/cogworker/manager.rb

@@ -18,11 +18,11 @@ module Cogworker
       @config.queues
     end
 
-    # Resolved on first use — from a processor thread's first fetch, after
-    # the Heartbeat's first beat proved Redis reachable — not in the
-    # constructor: that needed Redis just to build a Manager, and when
-    # Redis was unreachable at that moment, the version check couldn't run
-    # at all (see ReliableFetch.supported?).
+    # Resolved on first use, not in the constructor (that needed Redis
+    # just to build a Manager). First use is normally the Heartbeat's first
+    # beat; if Redis can't be reached then, ReliableFetch.supported? raises
+    # and nothing is memoized — the beat fails and is retried, and this
+    # resolves again next time — rather than memoizing a guess.
     def fetch_class
       @fetch_mutex.synchronize { @fetch_class ||= resolve_fetch_class }
     end

data/lib/cogworker/owned_key.rb

@@ -32,9 +32,9 @@ module Cogworker
     end
 
     def run(conn, script, key, argv)
-      return conn.eval(script, keys: [key], argv: argv) if conn
+      return LuaScript.run(conn, script, keys: [key], argv: argv) if conn
 
-      Cogworker.config.redis { |c| c.eval(script, keys: [key], argv: argv) }
+      Cogworker.config.redis { |c| LuaScript.run(c, script, keys: [key], argv: argv) }
     end
   end
 end

data/lib/cogworker/periodic/ticker.rb

@@ -36,12 +36,14 @@ module Cogworker
         return 1
       LUA
 
-      def initialize(manager, entries, catch_up: true)
+      def initialize(manager, entries, catch_up: true, ready: -> { true })
         @manager = manager
         @entries = entries
         @catch_up = catch_up
+        @ready = ready
         @last_checked_slot = {}
         @cron_cache = {}
+        @pending_rollbacks = []
       end
 
       def start!
@@ -82,7 +84,7 @@ module Cogworker
       def run
         until @manager.stopping?
           begin
-            tick unless @manager.quiet?
+            tick unless @manager.quiet? || !@ready.call
           rescue StandardError => e
             Cogworker.logger.error { "Periodic tick failed: #{e.class}: #{e.message}" }
           end
@@ -91,6 +93,7 @@ module Cogworker
       end
 
       def tick
+        retry_pending_rollbacks
         now = Time.now
         @entries.each do |entry|
           slot = cron_for(entry).previous_time(now).to_i
@@ -114,13 +117,28 @@ module Cogworker
       # persisted last_slot) still advances either way, exactly as it
       # already did before this entry ever had a disabled state — it only
       # stops this same tick loop from re-evaluating the same slot twice.
+      # The usual reason a push fails — Redis unreachable — makes this fail
+      # too; it's then kept and retried at the start of every tick until it
+      # goes through (before that tick looks at the slot again), rather than
+      # just logged and the slot lost. (A push that timed out *after* Redis
+      # had applied it is the one case this turns into a second run of the
+      # slot: indistinguishable from one that never arrived.)
       def rollback_claim(entry, slot, previous_slot)
         Cogworker.config.redis do |c|
           keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
-          c.eval(ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
         end
+        true
       rescue StandardError => e
-        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot}: #{e.class}: #{e.message}" }
+        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
+        @pending_rollbacks << [entry, slot, previous_slot]
+        false
+      end
+
+      def retry_pending_rollbacks
+        pending = @pending_rollbacks
+        @pending_rollbacks = []
+        pending.each { |args| rollback_claim(*args) }
       end
 
       def disabled?(entry)
@@ -137,7 +155,7 @@ module Cogworker
         return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
-          c.eval(CLAIM_SCRIPT,
+          LuaScript.run(c, CLAIM_SCRIPT,
                  keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
                         RedisKeys.periodic_lock(entry.pjid, slot)],
                  argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])

data/lib/cogworker/processor.rb

@@ -6,9 +6,26 @@ require 'securerandom'
 module Cogworker
   # One thread of a Manager's pool: fetch -> register in-flight -> run the
   # server middleware chain around perform -> stats/retry -> deregister.
+  #
+  # What happens to the fetched job afterwards depends on how far it got —
+  # the one rule being that a job is never re-run because of a failure that
+  # happened *after* it ran:
+  # - failed before `perform` (registering it in `cogworker:workers`):
+  #   `give_back` — straight back on its queue, it hasn't run yet;
+  # - ran (succeeded, or failed and was recorded in retry/dead): acknowledge.
+  #   Everything after `perform` other than that one retry/dead write —
+  #   stats, throughput, deregistering, lock extension, the attempts log —
+  #   is best-effort: logged, never raised;
+  # - ran and failed, but the retry/dead write itself failed: `interrupt` —
+  #   into `retry` after INTERRUPT_DELAY, counting `interrupted_count`, and
+  #   into `dead` once that passes MAX_INTERRUPTS. Never straight back on
+  #   the queue: a write that fails deterministically used to make that
+  #   loop forever, re-running the job every second.
   class Processor
     ERROR_BACKOFF = 1 # seconds
     ACK_ATTEMPTS = 3
+    MAX_INTERRUPTS = 3
+    INTERRUPT_DELAY = 30 # seconds, times the interruption count
 
     attr_reader :thread, :tid
 
@@ -70,29 +87,58 @@ module Cogworker
 
       @manager.processor_busy!
       @busy = true
-      finished = false
+      disposition = :give_back
       begin
-        execute(work)
-        finished = true
+        disposition = execute(work)
       ensure
         @busy = false
         @manager.processor_idle!
-        finished ? acknowledge(work) : give_back(work)
+        settle(work, disposition)
       end
     end
 
-    # `execute` raising means bookkeeping failed (Redis, mid-way through
-    # registering the job or routing its failure to retry/dead) — not the
-    # job itself, whose own errors `execute` always handles. Acknowledging
-    # here would drop a job that never made it to retry/dead; instead it
-    # goes straight back on its queue (it may run again: at-least-once). If
-    # even that fails, it simply stays on the in-progress list, which
-    # `Manager#stop!`/orphan recovery requeue later.
+    def settle(work, disposition)
+      case disposition
+      when :acknowledge then acknowledge(work)
+      when :give_back then give_back(work)
+      else interrupt(work, *disposition.drop(1))
+      end
+    end
+
+    # Only for a job that hasn't run yet (see the class comment). If even
+    # this fails, it stays on the in-progress list, which `Manager#stop!` /
+    # orphan recovery / `ReliableFetch.reconcile` requeue later.
     def give_back(work)
       fetcher.give_back(work)
     rescue StandardError => e
       Cogworker.logger.error do
-        "couldn't hand back jid=#{jid_of(work)} after a failed run (left in progress): #{e.class}: #{e.message}"
+        "couldn't hand back jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
+      end
+    end
+
+    # The job ran and failed, but recording that in retry/dead failed. Built
+    # from the original payload (always serializable — it came from JSON)
+    # plus the failure fields, never from the in-memory job hash: whatever
+    # made that one fail to serialize must not do it again here.
+    def interrupt(work, job, error)
+      payload = JSON.parse(work.raw_job)
+      count = payload['interrupted_count'].to_i + 1
+      payload.merge!('interrupted_count' => count, 'retry_count' => job['retry_count'],
+                     'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error),
+                     'failed_at' => Time.now.to_f)
+      set, score = if count > MAX_INTERRUPTS
+                     [RedisKeys::DEAD, Time.now.to_f]
+                   else
+                     [RedisKeys::RETRY, Time.now.to_f + (INTERRUPT_DELAY * count)]
+                   end
+      fetcher.interrupt(work, set, score, JSON.generate(payload))
+      Cogworker.logger.error do
+        "couldn't record the failure of jid=#{payload['jid']} (#{error.class}); " \
+          "moved to #{set.delete_prefix('cogworker:')} (interruption #{count}/#{MAX_INTERRUPTS})"
+      end
+    rescue StandardError => e
+      Cogworker.logger.error do
+        "couldn't record or interrupt jid=#{jid_of(work)} (left in progress): #{e.class}: #{e.message}"
       end
     end
 
@@ -123,53 +169,80 @@ module Cogworker
       '?'
     end
 
+    # Returns how `settle` should dispose of the fetched job; raises only
+    # before the job has run (then `process_one`'s default, :give_back,
+    # applies).
     def execute(work)
       job = parse_job(work)
-      return unless job
+      return :acknowledge unless job
 
       register_in_workers(work.queue, job)
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
-
       begin
-        # `build_worker` itself can raise (most commonly `NameError`, e.g. a
-        # class the caller registered/enqueued but never actually defined —
-        # deliberately still routed through the *same* middleware chain
-        # below rather than caught here directly: `History::Middleware`/
-        # `Status::ServerMiddleware` (and any custom server middleware)
-        # only ever see a job by wrapping this `chain.invoke` call, so a
-        # resolution failure caught out here, before `invoke` ever runs,
-        # would never reach them — this job would fail/retry/die with no
-        # History entry and no status update at all, silently. Catching it
-        # here and re-raising it as the chain's own "final block" gives
-        # every middleware the same crack at observing this failure as any
-        # perform-time one, with `worker` as `nil` in that case (every
-        # built-in server middleware ignores the `worker` arg entirely
-        # already; a custom one that needs a real instance simply can't act
-        # on this failure either way — there's no worker to act on).
-        resolution_error = nil
-        worker = begin
-          build_worker(job)
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          resolution_error = e
-          nil
-        end
+        error = run_job(work, job)
+        error ? finish_failure(job, error) : finish_success(job)
+      ensure
+        best_effort('deregister') { deregister_from_workers }
+      end
+    end
 
-        Cogworker.config.server_chain.invoke(worker, job, work.queue) do
-          raise resolution_error if resolution_error
+    # The job's own outcome: nil, or what it (or its middleware) raised.
+    def run_job(work, job)
+      # `build_worker` itself can raise (most commonly `NameError`, e.g. a
+      # class the caller registered/enqueued but never actually defined —
+      # deliberately still routed through the *same* middleware chain
+      # below rather than caught here directly: `History::Middleware`/
+      # `Status::ServerMiddleware` (and any custom server middleware)
+      # only ever see a job by wrapping this `chain.invoke` call, so a
+      # resolution failure caught out here, before `invoke` ever runs,
+      # would never reach them — this job would fail/retry/die with no
+      # History entry and no status update at all, silently. Catching it
+      # here and re-raising it as the chain's own "final block" gives
+      # every middleware the same crack at observing this failure as any
+      # perform-time one, with `worker` as `nil` in that case (every
+      # built-in server middleware ignores the `worker` arg entirely
+      # already; a custom one that needs a real instance simply can't act
+      # on this failure either way — there's no worker to act on).
+      resolution_error = nil
+      worker = begin
+        build_worker(job)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        resolution_error = e
+        nil
+      end
 
-          worker.perform(*job['args'])
-        end
+      Cogworker.config.server_chain.invoke(worker, job, work.queue) do
+        raise resolution_error if resolution_error
+
+        worker.perform(*job['args'])
+      end
+      nil
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      e
+    end
+
+    def finish_success(job)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
         Throughput.record('processed')
-        Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
-      rescue Exception => e # rubocop:disable Lint/RescueException
+      end
+      Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
+      :acknowledge
+    end
+
+    def finish_failure(job, error)
+      best_effort('stats') do
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
         Throughput.record('failed')
-        route_failure(job, e)
-        Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
       end
-    ensure
-      deregister_from_workers
+      Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}" }
+      route_failure(job, error) ? :acknowledge : [:interrupt, job, error]
+    end
+
+    def best_effort(what)
+      yield
+    rescue StandardError => e
+      Cogworker.logger.error { "#{what} failed (ignored): #{e.class}: #{e.message}" }
     end
 
     # A payload that isn't a JSON object goes straight to `dead`
@@ -199,7 +272,7 @@ module Cogworker
     # Built on first use, not in the constructor (see Manager#fetch_class),
     # and rebuilt after a fall-back to BasicFetch.
     def fetcher
-      @fetcher ||= @manager.fetch_class.new(@manager.queues)
+      @fetcher ||= @manager.fetch_class.new(@manager.queues, stopping: -> { @manager.stopping? })
     end
 
     def build_worker(job)
@@ -209,24 +282,35 @@ module Cogworker
       worker
     end
 
+    # True once the failure is recorded in retry/dead — the one write that
+    # must happen; everything after it is best-effort. False if that write
+    # failed (`finish_failure` then interrupts the job instead).
     def route_failure(job, error)
       job['error_class'] = error.class.name
-      job['error_message'] = error.message.to_s[0, 10_000]
+      job['error_message'] = JobUtil.error_message(error)
       job['failed_at'] ||= Time.now.to_f
 
       max_retries = JobUtil.max_retries(job)
       new_count = job['retry_count'].to_i + 1
       job['retry_count'] = new_count
+      retrying = new_count <= max_retries
+      delay = retry_delay(new_count) if retrying
+
+      begin
+        payload = JSON.generate(job)
+        Cogworker.config.redis do |c|
+          retrying ? c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, payload) : c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "couldn't record failure of jid=#{job['jid']}: #{e.class}: #{e.message}" }
+        return false
+      end
 
-      if new_count <= max_retries
-        delay = retry_delay(new_count)
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
-        extend_locks_for_retry(job, delay)
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
-      else
-        Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
-        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'dead')
+      best_effort('lock extension') { extend_locks_for_retry(job, delay) } if retrying
+      best_effort('attempts log') do
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: retrying ? 'retrying' : 'dead')
       end
+      true
     end
 
     # A job waiting out its retry backoff still holds its `until_executed`

data/lib/cogworker/reliable_fetch.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'digest/sha1'
+require 'json'
 
 module Cogworker
   # The default fetch (`config.fetch = :reliable`): every job is moved
@@ -41,109 +42,181 @@ module Cogworker
       return false
     LUA
 
-    # The hot path runs this by hash (EVALSHA), not by shipping the whole
-    # script text on every poll; see #run_fetch_script.
-    FETCH_SCRIPT_SHA = Digest::SHA1.hexdigest(FETCH_SCRIPT)
-
-    # KEYS[1] = an in-progress list; ARGV[1] = queue key prefix; ARGV[2] =
-    # queue for a payload whose own `queue` can't be read (the Processor
-    # then buries it as unparseable); ARGV[3] = periodic running-lock key
-    # prefix, ARGV[4] = its queued TTL. Oldest first, each onto the end its
-    # queue is popped from, so recovered jobs run next, in their original
-    # order. Atomic per call: two processes recovering the same list can't
-    # both move one job. A job from an `until_executed` periodic entry
-    # re-takes its running lock on the way (same rule as
-    # Periodic::RunningLock::RETAKE_FUNCTION), so the ticker can't start
-    # the next slot alongside the recovered run.
-    REQUEUE_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
-      local moved = 0
-      while true do
-        local job = redis.call("LPOP", KEYS[1])
-        if not job then
-          return moved
-        end
-        local queue = ARGV[2]
+    # (Run by hash, like every script here — see LuaScript.)
+    FETCH_SCRIPT_SHA = LuaScript.sha(FETCH_SCRIPT)
+
+    # Lua shared by every script here that puts a job back on its queue:
+    # onto the end its queue is popped from (so it runs next), onto
+    # `default_queue` if its own `queue` can't be read (the Processor then
+    # buries it as unparseable) — and a job from an `until_executed`
+    # periodic entry re-takes its running lock on the way
+    # (Periodic::RunningLock::RETAKE_FUNCTION), so the ticker can't start
+    # the next slot alongside the requeued run.
+    REQUEUE_JOB_FUNCTION = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      local function requeue_job(job, queue_prefix, default_queue, lock_prefix, lock_ttl)
+        local queue = default_queue
         local ok, decoded = pcall(cjson.decode, job)
         if ok and type(decoded) == "table" then
-          if type(decoded["queue"]) == "string" then
+          if type(decoded["queue"]) == "string" and decoded["queue"] ~= "" then
             queue = decoded["queue"]
           end
           local pjid, jid = decoded["periodic_pjid"], decoded["jid"]
           if decoded["periodic_until_executed"] == true and type(pjid) == "string" and type(jid) == "string" then
-            retake_running_lock(ARGV[3] .. pjid, jid, ARGV[4])
+            retake_running_lock(lock_prefix .. pjid, jid, lock_ttl)
           end
         end
-        redis.call("RPUSH", ARGV[1] .. queue, job)
+        redis.call("RPUSH", queue_prefix .. queue, job)
+      end
+    LUA
+
+    # KEYS[1] = an in-progress list, KEYS[2] = its owner's presence key,
+    # KEYS[3] = LAST_BEAT; ARGV[1..4] = requeue_job's prefix/default
+    # queue/lock prefix/lock TTL, ARGV[5] = owner identity, ARGV[6] = orphan
+    # threshold, or "" to requeue unconditionally (this process's own list,
+    # on shutdown). Returns how many it moved, or -1 if the owner turned
+    # out to be alive. Oldest first, so recovered jobs keep their order. The
+    # "is the owner really dead?" check (see ReliableFetch.dead?) is made
+    # here, atomically with the requeue: checked separately, an owner that
+    # beat in between would lose jobs it is still running to a second run.
+    # Atomic per call, so two processes recovering one list can't both move
+    # a job.
+    REQUEUE_SCRIPT = REQUEUE_JOB_FUNCTION + <<~LUA
+      if ARGV[6] ~= "" then
+        if redis.call("EXISTS", KEYS[2]) == 1 then
+          return -1
+        end
+        local last = redis.call("HGET", KEYS[3], ARGV[5])
+        if last then
+          if redis.replicate_commands then redis.replicate_commands() end
+          local now = tonumber(redis.call("TIME")[1])
+          if now - tonumber(last) <= tonumber(ARGV[6]) then
+            return -1
+          end
+        end
+      end
+      local moved = 0
+      while true do
+        local job = redis.call("LPOP", KEYS[1])
+        if not job then
+          return moved
+        end
+        requeue_job(job, ARGV[1], ARGV[2], ARGV[3], ARGV[4])
         moved = moved + 1
       end
     LUA
 
-    # KEYS[1] = in-progress list, KEYS[2] = the job's queue; ARGV[1] = job.
-    GIVE_BACK_SCRIPT = <<~LUA
+    # KEYS[1] = an in-progress list; ARGV[1] = one job on it, ARGV[2..5] =
+    # requeue_job's arguments. That one job, if still on the list, back on
+    # its queue: `give_back`, and `reconcile`'s strays.
+    REQUEUE_ONE_SCRIPT = REQUEUE_JOB_FUNCTION + <<~LUA
       if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
-        redis.call("RPUSH", KEYS[2], ARGV[1])
+        requeue_job(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
+        return 1
       end
+      return 0
+    LUA
+
+    # KEYS[1] = in-progress list, KEYS[2] = retry/dead ZSET; ARGV[1] = the
+    # job as fetched, ARGV[2] = score, ARGV[3] = payload to file. Off the
+    # in-progress list and into the ZSET in one step; a no-op if it's no
+    # longer in progress (already requeued by a shutdown).
+    INTERRUPT_SCRIPT = <<~LUA
+      if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
+        redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
+        return 1
+      end
+      return 0
     LUA
 
     class << self
+      # A connection error propagates: can't tell yet, so the caller
+      # (Manager#fetch_class) resolves again on its next use, instead of
+      # memoizing a guess made while Redis was unreachable. Anything else
+      # (INFO restricted, an odd version string) assumes a current server —
+      # the runtime fall-back (UnsupportedError) still catches a wrong guess.
       def supported?
         version = Cogworker.config.redis { |c| c.info('server')['redis_version'] }
         Gem::Version.new(version) >= MIN_REDIS_VERSION
+      rescue Redis::BaseConnectionError
+        raise
       rescue StandardError
-        true # can't tell (Redis not reachable yet): assume a current server
+        true
       end
 
       # Puts every job still on `identity`'s in-progress list back on its
-      # queue; returns how many. Used for this process's own unfinished jobs
-      # on shutdown (`Manager#stop!`) and for dead processes' lists.
-      def requeue_in_progress(identity)
+      # queue; returns how many — or -1 if `if_dead_for:` is given and the
+      # owner turns out not to be dead (see REQUEUE_SCRIPT). Used without
+      # it for this process's own unfinished jobs on shutdown
+      # (`Manager#stop!`), with it for presumed-dead processes' lists.
+      def requeue_in_progress(identity, if_dead_for: nil)
         Cogworker.config.redis do |c|
-          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)],
-                                 argv: [RedisKeys::QUEUE_PREFIX, 'default', RedisKeys.periodic_running(''),
-                                        Periodic::RunningLock.queued_ttl])
+          LuaScript.run(c, REQUEUE_SCRIPT,
+                 keys: [RedisKeys.in_progress(identity), RedisKeys.process(identity), RedisKeys::LAST_BEAT],
+                 argv: [*requeue_args, identity, if_dead_for.to_s])
         end
       end
 
-      # Requeues the in-progress list of every process that has gone
-      # `config.orphan_threshold` without a heartbeat — i.e. that died
-      # without a clean shutdown (see #dead?). A process always
-      # writes its heartbeat before its first fetch (`Heartbeat#start!`), so
-      # a live process's list is never mistaken for an orphan. Walks
-      # IN_PROGRESS_IDENTITIES (each reliable-fetch process registers itself
-      # there on every beat) rather than SCANning the keyspace; `scan: true`
-      # (once per process, at boot) also picks up lists from before that
-      # set existed.
+      def requeue_args
+        [RedisKeys::QUEUE_PREFIX, 'default', RedisKeys.periodic_running(''), Periodic::RunningLock.queued_ttl]
+      end
+
+      # Requeues the in-progress list of every process that has died without
+      # a clean shutdown. "Dead" means its presence key is gone (no beat for
+      # Heartbeat::TTL) *and* `config.orphan_threshold` has passed since its
+      # last recorded beat (LAST_BEAT, on Redis's clock) — the presence key
+      # alone expires after a minute, too eagerly for a live process that
+      # merely couldn't beat for a while (a Redis outage or failover, a long
+      # GVL-holding call), whose running jobs would then run twice. With no
+      # recorded beat at all (lists from before LAST_BEAT existed), the
+      # presence key alone decides. Checked inside REQUEUE_SCRIPT, atomically
+      # with the requeue. A process always beats before its first fetch
+      # (`Heartbeat#start!`). Walks IN_PROGRESS_IDENTITIES (each
+      # reliable-fetch process registers itself there on every beat) rather
+      # than SCANning the keyspace; `scan: true` (once per reliable process,
+      # at boot) also picks up lists from before that set existed.
       def recover_orphans(scan: false)
         own = Cogworker.identity
         candidate_identities(scan: scan).sum do |identity|
-          next 0 if identity == own || !dead?(identity)
+          next 0 if identity == own
 
-          requeue_in_progress(identity).tap do |moved|
-            forget_if_empty(identity)
-            next unless moved.positive?
+          moved = requeue_in_progress(identity, if_dead_for: Cogworker.config.orphan_threshold)
+          next 0 if moved.negative? # alive after all: keep its registration and last beat
 
-            Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" }
-          end
+          forget_if_empty(identity)
+          Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" } if moved.positive?
+          moved
         end
       end
 
-      # Its presence key gone (no beat for Heartbeat::TTL) is not enough on
-      # its own: a live process that just couldn't beat for a minute (a
-      # Redis outage or failover, a long GVL-holding call) would lose its
-      # in-progress jobs to a second, parallel run. It also has to have gone
-      # `orphan_threshold` since its last recorded beat — compared on
-      # Redis's clock on both sides, so host clock skew doesn't matter. No
-      # recorded beat at all (a list from before LAST_BEAT existed) falls
-      # back to the presence key alone.
-      def dead?(identity)
+      # This process's own in-progress entries that no processor thread is
+      # running (not in `cogworker:workers:<identity>`) — left behind when
+      # neither acknowledging nor handing a job back worked. Requeued once
+      # they've been seen stray on two calls in a row (the caller passes
+      # back what the previous call returned), which rules out a job caught
+      # in the instant between being fetched and registered. Returns this
+      # call's strays that weren't requeued yet.
+      def reconcile(identity, previous_strays = [])
         Cogworker.config.redis do |c|
-          next false if c.exists?(RedisKeys.process(identity))
+          listed = c.lrange(RedisKeys.in_progress(identity), 0, -1)
+          running = c.hvals(RedisKeys.workers(identity)).filter_map { |raw| parse_hash(raw)&.dig('payload', 'jid') }
+          strays = listed.reject { |raw| running.include?(parse_hash(raw)&.fetch('jid', nil)) }
+          (strays & previous_strays).each do |raw|
+            next unless LuaScript.run(c, REQUEUE_ONE_SCRIPT, keys: [RedisKeys.in_progress(identity)],
+                                                            argv: [raw, *requeue_args]) == 1
 
-          last = c.hget(RedisKeys::LAST_BEAT, identity)
-          last.nil? || c.time.first - last.to_f > Cogworker.config.orphan_threshold
+            Cogworker.logger.warn { "requeued stray in-progress job jid=#{parse_hash(raw)&.fetch('jid', nil)}" }
+          end
+          strays - previous_strays
         end
       end
 
+      def parse_hash(raw)
+        value = JSON.parse(raw)
+        value.is_a?(Hash) ? value : nil
+      rescue JSON::ParserError
+        nil
+      end
+
       def candidate_identities(scan:)
         Cogworker.config.redis do |c|
           identities = c.smembers(RedisKeys::IN_PROGRESS_IDENTITIES)
@@ -169,10 +242,13 @@ module Cogworker
     def retrieve_work
       keys = active_keys
       if keys.empty?
-        sleep(TIMEOUT)
+        interruptible_sleep(TIMEOUT)
         return nil
       end
 
+      distinct = keys.uniq
+      return blocking_fetch(distinct.first) if distinct.size == 1
+
       result = run_fetch_script(keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
       unless result
         idle_sleep
@@ -189,8 +265,15 @@ module Cogworker
     # no longer in progress (already requeued by `requeue_in_progress`).
     def give_back(work)
       Cogworker.config.redis do |c|
-        c.eval(GIVE_BACK_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), RedisKeys.queue(work.queue)],
-                                 argv: [work.raw_job])
+        LuaScript.run(c, REQUEUE_ONE_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity)],
+                                            argv: [work.raw_job, *self.class.requeue_args])
+      end
+    end
+
+    def interrupt(work, set, score, payload)
+      Cogworker.config.redis do |c|
+        LuaScript.run(c, INTERRUPT_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), set],
+                                          argv: [work.raw_job, score, payload])
       end
     end
 
@@ -200,14 +283,22 @@ module Cogworker
 
     private
 
-    def run_fetch_script(keys)
-      Cogworker.config.redis do |c|
-        c.evalsha(FETCH_SCRIPT_SHA, keys: keys)
-      rescue Redis::CommandError => e
-        raise unless e.message.start_with?('NOSCRIPT')
-
-        c.eval(FETCH_SCRIPT, keys: keys) # loads it into the script cache for next time
+    # With a single (unpaused) queue there's no weighting to honor, so no
+    # need to poll: a blocking BLMOVE waits on the server and returns the
+    # moment a job arrives, for up to TIMEOUT — same as BasicFetch's BRPOP.
+    def blocking_fetch(queue_key)
+      raw_job = Cogworker.config.redis do |c|
+        c.blmove(queue_key, RedisKeys.in_progress(Cogworker.identity), 'RIGHT', 'LEFT', timeout: TIMEOUT)
       end
+      raw_job && UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
+    rescue Redis::CommandError => e
+      raise unless e.message.match?(/unknown (redis )?command/i)
+
+      raise UnsupportedError, e.message
+    end
+
+    def run_fetch_script(keys)
+      Cogworker.config.redis { |c| LuaScript.run(c, FETCH_SCRIPT, keys: keys) }
     rescue Redis::CommandError => e
       raise unless e.message.match?(/unknown (redis )?command/i)
 
@@ -219,7 +310,7 @@ module Cogworker
     def idle_sleep
       max = Cogworker.config.fetch_idle_max_interval
       @idle_interval = @idle_interval ? [@idle_interval * 2, max].min : [EMPTY_POLL_INTERVAL, max].min
-      sleep(@idle_interval * (0.75 + (rand * 0.25)))
+      interruptible_sleep(@idle_interval * (0.75 + (rand * 0.25)))
     end
   end
 end

data/lib/cogworker/scheduled.rb

@@ -55,6 +55,12 @@ module Cogworker
       @thread&.kill
     end
 
+    # True once the first orphan check has completed — what the periodic
+    # Ticker waits for before its first tick (see Launcher#initialize).
+    def recovered_once?
+      @recovered_once == true
+    end
+
     private
 
     # A failed poll is logged and retried next interval rather than ending
@@ -72,13 +78,17 @@ module Cogworker
       end
     end
 
+    # Next check scheduled only once this one has completed: a failed one
+    # (Redis down) is retried on the next poll rather than a minute later.
     def recover_orphans_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       return if @next_orphan_check && now < @next_orphan_check
 
-      scan = @next_orphan_check.nil? && @manager.fetch_class == ReliableFetch
+      reliable = @manager.fetch_class == ReliableFetch
+      ReliableFetch.recover_orphans(scan: !@recovered_once && reliable)
+      @strays = ReliableFetch.reconcile(Cogworker.identity, @strays || []) if reliable
+      @recovered_once = true
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
-      ReliableFetch.recover_orphans(scan: scan)
     end
 
     def enqueue_due_jobs
@@ -104,8 +114,8 @@ module Cogworker
     def bury(conn, set, raw)
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
-      buried = conn.eval(BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
-                                      argv: [raw, job['failed_at'], JSON.generate(job)])
+      buried = LuaScript.run(conn, BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
+                                               argv: [raw, job['failed_at'], JSON.generate(job)])
       return unless buried == 1
 
       Throughput.record('failed') # counted like Processor#bury_unparseable

data/lib/cogworker/status/server_middleware.rb

@@ -15,7 +15,7 @@ module Cogworker
         write(job, queue, 'complete')
       rescue Exception => e # rubocop:disable Lint/RescueException
         write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
-              error_class: e.class.name, error_message: e.message.to_s[0, 10_000])
+              error_class: e.class.name, error_message: JobUtil.error_message(e))
         raise e
       end
 

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
 end

data/lib/cogworker/web/optional_session.rb

@@ -20,6 +20,13 @@ module Cogworker
       def call(env)
         env['rack.session'] ? @app.call(env) : @with_session.call(env)
       end
+
+      # A session middleware added via `Web.use` sits *after* this one in
+      # the stack, so it can't be detected per request the way an upstream
+      # one is — `Web.build_app` checks for one up front instead.
+      def self.session_middleware?(middleware)
+        middleware.is_a?(Class) && middleware <= Rack::Session::Abstract::Persisted
+      end
     end
   end
 end

data/lib/cogworker/web.rb

@@ -104,9 +104,10 @@ module Cogworker
         @session_secret ||= ENV.fetch('COGWORKER_SESSION_SECRET') { SecureRandom.hex(32) }
       end
 
-      # `false` drops the built-in session cookie entirely (it's already
-      # skipped automatically whenever the host app provides a session —
-      # see OptionalSession). Default true.
+      # `false` drops the built-in session cookie entirely. It's already
+      # skipped automatically whenever the host app provides a session
+      # (OptionalSession) or one is added via `Web.use` (a
+      # `Rack::Session::*` class — see `build_app`). Default true.
       def builtin_session
         @builtin_session.nil? || @builtin_session
       end
@@ -258,7 +259,7 @@ module Cogworker
 
       def build_app
         builder = Rack::Builder.new
-        if builtin_session
+        if builtin_session && middlewares.none? { |mw, _| Cogworker::Web::OptionalSession.session_middleware?(mw) }
           builder.use(Cogworker::Web::OptionalSession, secret: session_secret, key: 'cogworker.session')
         end
         # No `cache_control:` (no `immutable`/long `max-age`): these files

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - vdonec
@@ -222,6 +222,7 @@ files:
 - lib/cogworker/job_util.rb
 - lib/cogworker/launcher.rb
 - lib/cogworker/logging.rb
+- lib/cogworker/lua_script.rb
 - lib/cogworker/manager.rb
 - lib/cogworker/middleware/chain.rb
 - lib/cogworker/owned_key.rb