cogworker 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13e1ad0fec77124674f55dc3e1f7213bd2a73834ebb474261ed82a337fc1bb3e
-  data.tar.gz: ed51fed2854035c83016b3180fa7872772e6f60bb2dd84e44089485a536f81e4
+  metadata.gz: bfb90fe3a477f31658c58fd09a41e4404a0c09999b74614789bd343ee704fad2
+  data.tar.gz: 407aa20293538fa35c8219f88f21fd8b1c15150d96e77d2b495d017ef21f924e
 SHA512:
-  metadata.gz: 0d78dd7a7c4bbdf8bfc8c8619467922e732879908562b1ac99fed71ce111a8a4925136e87a9c4e56cb5abed57867d76f06c5d929fcb822156d4fd0add7a182d9
-  data.tar.gz: 265d890a8f7d65b180dcb9a0f00f0d2ba6daf16fa9044f7bdb94b88b2cbeced1f6d61f5e11c86d5b107d281bbbc47591b54cb3fc6820f7a7fecc4732dec451d2
+  metadata.gz: c7df3eba977499a45acca66e8d25fd87cfe4669b9098c1e99be798f587de2ddf76edc75adb01953e8fe311ac324d7e4eb44cce66a33be264efe4f266e67d8f3e
+  data.tar.gz: 7fa4d7ba18f7be8ae8e80bd7b260be8bd8e9387122a623802812abd166652859f48669bbb42c16f72f15df1104d855934f4cfdb6cf6aede2737d740540d27278

data/lib/cogworker/basic_fetch.rb

@@ -62,7 +62,7 @@ module Cogworker
       Cogworker.config.redis do |c|
         c.hvals(RedisKeys.workers(identity)).count do |raw|
           entry = JSON.parse(raw)
-          c.rpush(RedisKeys.queue(entry.fetch('queue')), JSON.generate(entry.fetch('payload')))
+          requeue_entry(c, entry.fetch('queue'), entry.fetch('payload'))
           true
         rescue StandardError => e
           Cogworker.logger.error { "couldn't requeue in-flight entry #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -71,6 +71,16 @@ module Cogworker
       end
     end
 
+    def self.requeue_entry(conn, queue, job)
+      queue_key = RedisKeys.queue(queue)
+      retake = job['periodic_until_executed'] && job['periodic_pjid'] && job['jid']
+      lock_key = retake ? RedisKeys.periodic_running(job['periodic_pjid']) : queue_key
+      conn.eval(REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
+                                      argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
+                                             Periodic::RunningLock.queued_ttl])
+    end
+    private_class_method :requeue_entry
+
     private
 
     def active_keys
@@ -81,5 +91,18 @@ module Cogworker
     end
 
     UnitOfWork = Struct.new(:queue, :raw_job)
+
+    # KEYS[1] = the queue, KEYS[2] = the job's periodic running lock (or the
+    # queue again when there's none to take); ARGV[1] = payload, ARGV[2] =
+    # "1" to re-take that lock, ARGV[3] = jid, ARGV[4] = lock TTL. Requeue
+    # and lock in one step: as two commands, a failure in between left the
+    # job queued without its lock (and miscounted).
+    REQUEUE_ENTRY_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      redis.call("RPUSH", KEYS[1], ARGV[1])
+      if ARGV[2] == "1" then
+        retake_running_lock(KEYS[2], ARGV[3], ARGV[4])
+      end
+      return 1
+    LUA
   end
 end

data/lib/cogworker/config.rb

@@ -8,7 +8,8 @@ module Cogworker
   class Config
     FETCH_MODES = %i[reliable basic].freeze
 
-    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch
+    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
+                :orphan_threshold
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -39,9 +40,36 @@ module Cogworker
       # On an older Redis, `:reliable` falls back to `:basic` with a warning
       # (Manager#resolve_fetch_class).
       @fetch = :reliable
+      # Longest pause (seconds) ReliableFetch makes between polls of empty
+      # queues — the worst-case delay before an idle worker notices a new
+      # job. See ReliableFetch::EMPTY_POLL_INTERVAL.
+      @fetch_idle_max_interval = 1.0
+      # How long (seconds) a process must have gone without a heartbeat
+      # before ReliableFetch.recover_orphans treats it as dead and requeues
+      # its in-progress jobs. Higher: fewer false "dead" verdicts on a live
+      # process that merely couldn't beat for a while (a Redis outage or
+      # failover, a long GVL-holding call) — each of which runs its jobs
+      # twice; lower: a really crashed process's jobs come back sooner.
+      @orphan_threshold = 300
       register_default_middleware
     end
 
+    def orphan_threshold=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @orphan_threshold = seconds
+    end
+
+    def fetch_idle_max_interval=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "fetch_idle_max_interval must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @fetch_idle_max_interval = seconds
+    end
+
     def fetch=(mode)
       resolved = mode.to_s.to_sym
       unless FETCH_MODES.include?(resolved)

data/lib/cogworker/heartbeat.rb

@@ -15,6 +15,7 @@ module Cogworker
   class Heartbeat
     INTERVAL = 5
     TTL = 60
+    STARTUP_RETRY_INTERVAL = 1
 
     def initialize(manager)
       @manager = manager
@@ -23,12 +24,21 @@ module Cogworker
     # The first beat is synchronous, before any processor fetches a job:
     # ReliableFetch.recover_orphans treats an in-progress list with no live
     # heartbeat behind it as a dead process's, so this process must be
-    # visibly alive before its own list can have anything on it.
-    def start!
+    # visibly alive before its own list can have anything on it. With Redis
+    # unreachable, it keeps retrying (every STARTUP_RETRY_INTERVAL) rather
+    # than letting the Manager start fetching with no presence key — until
+    # it succeeds (returns true) or `abort_if` says to give up (returns
+    # false, e.g. a stop signal arrived meanwhile; nothing is started then).
+    def start!(abort_if: -> { false })
       @stopping = false
-      beat_safely
+      until beat_safely
+        return false if abort_if.call
+
+        sleep(STARTUP_RETRY_INTERVAL)
+      end
       @beat_thread = Thread.new { beat_loop }
       @signal_thread = Thread.new { subscribe_loop }
+      true
     end
 
     # Tears down both background threads, not just the Redis keys — leaving
@@ -67,6 +77,12 @@ module Cogworker
       Cogworker.config.redis do |c|
         c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
         c.srem?(RedisKeys::PROCESSES, Cogworker.identity)
+        # Clean shutdown: Manager#stop! already requeued this process's
+        # in-progress list, so there's nothing left for recovery to find.
+        if c.llen(RedisKeys.in_progress(Cogworker.identity)).zero?
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, Cogworker.identity)
+          c.hdel(RedisKeys::LAST_BEAT, Cogworker.identity)
+        end
       end
     end
 
@@ -83,12 +99,15 @@ module Cogworker
 
     def beat_safely
       beat
+      true
     rescue StandardError => e
       Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      false
     end
 
     def beat
       identity = Cogworker.identity
+      reliable = @manager.fetch_class == ReliableFetch
       info = {
         'hostname' => Cogworker.hostname,
         'pid' => ::Process.pid,
@@ -106,6 +125,10 @@ module Cogworker
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
         c.expire(RedisKeys.workers(identity), TTL)
+        if reliable
+          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hset(RedisKeys::LAST_BEAT, identity, c.time.first) # Redis's clock, not this host's
+        end
         touch_running_periodic_locks(c, identity)
       end
     end
@@ -120,7 +143,7 @@ module Cogworker
         job = JSON.parse(raw)['payload']
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
-        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock::ACTIVE_TTL, conn)
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
       rescue StandardError => e
         # Per entry: one bad entry mustn't leave the others' locks to expire.
         Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -194,6 +217,7 @@ module Cogworker
     def remote_stop!
       @manager.stop!
       cleanup_presence!
+      Cogworker.flush_output!
       ::Process.exit!(true)
     end
   end

data/lib/cogworker/job_util.rb

@@ -64,37 +64,42 @@ module Cogworker
     # `queue` may be nil).
     def requeueable(raw)
       job = JSON.parse(raw)
-      job.is_a?(Hash) && job['queue'].is_a?(String) ? job : nil
+      job.is_a?(Hash) && job['queue'].is_a?(String) && !job['queue'].empty? ? job : nil
     rescue JSON::ParserError, TypeError
       nil
     end
 
-    # Validate, then claim, then requeue — in that order, so an entry that
-    # can't be requeued is never `zrem`'d and lost (it stays where it is for
-    # the caller to deal with). Returns `:requeued`, `:gone` (someone else
-    # won the `zrem` first) or `:invalid`. The one implementation behind
-    # `Scheduled#graduate` and every Web UI retry action.
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:queues, KEYS[3] = the queue;
+    # ARGV[1] = the entry as stored, ARGV[2] = queue name, ARGV[3] = the
+    # payload to push. Claim and push in one atomic step: as separate
+    # commands, a crash or dropped connection between the ZREM and the
+    # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
+    # by Lua's cjson (which would round large integers in args).
+    CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("SADD", KEYS[2], ARGV[2])
+      redis.call("LPUSH", KEYS[3], ARGV[3])
+      return 1
+    LUA
+
+    # Validate, then (atomically) claim and requeue — an entry that can't be
+    # requeued is never removed and lost (it stays where it is for the
+    # caller to deal with). Returns `:requeued`, `:gone` (someone else
+    # claimed it first) or `:invalid`. The one implementation behind
+    # `Scheduled#graduate` and every Web UI retry action. Re-stamps
+    # `enqueued_at`: queue latency is measured from it, and a retried job
+    # still carried its original push time (a scheduled one, none at all).
     def claim_and_requeue(conn, set, raw)
       job = requeueable(raw)
       return :invalid unless job
-      return :gone unless conn.zrem(set, raw)
 
-      requeue(conn, job)
-      :requeued
-    end
-
-    # Puts a job taken off `cogworker:schedule`/`retry`/`dead` (the caller
-    # has already won its `zrem`) back onto its queue — the one shared
-    # implementation behind `Scheduled#graduate` and the Web UI's retry
-    # actions. Re-stamps `enqueued_at`: queue latency is measured from it,
-    # and a retried job still carried its original push time (a scheduled
-    # one, none at all), inflating the queue's latency by its whole history.
-    def requeue(conn, job)
       job['enqueued_at'] = Time.now.to_f
-      conn.multi do |tx|
-        tx.sadd?(RedisKeys::QUEUES, job['queue'])
-        tx.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
-      end
+      won = conn.eval(CLAIM_AND_REQUEUE_SCRIPT,
+                      keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
+                      argv: [raw, job['queue'], JSON.generate(job)])
+      won == 1 ? :requeued : :gone
     end
 
     UNPARSEABLE_CLASS = '(unparseable)'

data/lib/cogworker/launcher.rb

@@ -23,7 +23,8 @@ module Cogworker
       Cogworker.reset_identity!
       install_signal_traps
       resolve_periodic_classes
-      @heartbeat.start! # first: see Heartbeat#start!
+      return unless start_heartbeat
+
       @manager.start!
       @scheduled.start!
       @ticker.start!
@@ -44,6 +45,23 @@ module Cogworker
 
     private
 
+    # First, and blocking until it succeeds: see Heartbeat#start!. A stop
+    # signal arriving while Redis is still unreachable ends the wait (and
+    # the process) without ever starting anything; a quiet one is applied
+    # and the wait goes on.
+    def start_heartbeat
+      until @heartbeat.start!(abort_if: -> { !@signal_queue.empty? })
+        case @signal_queue.pop
+        when :stop
+          Cogworker.logger.info { 'Received stop signal before startup completed' }
+          stop!
+          return false
+        when :quiet then quiet!
+        end
+      end
+      true
+    end
+
     # Resolves every periodic entry's class here, on the main thread, before
     # any processor thread exists. `Processor#build_worker` otherwise does
     # the first `const_get` lazily on whichever processor thread picks the

data/lib/cogworker/logging.rb

@@ -7,6 +7,10 @@ module Cogworker
   # it), with a compact one-line-per-entry format including pid.
   module Logging
     def self.default_logger(io = $stdout)
+      # Unbuffered: processes here end via `Process.exit!` (swarm children,
+      # a remote stop), which skips Ruby's IO flush — with stdout a pipe
+      # (docker, systemd) the last lines logged before exit were lost.
+      io.sync = true if io.respond_to?(:sync=)
       logger = ::Logger.new(io)
       logger.level = ::Logger::INFO
       logger.formatter = proc do |severity, time, _progname, msg|

data/lib/cogworker/manager.rb

@@ -4,11 +4,9 @@ module Cogworker
   # Owns the `concurrency`-sized thread pool of Processors for one OS
   # process, and the quiet/stop state they all poll.
   class Manager
-    attr_reader :fetch_class
-
     def initialize(config = Cogworker.config)
       @config = config
-      @fetch_class = resolve_fetch_class
+      @fetch_mutex = Mutex.new
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -20,6 +18,31 @@ module Cogworker
       @config.queues
     end
 
+    # Resolved on first use — from a processor thread's first fetch, after
+    # the Heartbeat's first beat proved Redis reachable — not in the
+    # constructor: that needed Redis just to build a Manager, and when
+    # Redis was unreachable at that moment, the version check couldn't run
+    # at all (see ReliableFetch.supported?).
+    def fetch_class
+      @fetch_mutex.synchronize { @fetch_class ||= resolve_fetch_class }
+    end
+
+    # The server turned out not to support ReliableFetch after all
+    # (ReliableFetch::UnsupportedError on a real fetch): every processor
+    # switches to BasicFetch from its next fetch on. Safe, since no job can
+    # be on the in-progress list — no LMOVE ever succeeded.
+    def fall_back_to_basic_fetch!(error)
+      @fetch_mutex.synchronize do
+        next if @fetch_class == BasicFetch
+
+        Cogworker.logger.warn do
+          "config.fetch = :reliable isn't supported by this Redis (#{error.message[0, 120]}); " \
+            'falling back to :basic (a job is lost if its process dies mid-job)'
+        end
+        @fetch_class = BasicFetch
+      end
+    end
+
     def start!
       @processors.each(&:start!)
     end
@@ -87,7 +110,7 @@ module Cogworker
     end
 
     def requeue_unfinished
-      moved = @fetch_class.requeue_in_progress(Cogworker.identity)
+      moved = fetch_class.requeue_in_progress(Cogworker.identity)
       Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
     rescue StandardError => e
       Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }

data/lib/cogworker/periodic/claim.lua

@@ -7,8 +7,10 @@
 -- ARGV[4] = jid the caller will push this slot's job under
 -- ARGV[5] = running-lock TTL (seconds)
 --
--- Returns 1 if this call won the claim for this slot (the caller should
--- enqueue the job), 0 otherwise. Composes two guards: `last_slot` stops a
+-- Returns {1, <previous last_slot, or "" if none>} if this call won the
+-- claim for this slot (the caller should enqueue the job — and, if that
+-- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
+-- value), 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
@@ -20,7 +22,8 @@ if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
   return 0
 end
 
-local last = tonumber(redis.call("GET", KEYS[2]) or "0")
+local previous = redis.call("GET", KEYS[2])
+local last = tonumber(previous or "0")
 if tonumber(ARGV[1]) <= last then
   return 0
 end
@@ -33,4 +36,4 @@ redis.call("SET", KEYS[2], ARGV[1])
 if ARGV[2] == "until_executed" then
   redis.call("SET", KEYS[1], ARGV[4], "EX", ARGV[5])
 end
-return 1
+return {1, previous or ""}

data/lib/cogworker/periodic/release_middleware.rb

@@ -4,7 +4,7 @@ module Cogworker
   module Periodic
     # Drives the server side of the `periodic:running:<pjid>` lock (used
     # only for `unique: :until_executed` entries; see RunningLock for the
-    # whole lifecycle): shortens it to RunningLock::ACTIVE_TTL when the run
+    # whole lifecycle): shortens it to `RunningLock.active_ttl` when the run
     # starts, and releases it on success or on the terminal failed attempt
     # (a failure that will be retried is `Processor#route_failure`'s to
     # extend — only it knows the retry delay).
@@ -36,7 +36,7 @@ module Cogworker
       # the lock just keeps its longer queued TTL until this process's next
       # heartbeat shortens it — within Heartbeat::INTERVAL.
       def shorten_lock(pjid, jid)
-        RunningLock.touch(pjid, jid, RunningLock::ACTIVE_TTL)
+        RunningLock.touch(pjid, jid, RunningLock.active_ttl)
       rescue StandardError => e
         Cogworker.logger.warn { "couldn't shorten periodic lock for #{pjid}: #{e.class}: #{e.message}" }
       end

data/lib/cogworker/periodic/running_lock.rb

@@ -12,22 +12,47 @@ module Cogworker
     #   (and release a lock that didn't exist yet) before the lock was
     #   written, wedging the entry forever. TTL = `queued_ttl`, covering
     #   however long the job may wait in its queue.
-    # - shortened to ACTIVE_TTL once a processor starts it, then kept alive
+    # - shortened to `active_ttl` once a processor starts it, then kept alive
     #   by that process's Heartbeat every beat — so a run whose process dies
-    #   (OOM, SIGKILL, shutdown past the drain timeout) frees the entry
-    #   within ACTIVE_TTL instead of never.
+    #   (OOM, SIGKILL, shutdown past the drain timeout) can't hold the entry
+    #   forever. `active_ttl` deliberately outlives the time it takes for
+    #   that dead process's job to be recovered (`config.orphan_threshold`
+    #   plus one orphan check), so the lock is still there, still owned by
+    #   that job, when the job is requeued — at which point `retake` (or ReliableFetch's
+    #   REQUEUE_SCRIPT) extends it again for the wait in the queue. A shorter
+    #   TTL let the lock lapse first: the ticker would enqueue the next slot
+    #   while the recovered run was still to come, i.e. two runs at once.
     # - extended on a non-terminal failure to cover the retry backoff plus
     #   `queued_ttl` (`Processor#extend_locks_for_retry` — the job now waits
     #   in `cogworker:retry`, not on a process).
     # - deleted on success or on the terminal failure.
     module RunningLock
-      ACTIVE_TTL = 60
+      # Lua, prepended to every script that puts a job back on its queue
+      # (ReliableFetch::REQUEUE_SCRIPT, BasicFetch::REQUEUE_ENTRY_SCRIPT), so
+      # the requeue and the lock move together in one atomic step. Extends
+      # the lock if still that job's, takes it if it lapsed, leaves it if a
+      # newer run holds it.
+      RETAKE_FUNCTION = <<~LUA
+        local function retake_running_lock(lock, jid, ttl)
+          local owner = redis.call("GET", lock)
+          if not owner then
+            redis.call("SET", lock, jid, "EX", ttl)
+          elseif owner == jid then
+            redis.call("EXPIRE", lock, ttl)
+          end
+        end
+      LUA
 
       module_function
 
       # Whole seconds: it goes straight into `SET ... EX` in claim.lua, which
       # rejects a non-integer (a fractional `unique_lock_ttl` used to make
       # every claim — and so every tick — fail).
+      # A method, not a constant: it follows `config.orphan_threshold`.
+      def active_ttl
+        (Cogworker.config.orphan_threshold + Scheduled::ORPHAN_CHECK_INTERVAL + 60).to_f.ceil
+      end
+
       def queued_ttl
         Cogworker.config.unique_lock_ttl.to_f.ceil
       end

data/lib/cogworker/periodic/ticker.rb

@@ -17,6 +17,25 @@ module Cogworker
 
       CLAIM_SCRIPT = File.read(File.join(__dir__, 'claim.lua'))
 
+      # Undoes a won claim whose push then failed, so the slot isn't lost:
+      # puts `last_slot` back (only if it still is this slot — a later slot
+      # may have been claimed since) and drops the per-slot NX lock.
+      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
+      # previous last_slot ("" if none: then slot - 1, which still lets this
+      # slot fire, unlike deleting the key — with catch_up off, a missing
+      # last_slot would be re-primed to this very slot instead).
+      ROLLBACK_SCRIPT = <<~LUA
+        if redis.call("GET", KEYS[1]) == ARGV[1] then
+          if ARGV[2] == "" then
+            redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
+          else
+            redis.call("SET", KEYS[1], ARGV[2])
+          end
+        end
+        redis.call("DEL", KEYS[2])
+        return 1
+      LUA
+
       def initialize(manager, entries, catch_up: true)
         @manager = manager
         @entries = entries
@@ -57,8 +76,9 @@ module Cogworker
 
       # A failed tick is logged and retried next interval rather than ending
       # the thread — a single Redis blip used to silently stop every
-      # periodic entry in this process for good. A slot whose claim raised
-      # isn't recorded in `@last_checked_slot`, so the next tick retries it.
+      # periodic entry in this process for good. A slot whose claim or push
+      # raised isn't recorded in `@last_checked_slot` (and a failed push
+      # rolls its claim back — see #enqueue), so the next tick retries it.
       def run
         until @manager.stopping?
           begin
@@ -77,13 +97,14 @@ module Cogworker
           next if @last_checked_slot[entry.pjid] == slot
 
           jid = SecureRandom.hex(12)
-          enqueue(entry, slot, jid) if !disabled?(entry) && claim?(entry, slot, jid)
+          previous = claim(entry, slot, jid) unless disabled?(entry)
+          enqueue(entry, slot, jid, previous) if previous
           @last_checked_slot[entry.pjid] = slot
         end
       end
 
       # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
-      # step entirely, short-circuiting before `claim?` even runs, so
+      # step entirely, short-circuiting before `claim` even runs, so
       # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
       # while disabled: the Web UI's own "LastRun" column keeps showing the
       # last time it *actually* ran, not a due-but-skipped slot, and
@@ -93,6 +114,15 @@ module Cogworker
       # persisted last_slot) still advances either way, exactly as it
       # already did before this entry ever had a disabled state — it only
       # stops this same tick loop from re-evaluating the same slot twice.
+      def rollback_claim(entry, slot, previous_slot)
+        Cogworker.config.redis do |c|
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
+          c.eval(ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+        end
+      rescue StandardError => e
+        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot}: #{e.class}: #{e.message}" }
+      end
+
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
       end
@@ -101,8 +131,10 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      def claim?(entry, slot, jid)
-        return false if !@catch_up && priming_first_slot?(entry, slot)
+      # nil if this process didn't win the slot; otherwise the previous
+      # `last_slot` value ("" if none), for #rollback_claim.
+      def claim(entry, slot, jid)
+        return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           c.eval(CLAIM_SCRIPT,
@@ -110,7 +142,7 @@ module Cogworker
                         RedisKeys.periodic_lock(entry.pjid, slot)],
                  argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
-        result == 1
+        result.is_a?(Array) ? result[1].to_s : nil
       end
 
       # With catch-up disabled, an entry's very first tick — ever, across
@@ -140,15 +172,19 @@ module Cogworker
       # Only undone here if the push didn't actually produce a job (raised,
       # or a client middleware swallowed it), so a job that never existed
       # can't hold the entry for the whole `RunningLock.queued_ttl`.
-      def enqueue(entry, slot, jid)
+      def enqueue(entry, slot, jid, previous_slot)
         job = {
           'class' => entry.class_name, 'args' => entry.args, 'retry' => entry.retry,
           'periodic_pjid' => entry.pjid, 'periodic_slot' => slot, 'jid' => jid
         }
+        # Lets a requeue after a crash/shutdown re-take this run's lock
+        # (Periodic::RunningLock::RETAKE_FUNCTION, in both fetches' requeue scripts).
+        job['periodic_until_executed'] = true if entry.until_executed?
         pushed = begin
           Client.push(job)
         rescue StandardError
           RunningLock.release(entry.pjid, jid) if entry.until_executed?
+          rollback_claim(entry, slot, previous_slot)
           raise
         end
         return unless entry.until_executed?

data/lib/cogworker/processor.rb

@@ -16,7 +16,6 @@ module Cogworker
       @manager = manager
       @busy = false
       @tid = SecureRandom.hex(6)
-      @fetcher = manager.fetch_class.new(manager.queues)
     end
 
     def start!
@@ -54,23 +53,46 @@ module Cogworker
         return
       end
 
-      work = @fetcher.retrieve_work
+      work = begin
+        fetcher.retrieve_work
+      rescue ReliableFetch::UnsupportedError => e
+        @manager.fall_back_to_basic_fetch!(e)
+        @fetcher = nil
+        return
+      end
       return unless work
 
       # Fetched after shutdown began (e.g. still blocked in BRPOP when
       # `Manager#stop!` requeued unfinished work, which it then popped right
       # back): hand it back rather than start it, since the process is about
       # to exit underneath it.
-      return @fetcher.give_back(work) if @manager.stopping?
+      return fetcher.give_back(work) if @manager.stopping?
 
       @manager.processor_busy!
       @busy = true
+      finished = false
       begin
         execute(work)
+        finished = true
       ensure
         @busy = false
         @manager.processor_idle!
-        acknowledge(work)
+        finished ? acknowledge(work) : give_back(work)
+      end
+    end
+
+    # `execute` raising means bookkeeping failed (Redis, mid-way through
+    # registering the job or routing its failure to retry/dead) — not the
+    # job itself, whose own errors `execute` always handles. Acknowledging
+    # here would drop a job that never made it to retry/dead; instead it
+    # goes straight back on its queue (it may run again: at-least-once). If
+    # even that fails, it simply stays on the in-progress list, which
+    # `Manager#stop!`/orphan recovery requeue later.
+    def give_back(work)
+      fetcher.give_back(work)
+    rescue StandardError => e
+      Cogworker.logger.error do
+        "couldn't hand back jid=#{jid_of(work)} after a failed run (left in progress): #{e.class}: #{e.message}"
       end
     end
 
@@ -82,7 +104,7 @@ module Cogworker
       attempts = 0
       begin
         attempts += 1
-        @fetcher.acknowledge(work)
+        fetcher.acknowledge(work)
       rescue StandardError => e
         if attempts < ACK_ATTEMPTS
           sleep(0.1 * attempts)
@@ -174,6 +196,12 @@ module Cogworker
       Cogworker.logger.error { "unparseable job on queue #{work.queue} moved to dead: #{error.message}" }
     end
 
+    # Built on first use, not in the constructor (see Manager#fetch_class),
+    # and rebuilt after a fall-back to BasicFetch.
+    def fetcher
+      @fetcher ||= @manager.fetch_class.new(@manager.queues)
+    end
+
     def build_worker(job)
       klass = Object.const_get(job['class'])
       worker = klass.new

data/lib/cogworker/redis_keys.rb

@@ -10,6 +10,13 @@ module Cogworker
   module RedisKeys
     QUEUE_PREFIX = 'cogworker:queue:'
     IN_PROGRESS_PREFIX = 'cogworker:inprogress:'
+    # Every identity that has (or may have) an in-progress list — what
+    # orphan recovery walks instead of SCANning the whole keyspace.
+    IN_PROGRESS_IDENTITIES = 'cogworker:inprogress_identities'
+    # identity => Redis server time (epoch seconds) of its last heartbeat.
+    # No TTL, unlike `process(identity)`: it's what tells "dead for a
+    # while" apart from "missed a few beats" (Config#orphan_threshold).
+    LAST_BEAT = 'cogworker:last_beat'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'

data/lib/cogworker/reliable_fetch/unsupported_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Cogworker
+  class ReliableFetch
+    # Raised by ReliableFetch#retrieve_work when the server turns out not to
+    # know LMOVE (Redis < 6.2) even though the version check at boot let
+    # `:reliable` through (e.g. Redis wasn't reachable then). The Processor
+    # reacts by switching the whole Manager over to BasicFetch.
+    class UnsupportedError < StandardError; end
+  end
+end

data/lib/cogworker/reliable_fetch.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest/sha1'
+
 module Cogworker
   # The default fetch (`config.fetch = :reliable`): every job is moved
   # atomically (`LMOVE`, Redis >= 6.2) from its queue onto this process's own
@@ -8,8 +10,8 @@ module Cogworker
   # retry/dead. A job is therefore never only in a processor's memory: if the
   # process dies mid-job (OOM, SIGKILL, a host going away), the job is still
   # on that list, and `recover_orphans` (run by every live process's
-  # `Scheduled` poller) puts it back on its queue once the dead process's
-  # heartbeat has expired. The trade-off is at-least-once delivery: a job
+  # `Scheduled` poller) puts it back on its queue once the dead process has
+  # gone `config.orphan_threshold` (default 5 min) without a heartbeat. The trade-off is at-least-once delivery: a job
   # that was partly done when its process died runs again from the start.
   #
   # `LMOVE` can only take from one list at a time, so there's no blocking
@@ -19,7 +21,12 @@ module Cogworker
   # round sleeps EMPTY_POLL_INTERVAL before the next. Weighting and
   # `Queue#pause!` behave exactly as in BasicFetch.
   class ReliableFetch < BasicFetch
-    EMPTY_POLL_INTERVAL = 0.25 # seconds; worst-case pickup delay on an idle process
+    # The first pause after an empty round; each further empty round in a
+    # row doubles it, up to `config.fetch_idle_max_interval` (default 1s),
+    # and finding a job resets it. That cap is the worst-case pickup delay
+    # on an idle process — the trade-off against how often idle processors
+    # poll Redis (at 0.25s flat, 4 calls/s per thread).
+    EMPTY_POLL_INTERVAL = 0.25
     MIN_REDIS_VERSION = Gem::Version.new('6.2')
 
     # KEYS[1..n-1] = queue keys, in the order to try; KEYS[n] = in-progress list.
@@ -34,13 +41,21 @@ module Cogworker
       return false
     LUA
 
+    # The hot path runs this by hash (EVALSHA), not by shipping the whole
+    # script text on every poll; see #run_fetch_script.
+    FETCH_SCRIPT_SHA = Digest::SHA1.hexdigest(FETCH_SCRIPT)
+
     # KEYS[1] = an in-progress list; ARGV[1] = queue key prefix; ARGV[2] =
     # queue for a payload whose own `queue` can't be read (the Processor
-    # then buries it as unparseable). Oldest first, each onto the end its
+    # then buries it as unparseable); ARGV[3] = periodic running-lock key
+    # prefix, ARGV[4] = its queued TTL. Oldest first, each onto the end its
     # queue is popped from, so recovered jobs run next, in their original
     # order. Atomic per call: two processes recovering the same list can't
-    # both move one job.
-    REQUEUE_SCRIPT = <<~LUA
+    # both move one job. A job from an `until_executed` periodic entry
+    # re-takes its running lock on the way (same rule as
+    # Periodic::RunningLock::RETAKE_FUNCTION), so the ticker can't start
+    # the next slot alongside the recovered run.
+    REQUEUE_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
       local moved = 0
       while true do
         local job = redis.call("LPOP", KEYS[1])
@@ -49,8 +64,14 @@ module Cogworker
         end
         local queue = ARGV[2]
         local ok, decoded = pcall(cjson.decode, job)
-        if ok and type(decoded) == "table" and type(decoded["queue"]) == "string" then
-          queue = decoded["queue"]
+        if ok and type(decoded) == "table" then
+          if type(decoded["queue"]) == "string" then
+            queue = decoded["queue"]
+          end
+          local pjid, jid = decoded["periodic_pjid"], decoded["jid"]
+          if decoded["periodic_until_executed"] == true and type(pjid) == "string" and type(jid) == "string" then
+            retake_running_lock(ARGV[3] .. pjid, jid, ARGV[4])
+          end
         end
         redis.call("RPUSH", ARGV[1] .. queue, job)
         moved = moved + 1
@@ -77,28 +98,72 @@ module Cogworker
       # on shutdown (`Manager#stop!`) and for dead processes' lists.
       def requeue_in_progress(identity)
         Cogworker.config.redis do |c|
-          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)], argv: [RedisKeys::QUEUE_PREFIX, 'default'])
+          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)],
+                                 argv: [RedisKeys::QUEUE_PREFIX, 'default', RedisKeys.periodic_running(''),
+                                        Periodic::RunningLock.queued_ttl])
         end
       end
 
-      # Requeues the in-progress list of every process whose heartbeat key
-      # is gone — i.e. that died without a clean shutdown. A process always
+      # Requeues the in-progress list of every process that has gone
+      # `config.orphan_threshold` without a heartbeat — i.e. that died
+      # without a clean shutdown (see #dead?). A process always
       # writes its heartbeat before its first fetch (`Heartbeat#start!`), so
-      # a live process's list is never mistaken for an orphan.
-      def recover_orphans
+      # a live process's list is never mistaken for an orphan. Walks
+      # IN_PROGRESS_IDENTITIES (each reliable-fetch process registers itself
+      # there on every beat) rather than SCANning the keyspace; `scan: true`
+      # (once per process, at boot) also picks up lists from before that
+      # set existed.
+      def recover_orphans(scan: false)
         own = Cogworker.identity
-        keys = Cogworker.config.redis { |c| c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").to_a }
-        keys.sum do |key|
-          identity = key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
-          next 0 if identity == own || Cogworker.config.redis { |c| c.exists?(RedisKeys.process(identity)) }
+        candidate_identities(scan: scan).sum do |identity|
+          next 0 if identity == own || !dead?(identity)
 
           requeue_in_progress(identity).tap do |moved|
+            forget_if_empty(identity)
             next unless moved.positive?
 
             Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" }
           end
         end
       end
+
+      # Its presence key gone (no beat for Heartbeat::TTL) is not enough on
+      # its own: a live process that just couldn't beat for a minute (a
+      # Redis outage or failover, a long GVL-holding call) would lose its
+      # in-progress jobs to a second, parallel run. It also has to have gone
+      # `orphan_threshold` since its last recorded beat — compared on
+      # Redis's clock on both sides, so host clock skew doesn't matter. No
+      # recorded beat at all (a list from before LAST_BEAT existed) falls
+      # back to the presence key alone.
+      def dead?(identity)
+        Cogworker.config.redis do |c|
+          next false if c.exists?(RedisKeys.process(identity))
+
+          last = c.hget(RedisKeys::LAST_BEAT, identity)
+          last.nil? || c.time.first - last.to_f > Cogworker.config.orphan_threshold
+        end
+      end
+
+      def candidate_identities(scan:)
+        Cogworker.config.redis do |c|
+          identities = c.smembers(RedisKeys::IN_PROGRESS_IDENTITIES)
+          next identities unless scan
+
+          scanned = c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").map do |key|
+            key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
+          end
+          identities | scanned
+        end
+      end
+
+      def forget_if_empty(identity)
+        Cogworker.config.redis do |c|
+          next unless c.llen(RedisKeys.in_progress(identity)).zero?
+
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hdel(RedisKeys::LAST_BEAT, identity)
+        end
+      end
     end
 
     def retrieve_work
@@ -108,14 +173,13 @@ module Cogworker
         return nil
       end
 
-      result = Cogworker.config.redis do |c|
-        c.eval(FETCH_SCRIPT, keys: keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
-      end
+      result = run_fetch_script(keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
       unless result
-        sleep(EMPTY_POLL_INTERVAL)
+        idle_sleep
         return nil
       end
 
+      @idle_interval = nil
       queue_key, raw_job = result
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
@@ -133,5 +197,29 @@ module Cogworker
     def acknowledge(work)
       Cogworker.config.redis { |c| c.lrem(RedisKeys.in_progress(Cogworker.identity), 1, work.raw_job) }
     end
+
+    private
+
+    def run_fetch_script(keys)
+      Cogworker.config.redis do |c|
+        c.evalsha(FETCH_SCRIPT_SHA, keys: keys)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+
+        c.eval(FETCH_SCRIPT, keys: keys) # loads it into the script cache for next time
+      end
+    rescue Redis::CommandError => e
+      raise unless e.message.match?(/unknown (redis )?command/i)
+
+      raise UnsupportedError, e.message
+    end
+
+    # Jittered (75–100% of the interval) so a process's threads, all idle
+    # at once, don't keep polling in lockstep.
+    def idle_sleep
+      max = Cogworker.config.fetch_idle_max_interval
+      @idle_interval = @idle_interval ? [@idle_interval * 2, max].min : [EMPTY_POLL_INTERVAL, max].min
+      sleep(@idle_interval * (0.75 + (rand * 0.25)))
+    end
   end
 end

data/lib/cogworker/scheduled.rb

@@ -13,10 +13,28 @@ module Cogworker
     POLL_INTERVAL = 5
     # How often this poller also looks for jobs left in progress by a dead
     # process (ReliableFetch.recover_orphans) — the first time right away,
-    # at boot. Every live process does it; the requeue is atomic, so any
-    # number of them doing it at once is safe.
+    # at boot. Every live process does it, whatever its own fetch mode: in
+    # a mixed fleet, or after a runtime fall-back to `:basic`, the lists of
+    # crashed reliable processes still need someone to requeue them, and
+    # with nothing registered it's a single SMEMBERS. Only the one-off
+    # keyspace SCAN (for lists from before IN_PROGRESS_IDENTITIES existed)
+    # is limited to processes actually using ReliableFetch. The requeue is
+    # atomic, so any number of processes doing it at once is safe.
     ORPHAN_CHECK_INTERVAL = 60
 
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:dead, KEYS[3] = stats:failed;
+    # ARGV[1] = the entry, ARGV[2] = its dead-set score, ARGV[3] = the
+    # wrapper to store. Same claim-and-move-atomically shape as
+    # JobUtil::CLAIM_AND_REQUEUE_SCRIPT.
+    BURY_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
+      redis.call("INCR", KEYS[3])
+      return 1
+    LUA
+
     def initialize(manager)
       @manager = manager
     end
@@ -58,8 +76,9 @@ module Cogworker
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       return if @next_orphan_check && now < @next_orphan_check
 
+      scan = @next_orphan_check.nil? && @manager.fetch_class == ReliableFetch
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
-      ReliableFetch.recover_orphans
+      ReliableFetch.recover_orphans(scan: scan)
     end
 
     def enqueue_due_jobs
@@ -78,16 +97,18 @@ module Cogworker
       Cogworker.config.redis do |c|
         next unless JobUtil.claim_and_requeue(c, set, raw) == :invalid
 
-        bury(c, set, raw) if c.zrem(set, raw)
+        bury(c, set, raw)
       end
     end
 
     def bury(conn, set, raw)
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
-      conn.zadd(RedisKeys::DEAD, job['failed_at'], JSON.generate(job))
-      conn.incr(RedisKeys::STATS_FAILED) # counted like Processor#bury_unparseable
-      Throughput.record('failed')
+      buried = conn.eval(BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
+                                      argv: [raw, job['failed_at'], JSON.generate(job)])
+      return unless buried == 1
+
+      Throughput.record('failed') # counted like Processor#bury_unparseable
       Cogworker.logger.error { "unparseable entry in #{set} moved to dead: #{raw.to_s[0, 200]}" }
     end
   end

data/lib/cogworker/swarm.rb

@@ -69,6 +69,7 @@ module Cogworker
         # indefinitely, so this OS process would never actually go away —
         # and `Swarm#initiate_restart`'s own `Process.waitpid(pid)` for this
         # exact child would then block forever right along with it.
+        Cogworker.flush_output!
         ::Process.exit!(true)
       end
       @children[pid] = slot

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/cogworker/web/optional_session.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'rack/session/cookie'
+
+module Cogworker
+  class Web
+    # The Web UI's own session cookie, applied only when nothing upstream
+    # already provides `env['rack.session']`. Mounted inside an app that has
+    # its own session (Rails, or any `Rack::Session::*` in front), an
+    # unconditional `Rack::Session::Cookie` here replaced that session with
+    # a copy for everything inside the Web UI — so an auth middleware added
+    # via `Web.use` read and wrote the copy, never the host's real session,
+    # and every response carried a second `cogworker.session` cookie.
+    class OptionalSession
+      def initialize(app, **options)
+        @app = app
+        @with_session = Rack::Session::Cookie.new(app, **options)
+      end
+
+      def call(env)
+        env['rack.session'] ? @app.call(env) : @with_session.call(env)
+      end
+    end
+  end
+end

data/lib/cogworker/web/routes/jobs.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'cgi'
+require 'digest/sha1'
 require 'json'
 
 module Cogworker
@@ -63,33 +64,30 @@ module Cogworker
           app.post('/jobs/retrying/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::RETRY, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
           # Same "graduate back onto its queue" move `/jobs/dead/retry` below/
-          # `Cogworker::Scheduled#graduate` make — `zrem` winning (not
-          # losing) gates it so two tabs retrying the same entry at once
-          # can't both requeue it; the `redis` gem's `#zrem` returns a
-          # **Boolean** for a single member, so this checks truthiness, not
-          # `== 1` (`true == 1` is `false` in Ruby).
+          # `Cogworker::Scheduled#graduate` make (`JobUtil.claim_and_requeue`:
+          # validate, then claim + push atomically) — two tabs retrying the
+          # same entry at once can't both requeue it, and an entry that
+          # can't be requeued is left in place with a notice.
           app.post('/jobs/retrying/retry_now') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw)
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           app.post('/jobs/dead/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::DEAD, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
           app.post('/jobs/dead/delete_all') do
-            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| JSON.parse(raw)['jid'] }
+            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| Jobs.parse_entry(raw)['jid'] }
             Cogworker.config.redis { |c| c.del(RedisKeys::DEAD) }
             jids.each { |jid| Cogworker::Attempts.clear(jid) }
             Jobs.respond(self, params)
@@ -97,10 +95,8 @@ module Cogworker
 
           app.post('/jobs/dead/retry') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw)
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           # Moves this one entry's score on its own ZSET — "run sooner" or
@@ -144,11 +140,16 @@ module Cogworker
         # #jobs-content in place, keeping the same filter/search/selection;
         # a plain form submission (no JS) falls back to a normal redirect
         # to that same URL.
-        def respond(action, params)
+        # `notice:` is a key into NOTICES (never free text — it's echoed into
+        # the page). It shows once: the redirect carries it as `?notice=`,
+        # but `query_string` (and so every poll URL) leaves it out.
+        def respond(action, params, notice: nil)
           if action.hx_request?
-            render_content(action.request.script_name, params)
+            render_content(action.request.script_name, params.merge('notice' => notice))
           else
-            action.redirect(Layout.path(action.request.script_name, "jobs#{query_string(params)}"))
+            location = "jobs#{query_string(params)}"
+            location += "#{location.include?('?') ? '&' : '?'}notice=#{CGI.escape(notice)}" if notice
+            action.redirect(Layout.path(action.request.script_name, location))
           end
         end
 
@@ -163,6 +164,19 @@ module Cogworker
           parts.empty? ? '' : "?#{parts.join('&')}"
         end
 
+        NOTICES = {
+          'not_requeueable' => "This entry can't be put back on a queue — it's unreadable or has no queue " \
+                               'of its own. Delete it instead.'
+        }.freeze
+
+        def notice_banner(params)
+          text = NOTICES[params['notice'].to_s]
+          return '' unless text
+
+          style = 'padding: 10px 14px; border-left: 3px solid var(--color-danger);'
+          %(<div class="card" role="status" style="#{style}">#{Layout.h(text)}</div>)
+        end
+
         def render_content(script_name, params)
           status = STATUSES.include?(params['status']) ? params['status'] : 'All'
           query = params['q'].to_s
@@ -175,6 +189,7 @@ module Cogworker
 
           <<~HTML
             <div style="display: flex; flex-direction: column; gap: 16px;">
+              #{notice_banner(params)}
               #{page_header(script_name, status, query, filtered.size, rows.size)}
               <div style="display: grid; grid-template-columns: minmax(0, 1fr) #{selected ? 'minmax(320px, 400px)' : '0px'}; gap: 16px; align-items: start;">
                 #{jobs_table(filtered, script_name, params)}
@@ -259,8 +274,10 @@ module Cogworker
         # No Retry for an entry that can't go back on a queue (no queue of
         # its own) or that would only fail straight back into Dead (an
         # unparseable payload's wrapper) — Delete is all that makes sense.
+        # From the row's already-parsed fields — this runs for every row on
+        # every poll, so no re-parsing `raw` here.
         def requeueable?(row)
-          row[:klass] != JobUtil::UNPARSEABLE_CLASS && !JobUtil.requeueable(row[:raw].to_s).nil?
+          row[:klass] != JobUtil::UNPARSEABLE_CLASS && row[:queue].is_a?(String) && !row[:queue].empty?
         end
 
         def delete_button(bucket, script_name, row, params, extra: {})
@@ -430,10 +447,28 @@ module Cogworker
           end
         end
 
+        # A ZSET entry that isn't a job Hash (corrupt, or written by
+        # something else) shows up as an `(unparseable)` row — Delete is
+        # offered, Retry isn't (`requeueable?`) — instead of a JSON::ParserError
+        # taking the whole tab down.
+        def parse_entry(raw)
+          job = JSON.parse(raw)
+          return job if job.is_a?(Hash)
+
+          unparseable_entry(raw)
+        rescue JSON::ParserError
+          unparseable_entry(raw)
+        end
+
+        def unparseable_entry(raw)
+          { 'class' => JobUtil::UNPARSEABLE_CLASS, 'jid' => "unparseable-#{Digest::SHA1.hexdigest(raw.to_s)[0, 12]}",
+            'args' => [], 'error_message' => raw.to_s[0, 200] }
+        end
+
         def scheduled_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::SCHEDULE, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Scheduled', attempt: '—',
               next_action: score, at: job['created_at'], args: job['args'], error_class: nil, error_message: nil,
               raw: raw, source: :scheduled }
@@ -443,7 +478,7 @@ module Cogworker
         def retrying_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Retrying',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: score,
               at: job['failed_at'] || job['created_at'], args: job['args'], error_class: job['error_class'],
@@ -454,7 +489,7 @@ module Cogworker
         def dead_rows
           entries = Cogworker.config.redis { |c| c.zrevrange(RedisKeys::DEAD, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Dead',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: nil, at: score,
               args: job['args'], error_class: job['error_class'], error_message: job['error_message'], raw: raw,

data/lib/cogworker/web.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'rack'
-require 'rack/session/cookie'
 require 'rack/static'
 require 'securerandom'
 
@@ -105,6 +104,18 @@ module Cogworker
         @session_secret ||= ENV.fetch('COGWORKER_SESSION_SECRET') { SecureRandom.hex(32) }
       end
 
+      # `false` drops the built-in session cookie entirely (it's already
+      # skipped automatically whenever the host app provides a session —
+      # see OptionalSession). Default true.
+      def builtin_session
+        @builtin_session.nil? || @builtin_session
+      end
+
+      def builtin_session=(enabled)
+        @builtin_session = enabled
+        @app = nil
+      end
+
       def session_secret=(secret)
         @session_secret = secret
         @app = nil
@@ -247,7 +258,9 @@ module Cogworker
 
       def build_app
         builder = Rack::Builder.new
-        builder.use(Rack::Session::Cookie, secret: session_secret, key: 'cogworker.session')
+        if builtin_session
+          builder.use(Cogworker::Web::OptionalSession, secret: session_secret, key: 'cogworker.session')
+        end
         # No `cache_control:` (no `immutable`/long `max-age`): these files
         # are plain, unfingerprinted paths that *do* change — every time
         # a vendored stylesheet/script gets edited, or on any gem upgrade —

data/lib/cogworker.rb

@@ -83,6 +83,17 @@ module Cogworker
       !!@server_process
     end
 
+    # Called right before every `Process.exit!` (which, unlike `exit`,
+    # flushes nothing): a custom `Cogworker.logger` writing to a buffered
+    # stdout/stderr would otherwise lose its last lines.
+    def flush_output!
+      [$stdout, $stderr].each do |io|
+        io.flush
+      rescue StandardError
+        nil
+      end
+    end
+
     def server_process!
       @server_process = true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - vdonec
@@ -239,6 +239,7 @@ files:
 - lib/cogworker/redis_connection.rb
 - lib/cogworker/redis_keys.rb
 - lib/cogworker/reliable_fetch.rb
+- lib/cogworker/reliable_fetch/unsupported_error.rb
 - lib/cogworker/scheduled.rb
 - lib/cogworker/signals.rb
 - lib/cogworker/stats.rb
@@ -269,6 +270,7 @@ files:
 - lib/cogworker/web/assets/phosphor/style.css
 - lib/cogworker/web/history_query.rb
 - lib/cogworker/web/layout.rb
+- lib/cogworker/web/optional_session.rb
 - lib/cogworker/web/router.rb
 - lib/cogworker/web/routes/history.rb
 - lib/cogworker/web/routes/jobs.rb