cogworker 0.3.0 → 0.4.1

data/lib/cogworker/reliable_fetch.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'digest/sha1'
+require 'json'
+
 module Cogworker
   # The default fetch (`config.fetch = :reliable`): every job is moved
   # atomically (`LMOVE`, Redis >= 6.2) from its queue onto this process's own
@@ -8,8 +11,8 @@ module Cogworker
   # retry/dead. A job is therefore never only in a processor's memory: if the
   # process dies mid-job (OOM, SIGKILL, a host going away), the job is still
   # on that list, and `recover_orphans` (run by every live process's
-  # `Scheduled` poller) puts it back on its queue once the dead process's
-  # heartbeat has expired. The trade-off is at-least-once delivery: a job
+  # `Scheduled` poller) puts it back on its queue once the dead process has
+  # gone `config.orphan_threshold` (default 5 min) without a heartbeat. The trade-off is at-least-once delivery: a job
   # that was partly done when its process died runs again from the start.
   #
   # `LMOVE` can only take from one list at a time, so there's no blocking
@@ -19,7 +22,12 @@ module Cogworker
   # round sleeps EMPTY_POLL_INTERVAL before the next. Weighting and
   # `Queue#pause!` behave exactly as in BasicFetch.
   class ReliableFetch < BasicFetch
-    EMPTY_POLL_INTERVAL = 0.25 # seconds; worst-case pickup delay on an idle process
+    # The first pause after an empty round; each further empty round in a
+    # row doubles it, up to `config.fetch_idle_max_interval` (default 1s),
+    # and finding a job resets it. That cap is the worst-case pickup delay
+    # on an idle process — the trade-off against how often idle processors
+    # poll Redis (at 0.25s flat, 4 calls/s per thread).
+    EMPTY_POLL_INTERVAL = 0.25
     MIN_REDIS_VERSION = Gem::Version.new('6.2')
 
     # KEYS[1..n-1] = queue keys, in the order to try; KEYS[n] = in-progress list.
@@ -34,69 +42,199 @@ module Cogworker
       return false
     LUA
 
-    # KEYS[1] = an in-progress list; ARGV[1] = queue key prefix; ARGV[2] =
-    # queue for a payload whose own `queue` can't be read (the Processor
-    # then buries it as unparseable). Oldest first, each onto the end its
-    # queue is popped from, so recovered jobs run next, in their original
-    # order. Atomic per call: two processes recovering the same list can't
-    # both move one job.
-    REQUEUE_SCRIPT = <<~LUA
+    # (Run by hash, like every script here — see LuaScript.)
+    FETCH_SCRIPT_SHA = LuaScript.sha(FETCH_SCRIPT)
+
+    # Lua shared by every script here that puts a job back on its queue:
+    # onto the end its queue is popped from (so it runs next), onto
+    # `default_queue` if its own `queue` can't be read (the Processor then
+    # buries it as unparseable) — and a job from an `until_executed`
+    # periodic entry re-takes its running lock on the way
+    # (Periodic::RunningLock::RETAKE_FUNCTION), so the ticker can't start
+    # the next slot alongside the requeued run.
+    REQUEUE_JOB_FUNCTION = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      local function requeue_job(job, queue_prefix, default_queue, lock_prefix, lock_ttl)
+        local queue = default_queue
+        local ok, decoded = pcall(cjson.decode, job)
+        if ok and type(decoded) == "table" then
+          if type(decoded["queue"]) == "string" and decoded["queue"] ~= "" then
+            queue = decoded["queue"]
+          end
+          local pjid, jid = decoded["periodic_pjid"], decoded["jid"]
+          if decoded["periodic_until_executed"] == true and type(pjid) == "string" and type(jid) == "string" then
+            retake_running_lock(lock_prefix .. pjid, jid, lock_ttl)
+          end
+        end
+        redis.call("RPUSH", queue_prefix .. queue, job)
+      end
+    LUA
+
+    # KEYS[1] = an in-progress list, KEYS[2] = its owner's presence key,
+    # KEYS[3] = LAST_BEAT; ARGV[1..4] = requeue_job's prefix/default
+    # queue/lock prefix/lock TTL, ARGV[5] = owner identity, ARGV[6] = orphan
+    # threshold, or "" to requeue unconditionally (this process's own list,
+    # on shutdown). Returns how many it moved, or -1 if the owner turned
+    # out to be alive. Oldest first, so recovered jobs keep their order. The
+    # "is the owner really dead?" check (see ReliableFetch.dead?) is made
+    # here, atomically with the requeue: checked separately, an owner that
+    # beat in between would lose jobs it is still running to a second run.
+    # Atomic per call, so two processes recovering one list can't both move
+    # a job.
+    REQUEUE_SCRIPT = REQUEUE_JOB_FUNCTION + <<~LUA
+      if ARGV[6] ~= "" then
+        if redis.call("EXISTS", KEYS[2]) == 1 then
+          return -1
+        end
+        local last = redis.call("HGET", KEYS[3], ARGV[5])
+        if last then
+          if redis.replicate_commands then redis.replicate_commands() end
+          local now = tonumber(redis.call("TIME")[1])
+          if now - tonumber(last) <= tonumber(ARGV[6]) then
+            return -1
+          end
+        end
+      end
       local moved = 0
       while true do
         local job = redis.call("LPOP", KEYS[1])
         if not job then
           return moved
         end
-        local queue = ARGV[2]
-        local ok, decoded = pcall(cjson.decode, job)
-        if ok and type(decoded) == "table" and type(decoded["queue"]) == "string" then
-          queue = decoded["queue"]
-        end
-        redis.call("RPUSH", ARGV[1] .. queue, job)
+        requeue_job(job, ARGV[1], ARGV[2], ARGV[3], ARGV[4])
         moved = moved + 1
       end
     LUA
 
-    # KEYS[1] = in-progress list, KEYS[2] = the job's queue; ARGV[1] = job.
-    GIVE_BACK_SCRIPT = <<~LUA
+    # KEYS[1] = an in-progress list; ARGV[1] = one job on it, ARGV[2..5] =
+    # requeue_job's arguments. That one job, if still on the list, back on
+    # its queue: `give_back`, and `reconcile`'s strays.
+    REQUEUE_ONE_SCRIPT = REQUEUE_JOB_FUNCTION + <<~LUA
       if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
-        redis.call("RPUSH", KEYS[2], ARGV[1])
+        requeue_job(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
+        return 1
       end
+      return 0
+    LUA
+
+    # KEYS[1] = in-progress list, KEYS[2] = retry/dead ZSET; ARGV[1] = the
+    # job as fetched, ARGV[2] = score, ARGV[3] = payload to file. Off the
+    # in-progress list and into the ZSET in one step; a no-op if it's no
+    # longer in progress (already requeued by a shutdown).
+    INTERRUPT_SCRIPT = <<~LUA
+      if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
+        redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
+        return 1
+      end
+      return 0
     LUA
 
     class << self
+      # A connection error propagates: can't tell yet, so the caller
+      # (Manager#fetch_class) resolves again on its next use, instead of
+      # memoizing a guess made while Redis was unreachable. Anything else
+      # (INFO restricted, an odd version string) assumes a current server —
+      # the runtime fall-back (UnsupportedError) still catches a wrong guess.
       def supported?
         version = Cogworker.config.redis { |c| c.info('server')['redis_version'] }
         Gem::Version.new(version) >= MIN_REDIS_VERSION
+      rescue Redis::BaseConnectionError
+        raise
       rescue StandardError
-        true # can't tell (Redis not reachable yet): assume a current server
+        true
       end
 
       # Puts every job still on `identity`'s in-progress list back on its
-      # queue; returns how many. Used for this process's own unfinished jobs
-      # on shutdown (`Manager#stop!`) and for dead processes' lists.
-      def requeue_in_progress(identity)
+      # queue; returns how many — or -1 if `if_dead_for:` is given and the
+      # owner turns out not to be dead (see REQUEUE_SCRIPT). Used without
+      # it for this process's own unfinished jobs on shutdown
+      # (`Manager#stop!`), with it for presumed-dead processes' lists.
+      def requeue_in_progress(identity, if_dead_for: nil)
         Cogworker.config.redis do |c|
-          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)], argv: [RedisKeys::QUEUE_PREFIX, 'default'])
+          LuaScript.run(c, REQUEUE_SCRIPT,
+                 keys: [RedisKeys.in_progress(identity), RedisKeys.process(identity), RedisKeys::LAST_BEAT],
+                 argv: [*requeue_args, identity, if_dead_for.to_s])
         end
       end
 
-      # Requeues the in-progress list of every process whose heartbeat key
-      # is gone — i.e. that died without a clean shutdown. A process always
-      # writes its heartbeat before its first fetch (`Heartbeat#start!`), so
-      # a live process's list is never mistaken for an orphan.
-      def recover_orphans
+      def requeue_args
+        [RedisKeys::QUEUE_PREFIX, 'default', RedisKeys.periodic_running(''), Periodic::RunningLock.queued_ttl]
+      end
+
+      # Requeues the in-progress list of every process that has died without
+      # a clean shutdown. "Dead" means its presence key is gone (no beat for
+      # Heartbeat::TTL) *and* `config.orphan_threshold` has passed since its
+      # last recorded beat (LAST_BEAT, on Redis's clock) — the presence key
+      # alone expires after a minute, too eagerly for a live process that
+      # merely couldn't beat for a while (a Redis outage or failover, a long
+      # GVL-holding call), whose running jobs would then run twice. With no
+      # recorded beat at all (lists from before LAST_BEAT existed), the
+      # presence key alone decides. Checked inside REQUEUE_SCRIPT, atomically
+      # with the requeue. A process always beats before its first fetch
+      # (`Heartbeat#start!`). Walks IN_PROGRESS_IDENTITIES (each
+      # reliable-fetch process registers itself there on every beat) rather
+      # than SCANning the keyspace; `scan: true` (once per reliable process,
+      # at boot) also picks up lists from before that set existed.
+      def recover_orphans(scan: false)
         own = Cogworker.identity
-        keys = Cogworker.config.redis { |c| c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").to_a }
-        keys.sum do |key|
-          identity = key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
-          next 0 if identity == own || Cogworker.config.redis { |c| c.exists?(RedisKeys.process(identity)) }
+        candidate_identities(scan: scan).sum do |identity|
+          next 0 if identity == own
 
-          requeue_in_progress(identity).tap do |moved|
-            next unless moved.positive?
+          moved = requeue_in_progress(identity, if_dead_for: Cogworker.config.orphan_threshold)
+          next 0 if moved.negative? # alive after all: keep its registration and last beat
 
-            Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" }
+          forget_if_empty(identity)
+          Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" } if moved.positive?
+          moved
+        end
+      end
+
+      # This process's own in-progress entries that no processor thread is
+      # running (not in `cogworker:workers:<identity>`) — left behind when
+      # neither acknowledging nor handing a job back worked. Requeued once
+      # they've been seen stray on two calls in a row (the caller passes
+      # back what the previous call returned), which rules out a job caught
+      # in the instant between being fetched and registered. Returns this
+      # call's strays that weren't requeued yet.
+      def reconcile(identity, previous_strays = [])
+        Cogworker.config.redis do |c|
+          listed = c.lrange(RedisKeys.in_progress(identity), 0, -1)
+          running = c.hvals(RedisKeys.workers(identity)).filter_map { |raw| parse_hash(raw)&.dig('payload', 'jid') }
+          strays = listed.reject { |raw| running.include?(parse_hash(raw)&.fetch('jid', nil)) }
+          (strays & previous_strays).each do |raw|
+            next unless LuaScript.run(c, REQUEUE_ONE_SCRIPT, keys: [RedisKeys.in_progress(identity)],
+                                                            argv: [raw, *requeue_args]) == 1
+
+            Cogworker.logger.warn { "requeued stray in-progress job jid=#{parse_hash(raw)&.fetch('jid', nil)}" }
           end
+          strays - previous_strays
+        end
+      end
+
+      def parse_hash(raw)
+        value = JSON.parse(raw)
+        value.is_a?(Hash) ? value : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      def candidate_identities(scan:)
+        Cogworker.config.redis do |c|
+          identities = c.smembers(RedisKeys::IN_PROGRESS_IDENTITIES)
+          next identities unless scan
+
+          scanned = c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").map do |key|
+            key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
+          end
+          identities | scanned
+        end
+      end
+
+      def forget_if_empty(identity)
+        Cogworker.config.redis do |c|
+          next unless c.llen(RedisKeys.in_progress(identity)).zero?
+
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hdel(RedisKeys::LAST_BEAT, identity)
         end
       end
     end
@@ -104,18 +242,20 @@ module Cogworker
     def retrieve_work
       keys = active_keys
       if keys.empty?
-        sleep(TIMEOUT)
+        interruptible_sleep(TIMEOUT)
         return nil
       end
 
-      result = Cogworker.config.redis do |c|
-        c.eval(FETCH_SCRIPT, keys: keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
-      end
+      distinct = keys.uniq
+      return blocking_fetch(distinct.first) if distinct.size == 1
+
+      result = run_fetch_script(keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
       unless result
-        sleep(EMPTY_POLL_INTERVAL)
+        idle_sleep
         return nil
       end
 
+      @idle_interval = nil
       queue_key, raw_job = result
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
@@ -125,13 +265,52 @@ module Cogworker
     # no longer in progress (already requeued by `requeue_in_progress`).
     def give_back(work)
       Cogworker.config.redis do |c|
-        c.eval(GIVE_BACK_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), RedisKeys.queue(work.queue)],
-                                 argv: [work.raw_job])
+        LuaScript.run(c, REQUEUE_ONE_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity)],
+                                            argv: [work.raw_job, *self.class.requeue_args])
+      end
+    end
+
+    def interrupt(work, set, score, payload)
+      Cogworker.config.redis do |c|
+        LuaScript.run(c, INTERRUPT_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), set],
+                                          argv: [work.raw_job, score, payload])
       end
     end
 
     def acknowledge(work)
       Cogworker.config.redis { |c| c.lrem(RedisKeys.in_progress(Cogworker.identity), 1, work.raw_job) }
     end
+
+    private
+
+    # With a single (unpaused) queue there's no weighting to honor, so no
+    # need to poll: a blocking BLMOVE waits on the server and returns the
+    # moment a job arrives, for up to TIMEOUT — same as BasicFetch's BRPOP.
+    def blocking_fetch(queue_key)
+      raw_job = Cogworker.config.redis do |c|
+        c.blmove(queue_key, RedisKeys.in_progress(Cogworker.identity), 'RIGHT', 'LEFT', timeout: TIMEOUT)
+      end
+      raw_job && UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
+    rescue Redis::CommandError => e
+      raise unless e.message.match?(/unknown (redis )?command/i)
+
+      raise UnsupportedError, e.message
+    end
+
+    def run_fetch_script(keys)
+      Cogworker.config.redis { |c| LuaScript.run(c, FETCH_SCRIPT, keys: keys) }
+    rescue Redis::CommandError => e
+      raise unless e.message.match?(/unknown (redis )?command/i)
+
+      raise UnsupportedError, e.message
+    end
+
+    # Jittered (75–100% of the interval) so a process's threads, all idle
+    # at once, don't keep polling in lockstep.
+    def idle_sleep
+      max = Cogworker.config.fetch_idle_max_interval
+      @idle_interval = @idle_interval ? [@idle_interval * 2, max].min : [EMPTY_POLL_INTERVAL, max].min
+      interruptible_sleep(@idle_interval * (0.75 + (rand * 0.25)))
+    end
   end
 end

data/lib/cogworker/scheduled.rb

@@ -13,10 +13,28 @@ module Cogworker
     POLL_INTERVAL = 5
     # How often this poller also looks for jobs left in progress by a dead
     # process (ReliableFetch.recover_orphans) — the first time right away,
-    # at boot. Every live process does it; the requeue is atomic, so any
-    # number of them doing it at once is safe.
+    # at boot. Every live process does it, whatever its own fetch mode: in
+    # a mixed fleet, or after a runtime fall-back to `:basic`, the lists of
+    # crashed reliable processes still need someone to requeue them, and
+    # with nothing registered it's a single SMEMBERS. Only the one-off
+    # keyspace SCAN (for lists from before IN_PROGRESS_IDENTITIES existed)
+    # is limited to processes actually using ReliableFetch. The requeue is
+    # atomic, so any number of processes doing it at once is safe.
     ORPHAN_CHECK_INTERVAL = 60
 
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:dead, KEYS[3] = stats:failed;
+    # ARGV[1] = the entry, ARGV[2] = its dead-set score, ARGV[3] = the
+    # wrapper to store. Same claim-and-move-atomically shape as
+    # JobUtil::CLAIM_AND_REQUEUE_SCRIPT.
+    BURY_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
+      redis.call("INCR", KEYS[3])
+      return 1
+    LUA
+
     def initialize(manager)
       @manager = manager
     end
@@ -37,6 +55,12 @@ module Cogworker
       @thread&.kill
     end
 
+    # True once the first orphan check has completed — what the periodic
+    # Ticker waits for before its first tick (see Launcher#initialize).
+    def recovered_once?
+      @recovered_once == true
+    end
+
     private
 
     # A failed poll is logged and retried next interval rather than ending
@@ -54,12 +78,17 @@ module Cogworker
       end
     end
 
+    # Next check scheduled only once this one has completed: a failed one
+    # (Redis down) is retried on the next poll rather than a minute later.
     def recover_orphans_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       return if @next_orphan_check && now < @next_orphan_check
 
+      reliable = @manager.fetch_class == ReliableFetch
+      ReliableFetch.recover_orphans(scan: !@recovered_once && reliable)
+      @strays = ReliableFetch.reconcile(Cogworker.identity, @strays || []) if reliable
+      @recovered_once = true
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
-      ReliableFetch.recover_orphans
     end
 
     def enqueue_due_jobs
@@ -78,16 +107,18 @@ module Cogworker
       Cogworker.config.redis do |c|
         next unless JobUtil.claim_and_requeue(c, set, raw) == :invalid
 
-        bury(c, set, raw) if c.zrem(set, raw)
+        bury(c, set, raw)
       end
     end
 
     def bury(conn, set, raw)
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
-      conn.zadd(RedisKeys::DEAD, job['failed_at'], JSON.generate(job))
-      conn.incr(RedisKeys::STATS_FAILED) # counted like Processor#bury_unparseable
-      Throughput.record('failed')
+      buried = LuaScript.run(conn, BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
+                                               argv: [raw, job['failed_at'], JSON.generate(job)])
+      return unless buried == 1
+
+      Throughput.record('failed') # counted like Processor#bury_unparseable
       Cogworker.logger.error { "unparseable entry in #{set} moved to dead: #{raw.to_s[0, 200]}" }
     end
   end

data/lib/cogworker/status/server_middleware.rb

@@ -15,7 +15,7 @@ module Cogworker
         write(job, queue, 'complete')
       rescue Exception => e # rubocop:disable Lint/RescueException
         write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
-              error_class: e.class.name, error_message: e.message.to_s[0, 10_000])
+              error_class: e.class.name, error_message: JobUtil.error_message(e))
         raise e
       end
 

data/lib/cogworker/swarm.rb

@@ -69,6 +69,7 @@ module Cogworker
         # indefinitely, so this OS process would never actually go away —
         # and `Swarm#initiate_restart`'s own `Process.waitpid(pid)` for this
         # exact child would then block forever right along with it.
+        Cogworker.flush_output!
         ::Process.exit!(true)
       end
       @children[pid] = slot

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.3.0'
+  VERSION = '0.4.1'
 end

data/lib/cogworker/web/optional_session.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'rack/session/cookie'
+
+module Cogworker
+  class Web
+    # The Web UI's own session cookie, applied only when nothing upstream
+    # already provides `env['rack.session']`. Mounted inside an app that has
+    # its own session (Rails, or any `Rack::Session::*` in front), an
+    # unconditional `Rack::Session::Cookie` here replaced that session with
+    # a copy for everything inside the Web UI — so an auth middleware added
+    # via `Web.use` read and wrote the copy, never the host's real session,
+    # and every response carried a second `cogworker.session` cookie.
+    class OptionalSession
+      def initialize(app, **options)
+        @app = app
+        @with_session = Rack::Session::Cookie.new(app, **options)
+      end
+
+      def call(env)
+        env['rack.session'] ? @app.call(env) : @with_session.call(env)
+      end
+
+      # A session middleware added via `Web.use` sits *after* this one in
+      # the stack, so it can't be detected per request the way an upstream
+      # one is — `Web.build_app` checks for one up front instead.
+      def self.session_middleware?(middleware)
+        middleware.is_a?(Class) && middleware <= Rack::Session::Abstract::Persisted
+      end
+    end
+  end
+end

data/lib/cogworker/web/routes/jobs.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'cgi'
+require 'digest/sha1'
 require 'json'
 
 module Cogworker
@@ -63,33 +64,30 @@ module Cogworker
           app.post('/jobs/retrying/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::RETRY, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
           # Same "graduate back onto its queue" move `/jobs/dead/retry` below/
-          # `Cogworker::Scheduled#graduate` make — `zrem` winning (not
-          # losing) gates it so two tabs retrying the same entry at once
-          # can't both requeue it; the `redis` gem's `#zrem` returns a
-          # **Boolean** for a single member, so this checks truthiness, not
-          # `== 1` (`true == 1` is `false` in Ruby).
+          # `Cogworker::Scheduled#graduate` make (`JobUtil.claim_and_requeue`:
+          # validate, then claim + push atomically) — two tabs retrying the
+          # same entry at once can't both requeue it, and an entry that
+          # can't be requeued is left in place with a notice.
           app.post('/jobs/retrying/retry_now') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw)
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           app.post('/jobs/dead/delete') do
             raw = params['raw']
             Cogworker.config.redis { |c| c.zrem(RedisKeys::DEAD, raw) }
-            Cogworker::Attempts.clear(JSON.parse(raw)['jid'])
+            Cogworker::Attempts.clear(Jobs.parse_entry(raw)['jid'])
             Jobs.respond(self, params)
           end
 
           app.post('/jobs/dead/delete_all') do
-            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| JSON.parse(raw)['jid'] }
+            jids = Cogworker.config.redis { |c| c.zrange(RedisKeys::DEAD, 0, -1) }.map { |raw| Jobs.parse_entry(raw)['jid'] }
             Cogworker.config.redis { |c| c.del(RedisKeys::DEAD) }
             jids.each { |jid| Cogworker::Attempts.clear(jid) }
             Jobs.respond(self, params)
@@ -97,10 +95,8 @@ module Cogworker
 
           app.post('/jobs/dead/retry') do
             raw = params['raw']
-            Cogworker.config.redis do |c|
-              JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw)
-            end
-            Jobs.respond(self, params)
+            result = Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw) }
+            Jobs.respond(self, params, notice: result == :invalid ? 'not_requeueable' : nil)
           end
 
           # Moves this one entry's score on its own ZSET — "run sooner" or
@@ -144,11 +140,16 @@ module Cogworker
         # #jobs-content in place, keeping the same filter/search/selection;
         # a plain form submission (no JS) falls back to a normal redirect
         # to that same URL.
-        def respond(action, params)
+        # `notice:` is a key into NOTICES (never free text — it's echoed into
+        # the page). It shows once: the redirect carries it as `?notice=`,
+        # but `query_string` (and so every poll URL) leaves it out.
+        def respond(action, params, notice: nil)
           if action.hx_request?
-            render_content(action.request.script_name, params)
+            render_content(action.request.script_name, params.merge('notice' => notice))
           else
-            action.redirect(Layout.path(action.request.script_name, "jobs#{query_string(params)}"))
+            location = "jobs#{query_string(params)}"
+            location += "#{location.include?('?') ? '&' : '?'}notice=#{CGI.escape(notice)}" if notice
+            action.redirect(Layout.path(action.request.script_name, location))
           end
         end
 
@@ -163,6 +164,19 @@ module Cogworker
           parts.empty? ? '' : "?#{parts.join('&')}"
         end
 
+        NOTICES = {
+          'not_requeueable' => "This entry can't be put back on a queue — it's unreadable or has no queue " \
+                               'of its own. Delete it instead.'
+        }.freeze
+
+        def notice_banner(params)
+          text = NOTICES[params['notice'].to_s]
+          return '' unless text
+
+          style = 'padding: 10px 14px; border-left: 3px solid var(--color-danger);'
+          %(<div class="card" role="status" style="#{style}">#{Layout.h(text)}</div>)
+        end
+
         def render_content(script_name, params)
           status = STATUSES.include?(params['status']) ? params['status'] : 'All'
           query = params['q'].to_s
@@ -175,6 +189,7 @@ module Cogworker
 
           <<~HTML
             <div style="display: flex; flex-direction: column; gap: 16px;">
+              #{notice_banner(params)}
               #{page_header(script_name, status, query, filtered.size, rows.size)}
               <div style="display: grid; grid-template-columns: minmax(0, 1fr) #{selected ? 'minmax(320px, 400px)' : '0px'}; gap: 16px; align-items: start;">
                 #{jobs_table(filtered, script_name, params)}
@@ -259,8 +274,10 @@ module Cogworker
         # No Retry for an entry that can't go back on a queue (no queue of
         # its own) or that would only fail straight back into Dead (an
         # unparseable payload's wrapper) — Delete is all that makes sense.
+        # From the row's already-parsed fields — this runs for every row on
+        # every poll, so no re-parsing `raw` here.
         def requeueable?(row)
-          row[:klass] != JobUtil::UNPARSEABLE_CLASS && !JobUtil.requeueable(row[:raw].to_s).nil?
+          row[:klass] != JobUtil::UNPARSEABLE_CLASS && row[:queue].is_a?(String) && !row[:queue].empty?
         end
 
         def delete_button(bucket, script_name, row, params, extra: {})
@@ -430,10 +447,28 @@ module Cogworker
           end
         end
 
+        # A ZSET entry that isn't a job Hash (corrupt, or written by
+        # something else) shows up as an `(unparseable)` row — Delete is
+        # offered, Retry isn't (`requeueable?`) — instead of a JSON::ParserError
+        # taking the whole tab down.
+        def parse_entry(raw)
+          job = JSON.parse(raw)
+          return job if job.is_a?(Hash)
+
+          unparseable_entry(raw)
+        rescue JSON::ParserError
+          unparseable_entry(raw)
+        end
+
+        def unparseable_entry(raw)
+          { 'class' => JobUtil::UNPARSEABLE_CLASS, 'jid' => "unparseable-#{Digest::SHA1.hexdigest(raw.to_s)[0, 12]}",
+            'args' => [], 'error_message' => raw.to_s[0, 200] }
+        end
+
         def scheduled_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::SCHEDULE, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Scheduled', attempt: '—',
               next_action: score, at: job['created_at'], args: job['args'], error_class: nil, error_message: nil,
               raw: raw, source: :scheduled }
@@ -443,7 +478,7 @@ module Cogworker
         def retrying_rows
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Retrying',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: score,
               at: job['failed_at'] || job['created_at'], args: job['args'], error_class: job['error_class'],
@@ -454,7 +489,7 @@ module Cogworker
         def dead_rows
           entries = Cogworker.config.redis { |c| c.zrevrange(RedisKeys::DEAD, 0, -1, withscores: true) }
           entries.map do |raw, score|
-            job = JSON.parse(raw)
+            job = parse_entry(raw)
             { jid: job['jid'], klass: job['class'], queue: job['queue'], status: 'Dead',
               attempt: "#{job['retry_count'].to_i} of #{JobUtil.max_retries(job)}", next_action: nil, at: score,
               args: job['args'], error_class: job['error_class'], error_message: job['error_message'], raw: raw,