cogworker 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfb90fe3a477f31658c58fd09a41e4404a0c09999b74614789bd343ee704fad2
-  data.tar.gz: 407aa20293538fa35c8219f88f21fd8b1c15150d96e77d2b495d017ef21f924e
+  metadata.gz: e908112cee4c6a9f98588b6cca08c292437d1713ec351ac6287e1f5ef605c452
+  data.tar.gz: 0c271ea997d7ddf51df5aa5a16b45cb08c0b3231f71dcdb253956160c589bde3
 SHA512:
-  metadata.gz: c7df3eba977499a45acca66e8d25fd87cfe4669b9098c1e99be798f587de2ddf76edc75adb01953e8fe311ac324d7e4eb44cce66a33be264efe4f266e67d8f3e
-  data.tar.gz: 7fa4d7ba18f7be8ae8e80bd7b260be8bd8e9387122a623802812abd166652859f48669bbb42c16f72f15df1104d855934f4cfdb6cf6aede2737d740540d27278
+  metadata.gz: fb759fcff852bfa57315e4428250daeac0a0ae27771dc108e1879f3847616c073ce5d2fa8e079a77d73a51d7db3f18154d5cd6c7ea45c69b1282163cf817aa75
+  data.tar.gz: caea49e5286f9bf98e93c2931c2e8a77e18b9cce59e88c7ee7854f80bfe30b7b45649a148ed08c605ae87faeeb5ce87ae7f6fe23a523625db1728db1f215b120

data/lib/cogworker/attempts.rb

@@ -24,7 +24,7 @@ module Cogworker
     def record(jid, attempt:, error:, outcome:)
       entry = JSON.generate(
         'attempt' => attempt, 'failed_at' => Time.now.to_f, 'outcome' => outcome,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000]
+        'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error)
       )
       key = RedisKeys.job_attempts(jid)
       Cogworker.config.redis do |c|

data/lib/cogworker/basic_fetch.rb

@@ -17,8 +17,12 @@ module Cogworker
   class BasicFetch
     TIMEOUT = 2 # seconds; also how often a stopped/quieted processor notices and exits its fetch loop.
 
-    def initialize(queues)
+    # `stopping:` lets a fetcher cut its own waits short once shutdown has
+    # begun (Processor passes the Manager's `stopping?`), so a long idle
+    # pause can't hold up `Manager#stop!`.
+    def initialize(queues, stopping: -> { false })
       @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
+      @stopping = stopping
     end
 
     # Re-reads `cogworker:paused_queues` on every single fetch (not once at
@@ -33,7 +37,7 @@ module Cogworker
     def retrieve_work
       keys = active_keys
       if keys.empty?
-        sleep(TIMEOUT)
+        interruptible_sleep(TIMEOUT)
         return nil
       end
 
@@ -47,6 +51,12 @@ module Cogworker
     # Nothing to do: BRPOP already removed the job from Redis.
     def acknowledge(_work); end
 
+    # `Processor#interrupt`: the job ran, recording its failure failed —
+    # file it under `set` (retry/dead) at `score` instead.
+    def interrupt(_work, set, score, payload)
+      Cogworker.config.redis { |c| c.zadd(set, score, payload) }
+    end
+
     # Returns a fetched-but-unstarted job to the end of its queue that's
     # popped next.
     def give_back(work)
@@ -58,7 +68,8 @@ module Cogworker
     # that's what gets pushed back — onto the end its queue is popped from,
     # so it runs next. Returns how many. One entry that can't be read is
     # logged and skipped, never allowed to cost the others their requeue.
-    def self.requeue_in_progress(identity)
+    # (`except:` is ReliableFetch's — there's no in-progress list here.)
+    def self.requeue_in_progress(identity, except: [], keep: []) # rubocop:disable Lint/UnusedMethodArgument
       Cogworker.config.redis do |c|
         c.hvals(RedisKeys.workers(identity)).count do |raw|
           entry = JSON.parse(raw)
@@ -75,14 +86,24 @@ module Cogworker
       queue_key = RedisKeys.queue(queue)
       retake = job['periodic_until_executed'] && job['periodic_pjid'] && job['jid']
       lock_key = retake ? RedisKeys.periodic_running(job['periodic_pjid']) : queue_key
-      conn.eval(REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
-                                      argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
+      LuaScript.run(conn, REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
+                                               argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
                                              Periodic::RunningLock.queued_ttl])
     end
     private_class_method :requeue_entry
 
     private
 
+    def interruptible_sleep(seconds)
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + seconds
+      until @stopping.call
+        left = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        break if left <= 0
+
+        sleep([left, 0.25].min)
+      end
+    end
+
     def active_keys
       paused = Cogworker.config.redis { |c| c.smembers(RedisKeys::PAUSED_QUEUES) }
       return @queue_keys if paused.empty?

data/lib/cogworker/config.rb

@@ -9,7 +9,7 @@ module Cogworker
     FETCH_MODES = %i[reliable basic].freeze
 
     attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
-                :orphan_threshold
+                :orphan_threshold, :max_orphanings
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -51,9 +51,21 @@ module Cogworker
       # failover, a long GVL-holding call) — each of which runs its jobs
       # twice; lower: a really crashed process's jobs come back sooner.
       @orphan_threshold = 300
+      # How many times a job may be recovered from a process that died
+      # while running it before it goes to dead instead (ReliableFetch). The
+      # count can't single out the culprit: every job running in a process
+      # that crashed is counted, so keep this above the number of crashes an
+      # innocent job might plausibly sit through.
+      @max_orphanings = 3
       register_default_middleware
     end
 
+    def max_orphanings=(count)
+      raise ArgumentError, "max_orphanings must be a positive Integer, got #{count.inspect}" unless count.is_a?(Integer) && count.positive?
+
+      @max_orphanings = count
+    end
+
     def orphan_threshold=(seconds)
       unless seconds.is_a?(Numeric) && seconds.positive?
         raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"

data/lib/cogworker/heartbeat.rb

@@ -117,30 +117,53 @@ module Cogworker
         'rss_kb' => current_rss_kb
       }
 
+      # The presence key is what makes this process alive to everyone else
+      # (and what Heartbeat#start! waits for) — the one write a beat can't
+      # do without. Everything else is written on its own, best-effort: one
+      # shared key of the wrong type used to fail every beat of every
+      # process, so no new process could start anywhere in the fleet.
       Cogworker.config.redis do |c|
-        c.sadd?(RedisKeys::PROCESSES, identity)
         c.hset(RedisKeys.process(identity),
                'info', JSON.generate(info),
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
-        c.expire(RedisKeys.workers(identity), TTL)
+        secondary('process registry') { c.sadd?(RedisKeys::PROCESSES, identity) }
+        secondary('workers expiry') { c.expire(RedisKeys.workers(identity), TTL) }
         if reliable
-          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
-          c.hset(RedisKeys::LAST_BEAT, identity, c.time.first) # Redis's clock, not this host's
+          secondary('in-progress registry') { c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
+          secondary('last-beat stamp') { c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c)) }
         end
         touch_running_periodic_locks(c, identity)
       end
     end
 
+    def secondary(what)
+      yield
+    rescue Redis::BaseConnectionError
+      raise # Redis itself is gone: the beat as a whole failed
+    rescue StandardError => e
+      Cogworker.logger.error { "Heartbeat: #{what} not written: #{e.class}: #{e.message}" }
+    end
+
+    # Redis's clock, not this host's (so host clock skew doesn't matter to
+    # orphan detection) — falling back to this host's where `TIME` isn't
+    # allowed (some proxies/managed services): better a skewed stamp than a
+    # process that can never complete its first beat.
+    def redis_now(conn)
+      conn.time.first
+    rescue Redis::CommandError
+      Time.now.to_i
+    end
+
     # Keeps every in-flight `until_executed` periodic run's
     # `periodic:running:<pjid>` lock alive for as long as this process is —
-    # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
-    # Hash the Workers tab does, so it covers exactly the jobs actually
-    # executing here right now.
-    def touch_running_periodic_locks(conn, identity)
-      conn.hvals(RedisKeys.workers(identity)).each do |raw|
-        job = JSON.parse(raw)['payload']
+    # see Periodic::RunningLock. From the Manager's own in-memory record of
+    # what's running here, not `cogworker:workers` in Redis: a failover that
+    # lost that Hash would otherwise let a long run's lock lapse mid-run.
+    def touch_running_periodic_locks(conn, _identity)
+      @manager.running_jobs.each do |raw|
+        job = JSON.parse(raw)
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
         Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)

data/lib/cogworker/history/storage.rb

@@ -30,8 +30,8 @@ module Cogworker
         }
         if error
           entry['error_class'] = error.class.name
-          entry['error_message'] = error.message.to_s[0, 10_000]
-          entry['backtrace'] = (error.backtrace || []).first(200)
+          entry['error_message'] = JobUtil.error_message(error)
+          entry['backtrace'] = (error.backtrace || []).first(200).map { |line| JobUtil.safe_string(line) }
         end
 
         raw = JSON.generate(entry)

data/lib/cogworker/job_util.rb

@@ -37,11 +37,18 @@ module Cogworker
     # define `#to_i` (`nil.to_i` happens to be `0` — coincidentally correct
     # for absent — but `false.to_i` raises `NoMethodError` outright, which is
     # exactly the bug this method exists to not have).
+    #
+    # Anything else (a Hash, a non-numeric String…) means no retries — the
+    # cautious reading for a job with side effects — rather than raising
+    # from the middle of recording its failure.
     def max_retries(job)
-      case job['retry']
+      case (value = job['retry'])
       when false then 0
       when true, nil then DEFAULT_MAX_RETRY_ATTEMPTS
-      else job['retry'].to_i
+      when Integer then value
+      when Float then value.to_i
+      when String then Integer(value, exception: false) || 0
+      else 0
       end
     end
 
@@ -75,12 +82,17 @@ module Cogworker
     # commands, a crash or dropped connection between the ZREM and the
     # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
     # by Lua's cjson (which would round large integers in args).
+    # Present? Push, then remove — never remove first: a script stopped by
+    # an error keeps the writes made before it, so a ZREM followed by a
+    # failing LPUSH (the queue key holding the wrong type) lost the entry.
+    # The `cogworker:queues` listing is best-effort.
     CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
-      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+      if not redis.call("ZSCORE", KEYS[1], ARGV[1]) then
         return 0
       end
-      redis.call("SADD", KEYS[2], ARGV[2])
       redis.call("LPUSH", KEYS[3], ARGV[3])
+      redis.call("ZREM", KEYS[1], ARGV[1])
+      redis.pcall("SADD", KEYS[2], ARGV[2])
       return 1
     LUA
 
@@ -96,12 +108,40 @@ module Cogworker
       return :invalid unless job
 
       job['enqueued_at'] = Time.now.to_f
-      won = conn.eval(CLAIM_AND_REQUEUE_SCRIPT,
-                      keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
-                      argv: [raw, job['queue'], JSON.generate(job)])
+      won = LuaScript.run(conn, CLAIM_AND_REQUEUE_SCRIPT,
+                          keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
+                          argv: [raw, job['queue'], JSON.generate(job)])
       won == 1 ? :requeued : :gone
     end
 
+    ERROR_MESSAGE_LIMIT = 10_000
+
+    # Valid UTF-8, whatever came in — every string this gem stores from
+    # outside its control (an exception's message, a raw payload) goes
+    # through here before `JSON.generate`, which raises on invalid UTF-8.
+    # An unchecked `raise "bad \xff".b` used to make every attempt to record
+    # the failure itself raise, deterministically. Strings in another
+    # (valid) encoding are transcoded; binary/broken ones are scrubbed.
+    def safe_string(value, limit = nil)
+      str = value.to_s
+      str = if [Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII].include?(str.encoding)
+              str.dup.force_encoding(Encoding::UTF_8).scrub
+            else
+              str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+            end
+      limit ? str[0, limit] : str
+    end
+
+    # Never raises: an exception whose own `#message` raises (a broken
+    # custom exception class) used to escape from the middle of recording
+    # its failure — and, the job then counting as not-yet-run, it was handed
+    # back and re-run about once a second, forever.
+    def error_message(error)
+      safe_string(error.message, ERROR_MESSAGE_LIMIT)
+    rescue Exception # rubocop:disable Lint/RescueException
+      "#<#{error.class}>"
+    end
+
     UNPARSEABLE_CLASS = '(unparseable)'
 
     # A payload that isn't a JSON object can't be run, retried or even
@@ -113,8 +153,8 @@ module Cogworker
       {
         'class' => UNPARSEABLE_CLASS, 'args' => [], 'queue' => queue, 'jid' => SecureRandom.hex(12),
         'retry' => false, 'retry_count' => 0, 'enqueued_at' => now, 'failed_at' => now,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000],
-        'raw_payload' => raw.to_s[0, 100_000]
+        'error_class' => error.class.name, 'error_message' => error_message(error),
+        'raw_payload' => safe_string(raw, 100_000)
       }
     end
   end

data/lib/cogworker/key_guard.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Cogworker
+  # Finds this gem's Redis keys holding the wrong type of value — which
+  # only a manual write or data corruption produces, but which then breaks
+  # every command on that key, deterministically: a queue that can't be
+  # pushed to, a `dead` set nothing can be filed in, a registry that fails
+  # every beat — and moves each such key out of the way, to
+  # `cogworker:quarantine:<key>:<time>:<random>` (kept QUARANTINE_TTL for a human to
+  # look at), with an error in the log. Everything then carries on with a
+  # fresh, empty key of the right type. Run by every process's Scheduled
+  # poller at boot and every CHECK_INTERVAL.
+  module KeyGuard
+    CHECK_INTERVAL = 60
+    QUARANTINE_TTL = 30 * 24 * 60 * 60
+    QUARANTINE_PREFIX = 'cogworker:quarantine:'
+
+    FIXED = {
+      RedisKeys::SCHEDULE => 'zset', RedisKeys::RETRY => 'zset', RedisKeys::DEAD => 'zset',
+      RedisKeys::QUEUES => 'set', RedisKeys::PROCESSES => 'set', RedisKeys::PAUSED_QUEUES => 'set',
+      RedisKeys::IN_PROGRESS_IDENTITIES => 'set', RedisKeys::LAST_BEAT => 'hash',
+      RedisKeys::REPEAT_ORPHANS => 'list', RedisKeys::UNSETTLED => 'list', RedisKeys::PERIODIC_SCHEDULE => 'hash',
+      RedisKeys::PERIODIC_DISABLED => 'set', RedisKeys::STATS_PROCESSED => 'string',
+      RedisKeys::STATS_FAILED => 'string'
+    }.freeze
+
+    # KEYS[1] = the key, KEYS[2] = its quarantine name; ARGV[1] = expected
+    # type, ARGV[2] = quarantine TTL. Re-checked here, atomically with the
+    # move, so a key fixed (or recreated) in between is left alone.
+    QUARANTINE_SCRIPT = <<~LUA
+      local actual = redis.call("TYPE", KEYS[1])["ok"]
+      if actual == "none" or actual == ARGV[1] then
+        return 0
+      end
+      redis.call("RENAME", KEYS[1], KEYS[2])
+      redis.call("EXPIRE", KEYS[2], ARGV[2])
+      return 1
+    LUA
+
+    module_function
+
+    # Returns the keys quarantined.
+    def check(queues: Cogworker.config.queues)
+      expected = expected_types(queues)
+      actual = Cogworker.config.redis do |c|
+        c.pipelined { |p| expected.each_key { |key| p.type(key) } }
+      end
+      expected.keys.zip(actual).filter_map do |key, type|
+        next if type == 'none' || type == expected[key]
+
+        key if quarantine(key, expected[key], type)
+      end
+    end
+
+    # The fixed keys, plus the per-queue and per-process ones currently
+    # listed in them (only where the listing itself is readable).
+    def expected_types(queues)
+      types = FIXED.dup
+      queue_names = Array(queues) | readable_members(RedisKeys::QUEUES)
+      queue_names.each { |q| types[RedisKeys.queue(q)] = 'list' }
+      readable_members(RedisKeys::IN_PROGRESS_IDENTITIES).each { |id| types[RedisKeys.in_progress(id)] = 'list' }
+      readable_members(RedisKeys::PROCESSES).each do |id|
+        types[RedisKeys.process(id)] = 'hash'
+        types[RedisKeys.workers(id)] = 'hash'
+      end
+      types
+    end
+
+    def readable_members(set)
+      Cogworker.config.redis { |c| c.smembers(set) }
+    rescue Redis::CommandError
+      [] # itself of the wrong type: quarantined this round, listed again next
+    end
+
+    def quarantine(key, expected, actual)
+      # Unique per move: a second quarantine of the same key within the same
+      # second must not overwrite (RENAME) the first one's copy.
+      target = "#{QUARANTINE_PREFIX}#{key}:#{Time.now.to_i}:#{SecureRandom.hex(4)}"
+      moved = Cogworker.config.redis do |c|
+        LuaScript.run(c, QUARANTINE_SCRIPT, keys: [key, target], argv: [expected, QUARANTINE_TTL])
+      end
+      return false unless moved == 1
+
+      Cogworker.logger.error do
+        "#{key} held a #{actual}, not a #{expected}: moved to #{target} (kept #{QUARANTINE_TTL / 86_400} days)"
+      end
+      true
+    end
+  end
+end

data/lib/cogworker/launcher.rb

@@ -13,8 +13,15 @@ module Cogworker
       @manager = Manager.new(config)
       @scheduled = Scheduled.new(@manager)
       @heartbeat = Heartbeat.new(@manager)
+      # The first tick waits for the first orphan check that really
+      # completed (Scheduled#ready_for_ticker?): after the whole
+      # fleet was down longer than a running `until_executed` job's lock
+      # lives, the ticker would otherwise claim that entry's next slot before
+      # recovery had requeued the interrupted run (and re-taken its lock) —
+      # two runs at once.
       @ticker = Periodic::Ticker.new(
-        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up
+        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
+                                                   ready: -> { @scheduled.ready_for_ticker? }
       )
       @signal_queue = ::Queue.new
     end

data/lib/cogworker/lua_script.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+
+module Cogworker
+  # Runs a Lua script by its SHA1 (EVALSHA) instead of shipping the whole
+  # source on every call, falling back to EVAL — which also loads it into
+  # the server's script cache for next time — on NOSCRIPT (first use, a
+  # restarted or failed-over server, SCRIPT FLUSH). Every script in this
+  # gem goes through here.
+  module LuaScript
+    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
+    SHAS_MUTEX = Mutex.new
+
+    module_function
+
+    def run(conn, source, keys: [], argv: [])
+      begin
+        return conn.evalsha(sha(source), keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+      end
+      conn.eval(source, keys: keys, argv: argv)
+    end
+
+    def sha(source)
+      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+    end
+  end
+end

data/lib/cogworker/manager.rb

@@ -7,6 +7,16 @@ module Cogworker
     def initialize(config = Cogworker.config)
       @config = config
       @fetch_mutex = Mutex.new
+      # This process's own record of its jobs (raw payloads, as fetched):
+      # running right now (a count — two copies of one payload can run at
+      # once), and finished but not yet settled in Redis — `:ack` (the ack
+      # failed) or `[set, score, payload]` (the job ran and failed, and
+      # filing it in retry/dead failed too). In memory on purpose:
+      # ReliableFetch.reconcile trusts it over anything in Redis, which may
+      # be what's failing (or what lost its last writes in a failover).
+      @jobs_mutex = Mutex.new
+      @running_jobs = Hash.new(0)
+      @pending_settlements = {}
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -18,11 +28,11 @@ module Cogworker
       @config.queues
     end
 
-    # Resolved on first use — from a processor thread's first fetch, after
-    # the Heartbeat's first beat proved Redis reachable — not in the
-    # constructor: that needed Redis just to build a Manager, and when
-    # Redis was unreachable at that moment, the version check couldn't run
-    # at all (see ReliableFetch.supported?).
+    # Resolved on first use, not in the constructor (that needed Redis
+    # just to build a Manager). First use is normally the Heartbeat's first
+    # beat; if Redis can't be reached then, ReliableFetch.supported? raises
+    # and nothing is memoized — the beat fails and is retried, and this
+    # resolves again next time — rather than memoizing a guess.
     def fetch_class
       @fetch_mutex.synchronize { @fetch_class ||= resolve_fetch_class }
     end
@@ -63,6 +73,34 @@ module Cogworker
       @stopping
     end
 
+    def job_started(raw)
+      @jobs_mutex.synchronize { @running_jobs[raw] += 1 }
+    end
+
+    def job_finished(raw)
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] -= 1
+        @running_jobs.delete(raw) unless @running_jobs[raw].positive?
+      end
+    end
+
+    def running_jobs
+      @jobs_mutex.synchronize { @running_jobs.keys }
+    end
+
+    # `how`: :ack, or [set, score, payload] — see ReliableFetch.settle.
+    def settle_later(raw, how)
+      @jobs_mutex.synchronize { @pending_settlements[raw] = how }
+    end
+
+    def settled(raw)
+      @jobs_mutex.synchronize { @pending_settlements.delete(raw) }
+    end
+
+    def pending_settlements
+      @jobs_mutex.synchronize { @pending_settlements.dup }
+    end
+
     def busy_count
       @busy_mutex.synchronize { @busy_count }
     end
@@ -109,8 +147,27 @@ module Cogworker
       end
     end
 
+    # Jobs that finished but aren't settled in Redis yet are still on the
+    # in-progress list. One more try at settling them first; then `except:`
+    # drops the ones only waiting for their ack (which is what dropping
+    # them is), in the same atomic step as requeueing the rest — a separate
+    # ack pass that failed used to leave them to be requeued and run again.
+    # Ones that couldn't be filed in retry/dead even now are handed over to
+    # UNSETTLED for any live process to file later (ReliableFetch.park_unsettled)
+    # — never requeued: they already ran. Any that can't even be handed
+    # over are left on the list (`keep:`): orphan recovery requeues them
+    # once this process is gone, at most `max_orphanings` times.
     def requeue_unfinished
-      moved = fetch_class.requeue_in_progress(Cogworker.identity)
+      keep = []
+      if fetch_class == ReliableFetch
+        ReliableFetch.settle_pending(Cogworker.identity, pending_settlements) { |raw| settled(raw) }
+        unfiled = pending_settlements.reject { |_raw, how| how == :ack }
+        keep = ReliableFetch.park_unsettled(Cogworker.identity, unfiled)
+        (unfiled.keys - keep).each { |raw| settled(raw) }
+      end
+      to_ack = pending_settlements.select { |_raw, how| how == :ack }.keys
+      moved = fetch_class.requeue_in_progress(Cogworker.identity, except: to_ack, keep: keep)
+      to_ack.each { |raw| settled(raw) }
       Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
     rescue StandardError => e
       Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }

data/lib/cogworker/owned_key.rb

@@ -32,9 +32,9 @@ module Cogworker
     end
 
     def run(conn, script, key, argv)
-      return conn.eval(script, keys: [key], argv: argv) if conn
+      return LuaScript.run(conn, script, keys: [key], argv: argv) if conn
 
-      Cogworker.config.redis { |c| c.eval(script, keys: [key], argv: argv) }
+      Cogworker.config.redis { |c| LuaScript.run(c, script, keys: [key], argv: argv) }
     end
   end
 end