cogworker 0.2.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db7056e0bc7a73b9ed86025627dc3ab891e971f13797fb253ec2ea8df15362f9
-  data.tar.gz: bd4b8a4f65b9538284a8d8682513fd868c1e8bfbee7b73709016ae7826ed38cb
+  metadata.gz: bfb90fe3a477f31658c58fd09a41e4404a0c09999b74614789bd343ee704fad2
+  data.tar.gz: 407aa20293538fa35c8219f88f21fd8b1c15150d96e77d2b495d017ef21f924e
 SHA512:
-  metadata.gz: 9a9e6c877cf0d62f2cadece94b9ea07c5d9c3f0386ce914f0a0132825de5128da1a26494aa1ad4866f5ef02ce5f8f8d52e1f009fb93583a32f873652fa4407b6
-  data.tar.gz: f06acea355f8898b7c0fdc3558c8ea76576e16f5070182cb30e4c883a87226b96ef3454594b4c8deab8e99a0862a2b951d71efb8f237d079205517fd4e076f51
+  metadata.gz: c7df3eba977499a45acca66e8d25fd87cfe4669b9098c1e99be798f587de2ddf76edc75adb01953e8fe311ac324d7e4eb44cce66a33be264efe4f266e67d8f3e
+  data.tar.gz: 7fa4d7ba18f7be8ae8e80bd7b260be8bd8e9387122a623802812abd166652859f48669bbb42c16f72f15df1104d855934f4cfdb6cf6aede2737d740540d27278

data/lib/cogworker/basic_fetch.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Cogworker
-  # Pops jobs off Redis lists. Queue names repeated in the process's queue
-  # list represent weight; since BRPOP itself scans its key list
+  # `config.fetch = :basic`: pops jobs off Redis lists with one blocking
+  # `BRPOP`. Once popped, a job exists only in the processor's memory (and
+  # its `cogworker:workers:<identity>` entry, which expires with the
+  # process), so a process that dies mid-job loses it — see ReliableFetch,
+  # the default, for the alternative.
+  #
+  # Queue names repeated in the process's queue list represent weight;
+  # since BRPOP itself scans its key list
   # strictly left-to-right, weighting is implemented by shuffling the
   # (already-expanded, so repeats survive) key list before every fetch cycle
   # — the more often a name appears, the more likely it lands first.
@@ -36,6 +44,43 @@ module Cogworker
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
 
+    # Nothing to do: BRPOP already removed the job from Redis.
+    def acknowledge(_work); end
+
+    # Returns a fetched-but-unstarted job to the end of its queue that's
+    # popped next.
+    def give_back(work)
+      Cogworker.config.redis { |c| c.rpush(RedisKeys.queue(work.queue), work.raw_job) }
+    end
+
+    # Shutdown past the drain timeout (`Manager#stop!`): the only record of
+    # a still-running job is its `cogworker:workers:<identity>` entry, so
+    # that's what gets pushed back — onto the end its queue is popped from,
+    # so it runs next. Returns how many. One entry that can't be read is
+    # logged and skipped, never allowed to cost the others their requeue.
+    def self.requeue_in_progress(identity)
+      Cogworker.config.redis do |c|
+        c.hvals(RedisKeys.workers(identity)).count do |raw|
+          entry = JSON.parse(raw)
+          requeue_entry(c, entry.fetch('queue'), entry.fetch('payload'))
+          true
+        rescue StandardError => e
+          Cogworker.logger.error { "couldn't requeue in-flight entry #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
+          false
+        end
+      end
+    end
+
+    def self.requeue_entry(conn, queue, job)
+      queue_key = RedisKeys.queue(queue)
+      retake = job['periodic_until_executed'] && job['periodic_pjid'] && job['jid']
+      lock_key = retake ? RedisKeys.periodic_running(job['periodic_pjid']) : queue_key
+      conn.eval(REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
+                                      argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
+                                             Periodic::RunningLock.queued_ttl])
+    end
+    private_class_method :requeue_entry
+
     private
 
     def active_keys
@@ -46,5 +91,18 @@ module Cogworker
     end
 
     UnitOfWork = Struct.new(:queue, :raw_job)
+
+    # KEYS[1] = the queue, KEYS[2] = the job's periodic running lock (or the
+    # queue again when there's none to take); ARGV[1] = payload, ARGV[2] =
+    # "1" to re-take that lock, ARGV[3] = jid, ARGV[4] = lock TTL. Requeue
+    # and lock in one step: as two commands, a failure in between left the
+    # job queued without its lock (and miscounted).
+    REQUEUE_ENTRY_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      redis.call("RPUSH", KEYS[1], ARGV[1])
+      if ARGV[2] == "1" then
+        retake_running_lock(KEYS[2], ARGV[3], ARGV[4])
+      end
+      return 1
+    LUA
   end
 end

data/lib/cogworker/cli.rb

@@ -8,8 +8,15 @@ module Cogworker
   # `-r/--require <file>`, `-C/--config <file>`, `-L/--logfile <file>`,
   # `-q/--queue <name>[,<weight>]` (repeatable).
   class CLI
+    # Read (first set wins) for the default environment, and all written
+    # back with the resolved one — whichever of them the app's own boot
+    # code reads (Rails: RAILS_ENV, then RACK_ENV; Rack apps: RACK_ENV/
+    # APP_ENV) sees the same value `-e` asked for.
+    ENV_VARS = %w[APP_ENV RAILS_ENV RACK_ENV].freeze
+
     def self.parse(argv)
-      options = { queues: [], require_path: nil, environment: ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development' }
+      environment = ENV_VARS.filter_map { |k| ENV.fetch(k, nil) }.first || 'development'
+      options = { queues: [], require_path: nil, environment: environment }
 
       OptionParser.new do |o|
         o.on('-e ENV', '--environment ENV') { |v| options[:environment] = v }
@@ -34,11 +41,14 @@ module Cogworker
     # forked child with the exact same flags it was started with.
     def run(argv)
       options = self.class.parse(argv)
+      # Before the config file (its ERB may read these) and the app code.
+      ENV_VARS.each { |k| ENV[k] = options[:environment] }
       file_config = ConfigLoader.load(options[:config_path])
 
       Cogworker.server_process!
       Cogworker.config.concurrency = options[:concurrency] || file_config[:concurrency]&.to_i || Cogworker.config.concurrency
       Cogworker.config.queues = options[:queues].any? ? options[:queues] : (file_config[:queues] || Cogworker.config.queues)
+      Cogworker.config.fetch = file_config[:fetch] if file_config[:fetch]
 
       redirect_logfile(options[:logfile]) if options[:logfile]
       require_app_code(options[:require_path]) if options[:require_path]

data/lib/cogworker/client.rb

@@ -50,7 +50,7 @@ module Cogworker
           else
             job['enqueued_at'] = Time.now.to_f
             conn.multi do |pipeline|
-              pipeline.sadd(RedisKeys::QUEUES, job['queue'])
+              pipeline.sadd?(RedisKeys::QUEUES, job['queue'])
               pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
             end
           end

data/lib/cogworker/config.rb

@@ -6,7 +6,10 @@ module Cogworker
   # `configure_client` calls, so every block sees and extends the same
   # server/client middleware chains.
   class Config
-    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options
+    FETCH_MODES = %i[reliable basic].freeze
+
+    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
+                :orphan_threshold
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -31,11 +34,61 @@ module Cogworker
       # set higher/lower to match how long a unique job might legitimately
       # run plus however long its retries can take.
       @unique_lock_ttl = 24 * 60 * 60
+      # How processors take jobs off their queues — `:reliable` (ReliableFetch,
+      # needs Redis >= 6.2; a job survives its process dying) or `:basic`
+      # (BasicFetch, one blocking BRPOP; a job is lost if its process dies).
+      # On an older Redis, `:reliable` falls back to `:basic` with a warning
+      # (Manager#resolve_fetch_class).
+      @fetch = :reliable
+      # Longest pause (seconds) ReliableFetch makes between polls of empty
+      # queues — the worst-case delay before an idle worker notices a new
+      # job. See ReliableFetch::EMPTY_POLL_INTERVAL.
+      @fetch_idle_max_interval = 1.0
+      # How long (seconds) a process must have gone without a heartbeat
+      # before ReliableFetch.recover_orphans treats it as dead and requeues
+      # its in-progress jobs. Higher: fewer false "dead" verdicts on a live
+      # process that merely couldn't beat for a while (a Redis outage or
+      # failover, a long GVL-holding call) — each of which runs its jobs
+      # twice; lower: a really crashed process's jobs come back sooner.
+      @orphan_threshold = 300
       register_default_middleware
     end
 
+    def orphan_threshold=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @orphan_threshold = seconds
+    end
+
+    def fetch_idle_max_interval=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "fetch_idle_max_interval must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @fetch_idle_max_interval = seconds
+    end
+
+    def fetch=(mode)
+      resolved = mode.to_s.to_sym
+      unless FETCH_MODES.include?(resolved)
+        raise ArgumentError, "fetch must be one of #{FETCH_MODES.join(', ')}, got #{mode.inspect}"
+      end
+
+      @fetch = resolved
+    end
+
+    # Also drops an already-built pool: `redis_pool` is memoized on first
+    # use, so without this a `redis =` after anything had touched Redis
+    # (an init file configuring twice, a test switching databases) was
+    # silently ignored. The old pool's connections are closed as they're
+    # checked back in, so a caller still holding one finishes normally.
     def redis=(options)
       @redis_options = options
+      old_pool = @redis_pool
+      @redis_pool = nil
+      old_pool&.shutdown(&:close)
     end
 
     def redis_pool

data/lib/cogworker/heartbeat.rb

@@ -15,25 +15,47 @@ module Cogworker
   class Heartbeat
     INTERVAL = 5
     TTL = 60
+    STARTUP_RETRY_INTERVAL = 1
 
     def initialize(manager)
       @manager = manager
     end
 
-    def start!
+    # The first beat is synchronous, before any processor fetches a job:
+    # ReliableFetch.recover_orphans treats an in-progress list with no live
+    # heartbeat behind it as a dead process's, so this process must be
+    # visibly alive before its own list can have anything on it. With Redis
+    # unreachable, it keeps retrying (every STARTUP_RETRY_INTERVAL) rather
+    # than letting the Manager start fetching with no presence key — until
+    # it succeeds (returns true) or `abort_if` says to give up (returns
+    # false, e.g. a stop signal arrived meanwhile; nothing is started then).
+    def start!(abort_if: -> { false })
       @stopping = false
+      until beat_safely
+        return false if abort_if.call
+
+        sleep(STARTUP_RETRY_INTERVAL)
+      end
       @beat_thread = Thread.new { beat_loop }
       @signal_thread = Thread.new { subscribe_loop }
+      true
     end
 
     # Tears down both background threads, not just the Redis keys — leaving
     # them running would leak a thread (and a dedicated pub/sub connection)
     # per Heartbeat instance for the remaining life of the process.
+    #
+    # The subscriber thread is killed *before* its connection is closed, and
+    # never via `unsubscribe`: on redis-rb 4.x, `unsubscribe` from another
+    # thread waits on the client's monitor, which the subscriber thread
+    # holds for as long as it sits in its blocking read — so it hung
+    # forever, and with it every `TERM`. Killing the thread releases the
+    # monitor; closing the now-idle connection afterwards is safe on both
+    # 4.x and 5.x.
     def stop!
       @stopping = true
       @beat_thread&.kill
-      @subscribe_client&.unsubscribe
-      @signal_thread&.kill
+      stop_subscriber!
       cleanup_presence!
     rescue StandardError
       nil
@@ -41,24 +63,51 @@ module Cogworker
 
     private
 
+    def stop_subscriber!
+      if @signal_thread && @signal_thread != Thread.current
+        @signal_thread.kill
+        @signal_thread.join(1)
+      end
+      @subscribe_client&.close
+    rescue StandardError
+      nil
+    end
+
     def cleanup_presence!
       Cogworker.config.redis do |c|
         c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
-        c.srem(RedisKeys::PROCESSES, Cogworker.identity)
+        c.srem?(RedisKeys::PROCESSES, Cogworker.identity)
+        # Clean shutdown: Manager#stop! already requeued this process's
+        # in-progress list, so there's nothing left for recovery to find.
+        if c.llen(RedisKeys.in_progress(Cogworker.identity)).zero?
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, Cogworker.identity)
+          c.hdel(RedisKeys::LAST_BEAT, Cogworker.identity)
+        end
       end
     end
 
+    # A failed beat (Redis blip, failover) is logged and retried on the next
+    # interval, never allowed to end the loop: a dead heartbeat thread
+    # meant this process silently vanished from ProcessSet/the Workers tab
+    # once its TTL ran out, while still running jobs.
     def beat_loop
       loop do
-        beat
         sleep(INTERVAL)
+        beat_safely
       end
+    end
+
+    def beat_safely
+      beat
+      true
     rescue StandardError => e
-      Cogworker.logger.error { "Heartbeat died: #{e.class}: #{e.message}" }
+      Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      false
     end
 
     def beat
       identity = Cogworker.identity
+      reliable = @manager.fetch_class == ReliableFetch
       info = {
         'hostname' => Cogworker.hostname,
         'pid' => ::Process.pid,
@@ -69,13 +118,35 @@ module Cogworker
       }
 
       Cogworker.config.redis do |c|
-        c.sadd(RedisKeys::PROCESSES, identity)
+        c.sadd?(RedisKeys::PROCESSES, identity)
         c.hset(RedisKeys.process(identity),
                'info', JSON.generate(info),
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
         c.expire(RedisKeys.workers(identity), TTL)
+        if reliable
+          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hset(RedisKeys::LAST_BEAT, identity, c.time.first) # Redis's clock, not this host's
+        end
+        touch_running_periodic_locks(c, identity)
+      end
+    end
+
+    # Keeps every in-flight `until_executed` periodic run's
+    # `periodic:running:<pjid>` lock alive for as long as this process is —
+    # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
+    # Hash the Workers tab does, so it covers exactly the jobs actually
+    # executing here right now.
+    def touch_running_periodic_locks(conn, identity)
+      conn.hvals(RedisKeys.workers(identity)).each do |raw|
+        job = JSON.parse(raw)['payload']
+        next unless job.is_a?(Hash) && job['periodic_pjid']
+
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+      rescue StandardError => e
+        # Per entry: one bad entry mustn't leave the others' locks to expire.
+        Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
       end
     end
 
@@ -146,6 +217,7 @@ module Cogworker
     def remote_stop!
       @manager.stop!
       cleanup_presence!
+      Cogworker.flush_output!
       ::Process.exit!(true)
     end
   end

data/lib/cogworker/job.rb

@@ -50,6 +50,15 @@ module Cogworker
         Testing.jobs_for(name).clear
       end
 
+      # Run this class's recorded fake-mode jobs — see `Testing.perform_one`.
+      def perform_one
+        Testing.perform_one(name)
+      end
+
+      def drain
+        Testing.drain(name)
+      end
+
       private
 
       def job_payload(extra)

data/lib/cogworker/job_util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'json'
 require 'securerandom'
 
 module Cogworker
@@ -56,5 +57,65 @@ module Cogworker
     def terminal_failure?(job)
       job['retry_count'].to_i >= max_retries(job)
     end
+
+    # The job Hash a raw `schedule`/`retry`/`dead` entry would requeue as —
+    # or `nil` if it can't be: not JSON, not an object, or no String
+    # `queue` to push it onto (e.g. an `unparseable_job` wrapper, whose
+    # `queue` may be nil).
+    def requeueable(raw)
+      job = JSON.parse(raw)
+      job.is_a?(Hash) && job['queue'].is_a?(String) && !job['queue'].empty? ? job : nil
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:queues, KEYS[3] = the queue;
+    # ARGV[1] = the entry as stored, ARGV[2] = queue name, ARGV[3] = the
+    # payload to push. Claim and push in one atomic step: as separate
+    # commands, a crash or dropped connection between the ZREM and the
+    # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
+    # by Lua's cjson (which would round large integers in args).
+    CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("SADD", KEYS[2], ARGV[2])
+      redis.call("LPUSH", KEYS[3], ARGV[3])
+      return 1
+    LUA
+
+    # Validate, then (atomically) claim and requeue — an entry that can't be
+    # requeued is never removed and lost (it stays where it is for the
+    # caller to deal with). Returns `:requeued`, `:gone` (someone else
+    # claimed it first) or `:invalid`. The one implementation behind
+    # `Scheduled#graduate` and every Web UI retry action. Re-stamps
+    # `enqueued_at`: queue latency is measured from it, and a retried job
+    # still carried its original push time (a scheduled one, none at all).
+    def claim_and_requeue(conn, set, raw)
+      job = requeueable(raw)
+      return :invalid unless job
+
+      job['enqueued_at'] = Time.now.to_f
+      won = conn.eval(CLAIM_AND_REQUEUE_SCRIPT,
+                      keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
+                      argv: [raw, job['queue'], JSON.generate(job)])
+      won == 1 ? :requeued : :gone
+    end
+
+    UNPARSEABLE_CLASS = '(unparseable)'
+
+    # A payload that isn't a JSON object can't be run, retried or even
+    # attributed to a class. Wrapped into a regular job Hash (the Web UI
+    # parses every Dead entry as one) that keeps the original bytes under
+    # `raw_payload`, for whoever finds one to bury it in `dead`.
+    def unparseable_job(raw, queue:, error:)
+      now = Time.now.to_f
+      {
+        'class' => UNPARSEABLE_CLASS, 'args' => [], 'queue' => queue, 'jid' => SecureRandom.hex(12),
+        'retry' => false, 'retry_count' => 0, 'enqueued_at' => now, 'failed_at' => now,
+        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000],
+        'raw_payload' => raw.to_s[0, 100_000]
+      }
+    end
   end
 end

data/lib/cogworker/launcher.rb

@@ -22,9 +22,11 @@ module Cogworker
     def run
       Cogworker.reset_identity!
       install_signal_traps
+      resolve_periodic_classes
+      return unless start_heartbeat
+
       @manager.start!
       @scheduled.start!
-      @heartbeat.start!
       @ticker.start!
       log_startup_info
       watch_signals
@@ -43,6 +45,41 @@ module Cogworker
 
     private
 
+    # First, and blocking until it succeeds: see Heartbeat#start!. A stop
+    # signal arriving while Redis is still unreachable ends the wait (and
+    # the process) without ever starting anything; a quiet one is applied
+    # and the wait goes on.
+    def start_heartbeat
+      until @heartbeat.start!(abort_if: -> { !@signal_queue.empty? })
+        case @signal_queue.pop
+        when :stop
+          Cogworker.logger.info { 'Received stop signal before startup completed' }
+          stop!
+          return false
+        when :quiet then quiet!
+        end
+      end
+      true
+    end
+
+    # Resolves every periodic entry's class here, on the main thread, before
+    # any processor thread exists. `Processor#build_worker` otherwise does
+    # the first `const_get` lazily on whichever processor thread picks the
+    # job up — and with an app autoloader that isn't thread-safe, two
+    # threads racing to autoload the same class (or its base class) can see
+    # it half-defined (`undefined method 'perform'`). A class that can't be
+    # resolved is only logged: every run of that entry will fail the same
+    # way (and show up in Dead/History), but it shouldn't keep the rest of
+    # the process from booting. Regular job classes aren't known up front —
+    # eager-load the app in the init file for those (see README).
+    def resolve_periodic_classes
+      @config.periodic_manager.entries.map(&:class_name).uniq.each do |name|
+        Object.const_get(name)
+      rescue NameError => e
+        Cogworker.logger.warn { "periodic job class #{name} can't be resolved: #{e.message}" }
+      end
+    end
+
     # Logged once all components are up, so the operator's console shows a
     # single summary (version/identity/pid/concurrency/queues/redis target)
     # right at the point the process is actually ready to pull work, rather

data/lib/cogworker/logging.rb

@@ -7,6 +7,10 @@ module Cogworker
   # it), with a compact one-line-per-entry format including pid.
   module Logging
     def self.default_logger(io = $stdout)
+      # Unbuffered: processes here end via `Process.exit!` (swarm children,
+      # a remote stop), which skips Ruby's IO flush — with stdout a pipe
+      # (docker, systemd) the last lines logged before exit were lost.
+      io.sync = true if io.respond_to?(:sync=)
       logger = ::Logger.new(io)
       logger.level = ::Logger::INFO
       logger.formatter = proc do |severity, time, _progname, msg|

data/lib/cogworker/manager.rb

@@ -6,6 +6,7 @@ module Cogworker
   class Manager
     def initialize(config = Cogworker.config)
       @config = config
+      @fetch_mutex = Mutex.new
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -17,6 +18,31 @@ module Cogworker
       @config.queues
     end
 
+    # Resolved on first use — from a processor thread's first fetch, after
+    # the Heartbeat's first beat proved Redis reachable — not in the
+    # constructor: that needed Redis just to build a Manager, and when
+    # Redis was unreachable at that moment, the version check couldn't run
+    # at all (see ReliableFetch.supported?).
+    def fetch_class
+      @fetch_mutex.synchronize { @fetch_class ||= resolve_fetch_class }
+    end
+
+    # The server turned out not to support ReliableFetch after all
+    # (ReliableFetch::UnsupportedError on a real fetch): every processor
+    # switches to BasicFetch from its next fetch on. Safe, since no job can
+    # be on the in-progress list — no LMOVE ever succeeded.
+    def fall_back_to_basic_fetch!(error)
+      @fetch_mutex.synchronize do
+        next if @fetch_class == BasicFetch
+
+        Cogworker.logger.warn do
+          "config.fetch = :reliable isn't supported by this Redis (#{error.message[0, 120]}); " \
+            'falling back to :basic (a job is lost if its process dies mid-job)'
+        end
+        @fetch_class = BasicFetch
+      end
+    end
+
     def start!
       @processors.each(&:start!)
     end
@@ -49,10 +75,13 @@ module Cogworker
       @busy_mutex.synchronize { @busy_count -= 1 }
     end
 
-    # Waits (up to `timeout` seconds) for in-flight jobs to finish, then
-    # returns. Does not force-kill lingering threads past the deadline — a
-    # thread stuck past the deadline is the caller's (Launcher's) problem to
-    # decide whether to hard-exit the process.
+    # Waits (up to `timeout` seconds) for in-flight jobs to finish, then puts
+    # any job still unfinished back on its queue and returns. Does not
+    # force-kill lingering threads past the deadline — a thread stuck past
+    # the deadline is the caller's (Launcher's) problem to decide whether to
+    # hard-exit the process; its job is already safely requeued by then,
+    # rather than lost with the process (it may therefore run twice, if the
+    # thread does still finish it).
     def stop!(timeout: 25)
       @quiet = true
       @stopping = true
@@ -61,10 +90,43 @@ module Cogworker
         remaining = deadline - monotonic_now
         p.thread&.join([remaining, 0].max)
       end
+      wait_for_fetching_processors
+      requeue_unfinished
     end
 
     private
 
+    # A processor that isn't running a job is at most one fetch timeout away
+    # from noticing `stopping?` and exiting — but until it does, it can
+    # still pop a job (BRPOP blocks for up to BasicFetch::TIMEOUT), and one
+    # popped after `requeue_unfinished` had already run would be lost with
+    # the process. Waiting those out first means the only threads left
+    # alive are ones stuck inside a job, which never fetch again.
+    def wait_for_fetching_processors
+      deadline = monotonic_now + BasicFetch::TIMEOUT + 1
+      @processors.reject(&:busy?).each do |p|
+        p.thread&.join([deadline - monotonic_now, 0].max)
+      end
+    end
+
+    def requeue_unfinished
+      moved = fetch_class.requeue_in_progress(Cogworker.identity)
+      Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
+    rescue StandardError => e
+      Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }
+    end
+
+    def resolve_fetch_class
+      return BasicFetch if @config.fetch == :basic
+      return ReliableFetch if ReliableFetch.supported?
+
+      Cogworker.logger.warn do
+        "config.fetch = :reliable needs Redis >= #{ReliableFetch::MIN_REDIS_VERSION}; falling back to :basic " \
+          '(a job is lost if its process dies mid-job)'
+      end
+      BasicFetch
+    end
+
     # Fully qualified: inside the Cogworker namespace, a bare `Process` would
     # resolve to Cogworker::Process (the ProcessSet entry wrapper in api.rb),
     # not the ::Process kernel module.

data/lib/cogworker/owned_key.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Compare-and-act on a lock key whose value is its owner's jid
+  # (`periodic:running:<pjid>`, `cogworker:unique:<digest>`): the EXPIRE/DEL
+  # only happens if the key still holds that exact jid, atomically — so a
+  # stray older run can never shorten, extend or release a lock that has
+  # since passed to a newer one.
+  module OwnedKey
+    EXPIRE_IF_OWNED = <<~LUA
+      if redis.call("GET", KEYS[1]) == ARGV[1] then
+        return redis.call("EXPIRE", KEYS[1], ARGV[2])
+      end
+      return 0
+    LUA
+
+    DELETE_IF_OWNED = <<~LUA
+      if redis.call("GET", KEYS[1]) == ARGV[1] then
+        return redis.call("DEL", KEYS[1])
+      end
+      return 0
+    LUA
+
+    module_function
+
+    def expire(key, owner, ttl, conn = nil)
+      run(conn, EXPIRE_IF_OWNED, key, [owner, ttl.ceil])
+    end
+
+    def delete(key, owner, conn = nil)
+      run(conn, DELETE_IF_OWNED, key, [owner])
+    end
+
+    def run(conn, script, key, argv)
+      return conn.eval(script, keys: [key], argv: argv) if conn
+
+      Cogworker.config.redis { |c| c.eval(script, keys: [key], argv: argv) }
+    end
+  end
+end