cogworker 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db7056e0bc7a73b9ed86025627dc3ab891e971f13797fb253ec2ea8df15362f9
-  data.tar.gz: bd4b8a4f65b9538284a8d8682513fd868c1e8bfbee7b73709016ae7826ed38cb
+  metadata.gz: 13e1ad0fec77124674f55dc3e1f7213bd2a73834ebb474261ed82a337fc1bb3e
+  data.tar.gz: ed51fed2854035c83016b3180fa7872772e6f60bb2dd84e44089485a536f81e4
 SHA512:
-  metadata.gz: 9a9e6c877cf0d62f2cadece94b9ea07c5d9c3f0386ce914f0a0132825de5128da1a26494aa1ad4866f5ef02ce5f8f8d52e1f009fb93583a32f873652fa4407b6
-  data.tar.gz: f06acea355f8898b7c0fdc3558c8ea76576e16f5070182cb30e4c883a87226b96ef3454594b4c8deab8e99a0862a2b951d71efb8f237d079205517fd4e076f51
+  metadata.gz: 0d78dd7a7c4bbdf8bfc8c8619467922e732879908562b1ac99fed71ce111a8a4925136e87a9c4e56cb5abed57867d76f06c5d929fcb822156d4fd0add7a182d9
+  data.tar.gz: 265d890a8f7d65b180dcb9a0f00f0d2ba6daf16fa9044f7bdb94b88b2cbeced1f6d61f5e11c86d5b107d281bbbc47591b54cb3fc6820f7a7fecc4732dec451d2

data/lib/cogworker/basic_fetch.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Cogworker
-  # Pops jobs off Redis lists. Queue names repeated in the process's queue
-  # list represent weight; since BRPOP itself scans its key list
+  # `config.fetch = :basic`: pops jobs off Redis lists with one blocking
+  # `BRPOP`. Once popped, a job exists only in the processor's memory (and
+  # its `cogworker:workers:<identity>` entry, which expires with the
+  # process), so a process that dies mid-job loses it — see ReliableFetch,
+  # the default, for the alternative.
+  #
+  # Queue names repeated in the process's queue list represent weight;
+  # since BRPOP itself scans its key list
   # strictly left-to-right, weighting is implemented by shuffling the
   # (already-expanded, so repeats survive) key list before every fetch cycle
   # — the more often a name appears, the more likely it lands first.
@@ -36,6 +44,33 @@ module Cogworker
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
 
+    # Nothing to do: BRPOP already removed the job from Redis.
+    def acknowledge(_work); end
+
+    # Returns a fetched-but-unstarted job to the end of its queue that's
+    # popped next.
+    def give_back(work)
+      Cogworker.config.redis { |c| c.rpush(RedisKeys.queue(work.queue), work.raw_job) }
+    end
+
+    # Shutdown past the drain timeout (`Manager#stop!`): the only record of
+    # a still-running job is its `cogworker:workers:<identity>` entry, so
+    # that's what gets pushed back — onto the end its queue is popped from,
+    # so it runs next. Returns how many. One entry that can't be read is
+    # logged and skipped, never allowed to cost the others their requeue.
+    def self.requeue_in_progress(identity)
+      Cogworker.config.redis do |c|
+        c.hvals(RedisKeys.workers(identity)).count do |raw|
+          entry = JSON.parse(raw)
+          c.rpush(RedisKeys.queue(entry.fetch('queue')), JSON.generate(entry.fetch('payload')))
+          true
+        rescue StandardError => e
+          Cogworker.logger.error { "couldn't requeue in-flight entry #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
+          false
+        end
+      end
+    end
+
     private
 
     def active_keys

data/lib/cogworker/cli.rb

@@ -8,8 +8,15 @@ module Cogworker
   # `-r/--require <file>`, `-C/--config <file>`, `-L/--logfile <file>`,
   # `-q/--queue <name>[,<weight>]` (repeatable).
   class CLI
+    # Read (first set wins) for the default environment, and all written
+    # back with the resolved one — whichever of them the app's own boot
+    # code reads (Rails: RAILS_ENV, then RACK_ENV; Rack apps: RACK_ENV/
+    # APP_ENV) sees the same value `-e` asked for.
+    ENV_VARS = %w[APP_ENV RAILS_ENV RACK_ENV].freeze
+
     def self.parse(argv)
-      options = { queues: [], require_path: nil, environment: ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development' }
+      environment = ENV_VARS.filter_map { |k| ENV.fetch(k, nil) }.first || 'development'
+      options = { queues: [], require_path: nil, environment: environment }
 
       OptionParser.new do |o|
         o.on('-e ENV', '--environment ENV') { |v| options[:environment] = v }
@@ -34,11 +41,14 @@ module Cogworker
     # forked child with the exact same flags it was started with.
     def run(argv)
       options = self.class.parse(argv)
+      # Before the config file (its ERB may read these) and the app code.
+      ENV_VARS.each { |k| ENV[k] = options[:environment] }
       file_config = ConfigLoader.load(options[:config_path])
 
       Cogworker.server_process!
       Cogworker.config.concurrency = options[:concurrency] || file_config[:concurrency]&.to_i || Cogworker.config.concurrency
       Cogworker.config.queues = options[:queues].any? ? options[:queues] : (file_config[:queues] || Cogworker.config.queues)
+      Cogworker.config.fetch = file_config[:fetch] if file_config[:fetch]
 
       redirect_logfile(options[:logfile]) if options[:logfile]
       require_app_code(options[:require_path]) if options[:require_path]

data/lib/cogworker/client.rb

@@ -50,7 +50,7 @@ module Cogworker
           else
             job['enqueued_at'] = Time.now.to_f
             conn.multi do |pipeline|
-              pipeline.sadd(RedisKeys::QUEUES, job['queue'])
+              pipeline.sadd?(RedisKeys::QUEUES, job['queue'])
               pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
             end
           end

data/lib/cogworker/config.rb

@@ -6,7 +6,9 @@ module Cogworker
   # `configure_client` calls, so every block sees and extends the same
   # server/client middleware chains.
   class Config
-    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options
+    FETCH_MODES = %i[reliable basic].freeze
+
+    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -31,11 +33,34 @@ module Cogworker
       # set higher/lower to match how long a unique job might legitimately
       # run plus however long its retries can take.
       @unique_lock_ttl = 24 * 60 * 60
+      # How processors take jobs off their queues — `:reliable` (ReliableFetch,
+      # needs Redis >= 6.2; a job survives its process dying) or `:basic`
+      # (BasicFetch, one blocking BRPOP; a job is lost if its process dies).
+      # On an older Redis, `:reliable` falls back to `:basic` with a warning
+      # (Manager#resolve_fetch_class).
+      @fetch = :reliable
       register_default_middleware
     end
 
+    def fetch=(mode)
+      resolved = mode.to_s.to_sym
+      unless FETCH_MODES.include?(resolved)
+        raise ArgumentError, "fetch must be one of #{FETCH_MODES.join(', ')}, got #{mode.inspect}"
+      end
+
+      @fetch = resolved
+    end
+
+    # Also drops an already-built pool: `redis_pool` is memoized on first
+    # use, so without this a `redis =` after anything had touched Redis
+    # (an init file configuring twice, a test switching databases) was
+    # silently ignored. The old pool's connections are closed as they're
+    # checked back in, so a caller still holding one finishes normally.
     def redis=(options)
       @redis_options = options
+      old_pool = @redis_pool
+      @redis_pool = nil
+      old_pool&.shutdown(&:close)
     end
 
     def redis_pool

data/lib/cogworker/heartbeat.rb

@@ -20,8 +20,13 @@ module Cogworker
       @manager = manager
     end
 
+    # The first beat is synchronous, before any processor fetches a job:
+    # ReliableFetch.recover_orphans treats an in-progress list with no live
+    # heartbeat behind it as a dead process's, so this process must be
+    # visibly alive before its own list can have anything on it.
     def start!
       @stopping = false
+      beat_safely
       @beat_thread = Thread.new { beat_loop }
       @signal_thread = Thread.new { subscribe_loop }
     end
@@ -29,11 +34,18 @@ module Cogworker
     # Tears down both background threads, not just the Redis keys — leaving
     # them running would leak a thread (and a dedicated pub/sub connection)
     # per Heartbeat instance for the remaining life of the process.
+    #
+    # The subscriber thread is killed *before* its connection is closed, and
+    # never via `unsubscribe`: on redis-rb 4.x, `unsubscribe` from another
+    # thread waits on the client's monitor, which the subscriber thread
+    # holds for as long as it sits in its blocking read — so it hung
+    # forever, and with it every `TERM`. Killing the thread releases the
+    # monitor; closing the now-idle connection afterwards is safe on both
+    # 4.x and 5.x.
     def stop!
       @stopping = true
       @beat_thread&.kill
-      @subscribe_client&.unsubscribe
-      @signal_thread&.kill
+      stop_subscriber!
       cleanup_presence!
     rescue StandardError
       nil
@@ -41,20 +53,38 @@ module Cogworker
 
     private
 
+    def stop_subscriber!
+      if @signal_thread && @signal_thread != Thread.current
+        @signal_thread.kill
+        @signal_thread.join(1)
+      end
+      @subscribe_client&.close
+    rescue StandardError
+      nil
+    end
+
     def cleanup_presence!
       Cogworker.config.redis do |c|
         c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
-        c.srem(RedisKeys::PROCESSES, Cogworker.identity)
+        c.srem?(RedisKeys::PROCESSES, Cogworker.identity)
       end
     end
 
+    # A failed beat (Redis blip, failover) is logged and retried on the next
+    # interval, never allowed to end the loop: a dead heartbeat thread
+    # meant this process silently vanished from ProcessSet/the Workers tab
+    # once its TTL ran out, while still running jobs.
     def beat_loop
       loop do
-        beat
         sleep(INTERVAL)
+        beat_safely
       end
+    end
+
+    def beat_safely
+      beat
     rescue StandardError => e
-      Cogworker.logger.error { "Heartbeat died: #{e.class}: #{e.message}" }
+      Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
     end
 
     def beat
@@ -69,13 +99,31 @@ module Cogworker
       }
 
       Cogworker.config.redis do |c|
-        c.sadd(RedisKeys::PROCESSES, identity)
+        c.sadd?(RedisKeys::PROCESSES, identity)
         c.hset(RedisKeys.process(identity),
                'info', JSON.generate(info),
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
         c.expire(RedisKeys.workers(identity), TTL)
+        touch_running_periodic_locks(c, identity)
+      end
+    end
+
+    # Keeps every in-flight `until_executed` periodic run's
+    # `periodic:running:<pjid>` lock alive for as long as this process is —
+    # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
+    # Hash the Workers tab does, so it covers exactly the jobs actually
+    # executing here right now.
+    def touch_running_periodic_locks(conn, identity)
+      conn.hvals(RedisKeys.workers(identity)).each do |raw|
+        job = JSON.parse(raw)['payload']
+        next unless job.is_a?(Hash) && job['periodic_pjid']
+
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock::ACTIVE_TTL, conn)
+      rescue StandardError => e
+        # Per entry: one bad entry mustn't leave the others' locks to expire.
+        Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
       end
     end
 

data/lib/cogworker/job.rb

@@ -50,6 +50,15 @@ module Cogworker
         Testing.jobs_for(name).clear
       end
 
+      # Run this class's recorded fake-mode jobs — see `Testing.perform_one`.
+      def perform_one
+        Testing.perform_one(name)
+      end
+
+      def drain
+        Testing.drain(name)
+      end
+
       private
 
       def job_payload(extra)

data/lib/cogworker/job_util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'json'
 require 'securerandom'
 
 module Cogworker
@@ -56,5 +57,60 @@ module Cogworker
     def terminal_failure?(job)
       job['retry_count'].to_i >= max_retries(job)
     end
+
+    # The job Hash a raw `schedule`/`retry`/`dead` entry would requeue as —
+    # or `nil` if it can't be: not JSON, not an object, or no String
+    # `queue` to push it onto (e.g. an `unparseable_job` wrapper, whose
+    # `queue` may be nil).
+    def requeueable(raw)
+      job = JSON.parse(raw)
+      job.is_a?(Hash) && job['queue'].is_a?(String) ? job : nil
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+
+    # Validate, then claim, then requeue — in that order, so an entry that
+    # can't be requeued is never `zrem`'d and lost (it stays where it is for
+    # the caller to deal with). Returns `:requeued`, `:gone` (someone else
+    # won the `zrem` first) or `:invalid`. The one implementation behind
+    # `Scheduled#graduate` and every Web UI retry action.
+    def claim_and_requeue(conn, set, raw)
+      job = requeueable(raw)
+      return :invalid unless job
+      return :gone unless conn.zrem(set, raw)
+
+      requeue(conn, job)
+      :requeued
+    end
+
+    # Puts a job taken off `cogworker:schedule`/`retry`/`dead` (the caller
+    # has already won its `zrem`) back onto its queue — the one shared
+    # implementation behind `Scheduled#graduate` and the Web UI's retry
+    # actions. Re-stamps `enqueued_at`: queue latency is measured from it,
+    # and a retried job still carried its original push time (a scheduled
+    # one, none at all), inflating the queue's latency by its whole history.
+    def requeue(conn, job)
+      job['enqueued_at'] = Time.now.to_f
+      conn.multi do |tx|
+        tx.sadd?(RedisKeys::QUEUES, job['queue'])
+        tx.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
+      end
+    end
+
+    UNPARSEABLE_CLASS = '(unparseable)'
+
+    # A payload that isn't a JSON object can't be run, retried or even
+    # attributed to a class. Wrapped into a regular job Hash (the Web UI
+    # parses every Dead entry as one) that keeps the original bytes under
+    # `raw_payload`, for whoever finds one to bury it in `dead`.
+    def unparseable_job(raw, queue:, error:)
+      now = Time.now.to_f
+      {
+        'class' => UNPARSEABLE_CLASS, 'args' => [], 'queue' => queue, 'jid' => SecureRandom.hex(12),
+        'retry' => false, 'retry_count' => 0, 'enqueued_at' => now, 'failed_at' => now,
+        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000],
+        'raw_payload' => raw.to_s[0, 100_000]
+      }
+    end
   end
 end

data/lib/cogworker/launcher.rb

@@ -22,9 +22,10 @@ module Cogworker
     def run
       Cogworker.reset_identity!
       install_signal_traps
+      resolve_periodic_classes
+      @heartbeat.start! # first: see Heartbeat#start!
       @manager.start!
       @scheduled.start!
-      @heartbeat.start!
       @ticker.start!
       log_startup_info
       watch_signals
@@ -43,6 +44,24 @@ module Cogworker
 
     private
 
+    # Resolves every periodic entry's class here, on the main thread, before
+    # any processor thread exists. `Processor#build_worker` otherwise does
+    # the first `const_get` lazily on whichever processor thread picks the
+    # job up — and with an app autoloader that isn't thread-safe, two
+    # threads racing to autoload the same class (or its base class) can see
+    # it half-defined (`undefined method 'perform'`). A class that can't be
+    # resolved is only logged: every run of that entry will fail the same
+    # way (and show up in Dead/History), but it shouldn't keep the rest of
+    # the process from booting. Regular job classes aren't known up front —
+    # eager-load the app in the init file for those (see README).
+    def resolve_periodic_classes
+      @config.periodic_manager.entries.map(&:class_name).uniq.each do |name|
+        Object.const_get(name)
+      rescue NameError => e
+        Cogworker.logger.warn { "periodic job class #{name} can't be resolved: #{e.message}" }
+      end
+    end
+
     # Logged once all components are up, so the operator's console shows a
     # single summary (version/identity/pid/concurrency/queues/redis target)
     # right at the point the process is actually ready to pull work, rather

data/lib/cogworker/manager.rb

@@ -4,8 +4,11 @@ module Cogworker
   # Owns the `concurrency`-sized thread pool of Processors for one OS
   # process, and the quiet/stop state they all poll.
   class Manager
+    attr_reader :fetch_class
+
     def initialize(config = Cogworker.config)
       @config = config
+      @fetch_class = resolve_fetch_class
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -49,10 +52,13 @@ module Cogworker
       @busy_mutex.synchronize { @busy_count -= 1 }
     end
 
-    # Waits (up to `timeout` seconds) for in-flight jobs to finish, then
-    # returns. Does not force-kill lingering threads past the deadline — a
-    # thread stuck past the deadline is the caller's (Launcher's) problem to
-    # decide whether to hard-exit the process.
+    # Waits (up to `timeout` seconds) for in-flight jobs to finish, then puts
+    # any job still unfinished back on its queue and returns. Does not
+    # force-kill lingering threads past the deadline — a thread stuck past
+    # the deadline is the caller's (Launcher's) problem to decide whether to
+    # hard-exit the process; its job is already safely requeued by then,
+    # rather than lost with the process (it may therefore run twice, if the
+    # thread does still finish it).
     def stop!(timeout: 25)
       @quiet = true
       @stopping = true
@@ -61,10 +67,43 @@ module Cogworker
         remaining = deadline - monotonic_now
         p.thread&.join([remaining, 0].max)
       end
+      wait_for_fetching_processors
+      requeue_unfinished
     end
 
     private
 
+    # A processor that isn't running a job is at most one fetch timeout away
+    # from noticing `stopping?` and exiting — but until it does, it can
+    # still pop a job (BRPOP blocks for up to BasicFetch::TIMEOUT), and one
+    # popped after `requeue_unfinished` had already run would be lost with
+    # the process. Waiting those out first means the only threads left
+    # alive are ones stuck inside a job, which never fetch again.
+    def wait_for_fetching_processors
+      deadline = monotonic_now + BasicFetch::TIMEOUT + 1
+      @processors.reject(&:busy?).each do |p|
+        p.thread&.join([deadline - monotonic_now, 0].max)
+      end
+    end
+
+    def requeue_unfinished
+      moved = @fetch_class.requeue_in_progress(Cogworker.identity)
+      Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
+    rescue StandardError => e
+      Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }
+    end
+
+    def resolve_fetch_class
+      return BasicFetch if @config.fetch == :basic
+      return ReliableFetch if ReliableFetch.supported?
+
+      Cogworker.logger.warn do
+        "config.fetch = :reliable needs Redis >= #{ReliableFetch::MIN_REDIS_VERSION}; falling back to :basic " \
+          '(a job is lost if its process dies mid-job)'
+      end
+      BasicFetch
+    end
+
     # Fully qualified: inside the Cogworker namespace, a bare `Process` would
     # resolve to Cogworker::Process (the ProcessSet entry wrapper in api.rb),
     # not the ::Process kernel module.

data/lib/cogworker/owned_key.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Compare-and-act on a lock key whose value is its owner's jid
+  # (`periodic:running:<pjid>`, `cogworker:unique:<digest>`): the EXPIRE/DEL
+  # only happens if the key still holds that exact jid, atomically — so a
+  # stray older run can never shorten, extend or release a lock that has
+  # since passed to a newer one.
+  module OwnedKey
+    EXPIRE_IF_OWNED = <<~LUA
+      if redis.call("GET", KEYS[1]) == ARGV[1] then
+        return redis.call("EXPIRE", KEYS[1], ARGV[2])
+      end
+      return 0
+    LUA
+
+    DELETE_IF_OWNED = <<~LUA
+      if redis.call("GET", KEYS[1]) == ARGV[1] then
+        return redis.call("DEL", KEYS[1])
+      end
+      return 0
+    LUA
+
+    module_function
+
+    def expire(key, owner, ttl, conn = nil)
+      run(conn, EXPIRE_IF_OWNED, key, [owner, ttl.ceil])
+    end
+
+    def delete(key, owner, conn = nil)
+      run(conn, DELETE_IF_OWNED, key, [owner])
+    end
+
+    def run(conn, script, key, argv)
+      return conn.eval(script, keys: [key], argv: argv) if conn
+
+      Cogworker.config.redis { |c| c.eval(script, keys: [key], argv: argv) }
+    end
+  end
+end

data/lib/cogworker/periodic/claim.lua

@@ -4,12 +4,18 @@
 -- ARGV[1] = slot (epoch int)
 -- ARGV[2] = unique mode ("until_executed" or "")
 -- ARGV[3] = lock TTL (seconds)
+-- ARGV[4] = jid the caller will push this slot's job under
+-- ARGV[5] = running-lock TTL (seconds)
 --
 -- Returns 1 if this call won the claim for this slot (the caller should
 -- enqueue the job), 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
+--
+-- For `until_executed`, the running lock is written here, in the same
+-- atomic step as the claim and before the job exists anywhere — see
+-- Periodic::RunningLock for why it must never be written after the push.
 if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
   return 0
 end
@@ -24,4 +30,7 @@ if not redis.call("SET", KEYS[3], "1", "NX", "EX", ARGV[3]) then
 end
 
 redis.call("SET", KEYS[2], ARGV[1])
+if ARGV[2] == "until_executed" then
+  redis.call("SET", KEYS[1], ARGV[4], "EX", ARGV[5])
+end
 return 1

data/lib/cogworker/periodic/release_middleware.rb

@@ -2,25 +2,43 @@
 
 module Cogworker
   module Periodic
-    # Releases the `periodic:running:<pjid>` lock (used only for
-    # `unique: :until_executed` entries) on success, or on the terminal
-    # failed attempt. A no-op for any job that isn't periodic-scheduled
-    # (`job['periodic_pjid']` absent). Registered unconditionally in every
-    # Config, not gated on whether `config.periodic` is actually used, so it
-    # never becomes a hidden dependency of the (separate) job status layer.
+    # Drives the server side of the `periodic:running:<pjid>` lock (used
+    # only for `unique: :until_executed` entries; see RunningLock for the
+    # whole lifecycle): shortens it to RunningLock::ACTIVE_TTL when the run
+    # starts, and releases it on success or on the terminal failed attempt
+    # (a failure that will be retried is `Processor#route_failure`'s to
+    # extend — only it knows the retry delay).
+    # A no-op for any job that isn't periodic-scheduled (`job['periodic_pjid']`
+    # absent) — and every call is a no-op on a lock owned by another jid, so
+    # it's harmless for a non-`until_executed` entry too. Registered
+    # unconditionally in every Config, not gated on whether
+    # `config.periodic` is actually used, so it never becomes a hidden
+    # dependency of the (separate) job status layer.
     class ReleaseMiddleware
       def call(_worker, job, _queue)
-        yield
-        release(job) if job['periodic_pjid']
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        release(job) if job['periodic_pjid'] && JobUtil.terminal_failure?(job)
-        raise e
+        pjid = job['periodic_pjid']
+        return yield unless pjid
+
+        shorten_lock(pjid, job['jid'])
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          RunningLock.release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
+          raise e
+        end
+        RunningLock.release(pjid, job['jid'])
       end
 
       private
 
-      def release(job)
-        Cogworker.config.redis { |c| c.del(RedisKeys.periodic_running(job['periodic_pjid'])) }
+      # Best-effort: failing here must not stop the job from running (it
+      # would be routed to retry/dead as if it had failed). If it does fail,
+      # the lock just keeps its longer queued TTL until this process's next
+      # heartbeat shortens it — within Heartbeat::INTERVAL.
+      def shorten_lock(pjid, jid)
+        RunningLock.touch(pjid, jid, RunningLock::ACTIVE_TTL)
+      rescue StandardError => e
+        Cogworker.logger.warn { "couldn't shorten periodic lock for #{pjid}: #{e.class}: #{e.message}" }
       end
     end
   end

data/lib/cogworker/periodic/running_lock.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Periodic
+    # `periodic:running:<pjid>` — the "an earlier run of this entry hasn't
+    # finished yet" lock behind `unique: :until_executed` periodic entries.
+    #
+    # Lifecycle (every write keyed on the owning jid, so a stray older run
+    # can never shorten, extend or release a newer run's lock):
+    # - set by `claim.lua`, atomically with the slot claim itself, *before*
+    #   the job is pushed — setting it after the push let a fast job finish
+    #   (and release a lock that didn't exist yet) before the lock was
+    #   written, wedging the entry forever. TTL = `queued_ttl`, covering
+    #   however long the job may wait in its queue.
+    # - shortened to ACTIVE_TTL once a processor starts it, then kept alive
+    #   by that process's Heartbeat every beat — so a run whose process dies
+    #   (OOM, SIGKILL, shutdown past the drain timeout) frees the entry
+    #   within ACTIVE_TTL instead of never.
+    # - extended on a non-terminal failure to cover the retry backoff plus
+    #   `queued_ttl` (`Processor#extend_locks_for_retry` — the job now waits
+    #   in `cogworker:retry`, not on a process).
+    # - deleted on success or on the terminal failure.
+    module RunningLock
+      ACTIVE_TTL = 60
+
+      module_function
+
+      # Whole seconds: it goes straight into `SET ... EX` in claim.lua, which
+      # rejects a non-integer (a fractional `unique_lock_ttl` used to make
+      # every claim — and so every tick — fail).
+      def queued_ttl
+        Cogworker.config.unique_lock_ttl.to_f.ceil
+      end
+
+      def touch(pjid, jid, ttl, conn = nil)
+        OwnedKey.expire(RedisKeys.periodic_running(pjid), jid, ttl, conn)
+      end
+
+      def release(pjid, jid, conn = nil)
+        OwnedKey.delete(RedisKeys.periodic_running(pjid), jid, conn)
+      end
+    end
+  end
+end