cogworker 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13e1ad0fec77124674f55dc3e1f7213bd2a73834ebb474261ed82a337fc1bb3e
-  data.tar.gz: ed51fed2854035c83016b3180fa7872772e6f60bb2dd84e44089485a536f81e4
+  metadata.gz: 680a06059bb5853e9d7843ff6d5313d7cd9431b222b846b496c9ca965d6b8844
+  data.tar.gz: fd688cdc23fa56e86e6d4c50a7f319e98698d4c6166d8213359c38eb0f94aa71
 SHA512:
-  metadata.gz: 0d78dd7a7c4bbdf8bfc8c8619467922e732879908562b1ac99fed71ce111a8a4925136e87a9c4e56cb5abed57867d76f06c5d929fcb822156d4fd0add7a182d9
-  data.tar.gz: 265d890a8f7d65b180dcb9a0f00f0d2ba6daf16fa9044f7bdb94b88b2cbeced1f6d61f5e11c86d5b107d281bbbc47591b54cb3fc6820f7a7fecc4732dec451d2
+  metadata.gz: fc5d3f1a358e0c826b2801ddd73f2b039b31b1e29616a38947da6df0cd85edfdf3b9b8c877e7123fc6283617f7f34317b9503c76db4eeb89a881677e335c5cfb
+  data.tar.gz: 486aff800ab67a15ce9c815f659a70ee95abcfd28218e7c862329a26a0f3643dce3a3c24dc79c539531a9b29b50f745e81482e617ba908c7d80ea131fe7b6d60

data/lib/cogworker/attempts.rb

@@ -24,7 +24,7 @@ module Cogworker
     def record(jid, attempt:, error:, outcome:)
       entry = JSON.generate(
         'attempt' => attempt, 'failed_at' => Time.now.to_f, 'outcome' => outcome,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000]
+        'error_class' => error.class.name, 'error_message' => JobUtil.error_message(error)
       )
       key = RedisKeys.job_attempts(jid)
       Cogworker.config.redis do |c|

data/lib/cogworker/basic_fetch.rb

@@ -17,8 +17,12 @@ module Cogworker
   class BasicFetch
     TIMEOUT = 2 # seconds; also how often a stopped/quieted processor notices and exits its fetch loop.
 
-    def initialize(queues)
+    # `stopping:` lets a fetcher cut its own waits short once shutdown has
+    # begun (Processor passes the Manager's `stopping?`), so a long idle
+    # pause can't hold up `Manager#stop!`.
+    def initialize(queues, stopping: -> { false })
       @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
+      @stopping = stopping
     end
 
     # Re-reads `cogworker:paused_queues` on every single fetch (not once at
@@ -33,7 +37,7 @@ module Cogworker
     def retrieve_work
       keys = active_keys
       if keys.empty?
-        sleep(TIMEOUT)
+        interruptible_sleep(TIMEOUT)
         return nil
       end
 
@@ -47,6 +51,12 @@ module Cogworker
     # Nothing to do: BRPOP already removed the job from Redis.
     def acknowledge(_work); end
 
+    # `Processor#interrupt`: the job ran, recording its failure failed —
+    # file it under `set` (retry/dead) at `score` instead.
+    def interrupt(_work, set, score, payload)
+      Cogworker.config.redis { |c| c.zadd(set, score, payload) }
+    end
+
     # Returns a fetched-but-unstarted job to the end of its queue that's
     # popped next.
     def give_back(work)
@@ -62,7 +72,7 @@ module Cogworker
       Cogworker.config.redis do |c|
         c.hvals(RedisKeys.workers(identity)).count do |raw|
           entry = JSON.parse(raw)
-          c.rpush(RedisKeys.queue(entry.fetch('queue')), JSON.generate(entry.fetch('payload')))
+          requeue_entry(c, entry.fetch('queue'), entry.fetch('payload'))
           true
         rescue StandardError => e
           Cogworker.logger.error { "couldn't requeue in-flight entry #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -71,8 +81,28 @@ module Cogworker
       end
     end
 
+    def self.requeue_entry(conn, queue, job)
+      queue_key = RedisKeys.queue(queue)
+      retake = job['periodic_until_executed'] && job['periodic_pjid'] && job['jid']
+      lock_key = retake ? RedisKeys.periodic_running(job['periodic_pjid']) : queue_key
+      LuaScript.run(conn, REQUEUE_ENTRY_SCRIPT, keys: [queue_key, lock_key],
+                                               argv: [JSON.generate(job), retake ? '1' : '0', job['jid'].to_s,
+                                             Periodic::RunningLock.queued_ttl])
+    end
+    private_class_method :requeue_entry
+
     private
 
+    def interruptible_sleep(seconds)
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + seconds
+      until @stopping.call
+        left = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        break if left <= 0
+
+        sleep([left, 0.25].min)
+      end
+    end
+
     def active_keys
       paused = Cogworker.config.redis { |c| c.smembers(RedisKeys::PAUSED_QUEUES) }
       return @queue_keys if paused.empty?
@@ -81,5 +111,18 @@ module Cogworker
     end
 
     UnitOfWork = Struct.new(:queue, :raw_job)
+
+    # KEYS[1] = the queue, KEYS[2] = the job's periodic running lock (or the
+    # queue again when there's none to take); ARGV[1] = payload, ARGV[2] =
+    # "1" to re-take that lock, ARGV[3] = jid, ARGV[4] = lock TTL. Requeue
+    # and lock in one step: as two commands, a failure in between left the
+    # job queued without its lock (and miscounted).
+    REQUEUE_ENTRY_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      redis.call("RPUSH", KEYS[1], ARGV[1])
+      if ARGV[2] == "1" then
+        retake_running_lock(KEYS[2], ARGV[3], ARGV[4])
+      end
+      return 1
+    LUA
   end
 end

data/lib/cogworker/config.rb

@@ -8,7 +8,8 @@ module Cogworker
   class Config
     FETCH_MODES = %i[reliable basic].freeze
 
-    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch
+    attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
+                :orphan_threshold
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -39,9 +40,36 @@ module Cogworker
       # On an older Redis, `:reliable` falls back to `:basic` with a warning
       # (Manager#resolve_fetch_class).
       @fetch = :reliable
+      # Longest pause (seconds) ReliableFetch makes between polls of empty
+      # queues — the worst-case delay before an idle worker notices a new
+      # job. See ReliableFetch::EMPTY_POLL_INTERVAL.
+      @fetch_idle_max_interval = 1.0
+      # How long (seconds) a process must have gone without a heartbeat
+      # before ReliableFetch.recover_orphans treats it as dead and requeues
+      # its in-progress jobs. Higher: fewer false "dead" verdicts on a live
+      # process that merely couldn't beat for a while (a Redis outage or
+      # failover, a long GVL-holding call) — each of which runs its jobs
+      # twice; lower: a really crashed process's jobs come back sooner.
+      @orphan_threshold = 300
       register_default_middleware
     end
 
+    def orphan_threshold=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @orphan_threshold = seconds
+    end
+
+    def fetch_idle_max_interval=(seconds)
+      unless seconds.is_a?(Numeric) && seconds.positive?
+        raise ArgumentError, "fetch_idle_max_interval must be a positive number of seconds, got #{seconds.inspect}"
+      end
+
+      @fetch_idle_max_interval = seconds
+    end
+
     def fetch=(mode)
       resolved = mode.to_s.to_sym
       unless FETCH_MODES.include?(resolved)

data/lib/cogworker/heartbeat.rb

@@ -15,6 +15,7 @@ module Cogworker
   class Heartbeat
     INTERVAL = 5
     TTL = 60
+    STARTUP_RETRY_INTERVAL = 1
 
     def initialize(manager)
       @manager = manager
@@ -23,12 +24,21 @@ module Cogworker
     # The first beat is synchronous, before any processor fetches a job:
     # ReliableFetch.recover_orphans treats an in-progress list with no live
     # heartbeat behind it as a dead process's, so this process must be
-    # visibly alive before its own list can have anything on it.
-    def start!
+    # visibly alive before its own list can have anything on it. With Redis
+    # unreachable, it keeps retrying (every STARTUP_RETRY_INTERVAL) rather
+    # than letting the Manager start fetching with no presence key — until
+    # it succeeds (returns true) or `abort_if` says to give up (returns
+    # false, e.g. a stop signal arrived meanwhile; nothing is started then).
+    def start!(abort_if: -> { false })
       @stopping = false
-      beat_safely
+      until beat_safely
+        return false if abort_if.call
+
+        sleep(STARTUP_RETRY_INTERVAL)
+      end
       @beat_thread = Thread.new { beat_loop }
       @signal_thread = Thread.new { subscribe_loop }
+      true
     end
 
     # Tears down both background threads, not just the Redis keys — leaving
@@ -67,6 +77,12 @@ module Cogworker
       Cogworker.config.redis do |c|
         c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
         c.srem?(RedisKeys::PROCESSES, Cogworker.identity)
+        # Clean shutdown: Manager#stop! already requeued this process's
+        # in-progress list, so there's nothing left for recovery to find.
+        if c.llen(RedisKeys.in_progress(Cogworker.identity)).zero?
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, Cogworker.identity)
+          c.hdel(RedisKeys::LAST_BEAT, Cogworker.identity)
+        end
       end
     end
 
@@ -83,12 +99,15 @@ module Cogworker
 
     def beat_safely
       beat
+      true
     rescue StandardError => e
       Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      false
     end
 
     def beat
       identity = Cogworker.identity
+      reliable = @manager.fetch_class == ReliableFetch
       info = {
         'hostname' => Cogworker.hostname,
         'pid' => ::Process.pid,
@@ -106,10 +125,24 @@ module Cogworker
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
         c.expire(RedisKeys.workers(identity), TTL)
+        if reliable
+          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c))
+        end
         touch_running_periodic_locks(c, identity)
       end
     end
 
+    # Redis's clock, not this host's (so host clock skew doesn't matter to
+    # orphan detection) — falling back to this host's where `TIME` isn't
+    # allowed (some proxies/managed services): better a skewed stamp than a
+    # process that can never complete its first beat.
+    def redis_now(conn)
+      conn.time.first
+    rescue Redis::CommandError
+      Time.now.to_i
+    end
+
     # Keeps every in-flight `until_executed` periodic run's
     # `periodic:running:<pjid>` lock alive for as long as this process is —
     # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
@@ -120,7 +153,7 @@ module Cogworker
         job = JSON.parse(raw)['payload']
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
-        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock::ACTIVE_TTL, conn)
+        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
       rescue StandardError => e
         # Per entry: one bad entry mustn't leave the others' locks to expire.
         Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -194,6 +227,7 @@ module Cogworker
     def remote_stop!
       @manager.stop!
       cleanup_presence!
+      Cogworker.flush_output!
       ::Process.exit!(true)
     end
   end

data/lib/cogworker/history/storage.rb

@@ -30,8 +30,8 @@ module Cogworker
         }
         if error
           entry['error_class'] = error.class.name
-          entry['error_message'] = error.message.to_s[0, 10_000]
-          entry['backtrace'] = (error.backtrace || []).first(200)
+          entry['error_message'] = JobUtil.error_message(error)
+          entry['backtrace'] = (error.backtrace || []).first(200).map { |line| JobUtil.safe_string(line) }
         end
 
         raw = JSON.generate(entry)

data/lib/cogworker/job_util.rb

@@ -64,37 +64,64 @@ module Cogworker
     # `queue` may be nil).
     def requeueable(raw)
       job = JSON.parse(raw)
-      job.is_a?(Hash) && job['queue'].is_a?(String) ? job : nil
+      job.is_a?(Hash) && job['queue'].is_a?(String) && !job['queue'].empty? ? job : nil
     rescue JSON::ParserError, TypeError
       nil
     end
 
-    # Validate, then claim, then requeue — in that order, so an entry that
-    # can't be requeued is never `zrem`'d and lost (it stays where it is for
-    # the caller to deal with). Returns `:requeued`, `:gone` (someone else
-    # won the `zrem` first) or `:invalid`. The one implementation behind
-    # `Scheduled#graduate` and every Web UI retry action.
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:queues, KEYS[3] = the queue;
+    # ARGV[1] = the entry as stored, ARGV[2] = queue name, ARGV[3] = the
+    # payload to push. Claim and push in one atomic step: as separate
+    # commands, a crash or dropped connection between the ZREM and the
+    # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
+    # by Lua's cjson (which would round large integers in args).
+    CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("SADD", KEYS[2], ARGV[2])
+      redis.call("LPUSH", KEYS[3], ARGV[3])
+      return 1
+    LUA
+
+    # Validate, then (atomically) claim and requeue — an entry that can't be
+    # requeued is never removed and lost (it stays where it is for the
+    # caller to deal with). Returns `:requeued`, `:gone` (someone else
+    # claimed it first) or `:invalid`. The one implementation behind
+    # `Scheduled#graduate` and every Web UI retry action. Re-stamps
+    # `enqueued_at`: queue latency is measured from it, and a retried job
+    # still carried its original push time (a scheduled one, none at all).
     def claim_and_requeue(conn, set, raw)
       job = requeueable(raw)
       return :invalid unless job
-      return :gone unless conn.zrem(set, raw)
 
-      requeue(conn, job)
-      :requeued
+      job['enqueued_at'] = Time.now.to_f
+      won = LuaScript.run(conn, CLAIM_AND_REQUEUE_SCRIPT,
+                          keys: [set, RedisKeys::QUEUES, RedisKeys.queue(job['queue'])],
+                          argv: [raw, job['queue'], JSON.generate(job)])
+      won == 1 ? :requeued : :gone
     end
 
-    # Puts a job taken off `cogworker:schedule`/`retry`/`dead` (the caller
-    # has already won its `zrem`) back onto its queue — the one shared
-    # implementation behind `Scheduled#graduate` and the Web UI's retry
-    # actions. Re-stamps `enqueued_at`: queue latency is measured from it,
-    # and a retried job still carried its original push time (a scheduled
-    # one, none at all), inflating the queue's latency by its whole history.
-    def requeue(conn, job)
-      job['enqueued_at'] = Time.now.to_f
-      conn.multi do |tx|
-        tx.sadd?(RedisKeys::QUEUES, job['queue'])
-        tx.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
-      end
+    ERROR_MESSAGE_LIMIT = 10_000
+
+    # Valid UTF-8, whatever came in — every string this gem stores from
+    # outside its control (an exception's message, a raw payload) goes
+    # through here before `JSON.generate`, which raises on invalid UTF-8.
+    # An unchecked `raise "bad \xff".b` used to make every attempt to record
+    # the failure itself raise, deterministically. Strings in another
+    # (valid) encoding are transcoded; binary/broken ones are scrubbed.
+    def safe_string(value, limit = nil)
+      str = value.to_s
+      str = if [Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII].include?(str.encoding)
+              str.dup.force_encoding(Encoding::UTF_8).scrub
+            else
+              str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+            end
+      limit ? str[0, limit] : str
+    end
+
+    def error_message(error)
+      safe_string(error.message, ERROR_MESSAGE_LIMIT)
     end
 
     UNPARSEABLE_CLASS = '(unparseable)'
@@ -108,8 +135,8 @@ module Cogworker
       {
         'class' => UNPARSEABLE_CLASS, 'args' => [], 'queue' => queue, 'jid' => SecureRandom.hex(12),
         'retry' => false, 'retry_count' => 0, 'enqueued_at' => now, 'failed_at' => now,
-        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000],
-        'raw_payload' => raw.to_s[0, 100_000]
+        'error_class' => error.class.name, 'error_message' => error_message(error),
+        'raw_payload' => safe_string(raw, 100_000)
       }
     end
   end

data/lib/cogworker/launcher.rb

@@ -13,8 +13,14 @@ module Cogworker
       @manager = Manager.new(config)
       @scheduled = Scheduled.new(@manager)
       @heartbeat = Heartbeat.new(@manager)
+      # The first tick waits for the first orphan check: after the whole
+      # fleet was down longer than a running `until_executed` job's lock
+      # lives, the ticker would otherwise claim that entry's next slot before
+      # recovery had requeued the interrupted run (and re-taken its lock) —
+      # two runs at once.
       @ticker = Periodic::Ticker.new(
-        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up
+        @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
+                                                   ready: -> { @scheduled.recovered_once? }
       )
       @signal_queue = ::Queue.new
     end
@@ -23,7 +29,8 @@ module Cogworker
       Cogworker.reset_identity!
       install_signal_traps
       resolve_periodic_classes
-      @heartbeat.start! # first: see Heartbeat#start!
+      return unless start_heartbeat
+
       @manager.start!
       @scheduled.start!
       @ticker.start!
@@ -44,6 +51,23 @@ module Cogworker
 
     private
 
+    # First, and blocking until it succeeds: see Heartbeat#start!. A stop
+    # signal arriving while Redis is still unreachable ends the wait (and
+    # the process) without ever starting anything; a quiet one is applied
+    # and the wait goes on.
+    def start_heartbeat
+      until @heartbeat.start!(abort_if: -> { !@signal_queue.empty? })
+        case @signal_queue.pop
+        when :stop
+          Cogworker.logger.info { 'Received stop signal before startup completed' }
+          stop!
+          return false
+        when :quiet then quiet!
+        end
+      end
+      true
+    end
+
     # Resolves every periodic entry's class here, on the main thread, before
     # any processor thread exists. `Processor#build_worker` otherwise does
     # the first `const_get` lazily on whichever processor thread picks the

data/lib/cogworker/logging.rb

@@ -7,6 +7,10 @@ module Cogworker
   # it), with a compact one-line-per-entry format including pid.
   module Logging
     def self.default_logger(io = $stdout)
+      # Unbuffered: processes here end via `Process.exit!` (swarm children,
+      # a remote stop), which skips Ruby's IO flush — with stdout a pipe
+      # (docker, systemd) the last lines logged before exit were lost.
+      io.sync = true if io.respond_to?(:sync=)
       logger = ::Logger.new(io)
       logger.level = ::Logger::INFO
       logger.formatter = proc do |severity, time, _progname, msg|

data/lib/cogworker/lua_script.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+
+module Cogworker
+  # Runs a Lua script by its SHA1 (EVALSHA) instead of shipping the whole
+  # source on every call, falling back to EVAL — which also loads it into
+  # the server's script cache for next time — on NOSCRIPT (first use, a
+  # restarted or failed-over server, SCRIPT FLUSH). Every script in this
+  # gem goes through here.
+  module LuaScript
+    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
+    SHAS_MUTEX = Mutex.new
+
+    module_function
+
+    def run(conn, source, keys: [], argv: [])
+      begin
+        return conn.evalsha(sha(source), keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+      end
+      conn.eval(source, keys: keys, argv: argv)
+    end
+
+    def sha(source)
+      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+    end
+  end
+end

data/lib/cogworker/manager.rb

@@ -4,11 +4,9 @@ module Cogworker
   # Owns the `concurrency`-sized thread pool of Processors for one OS
   # process, and the quiet/stop state they all poll.
   class Manager
-    attr_reader :fetch_class
-
     def initialize(config = Cogworker.config)
       @config = config
-      @fetch_class = resolve_fetch_class
+      @fetch_mutex = Mutex.new
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -20,6 +18,31 @@ module Cogworker
       @config.queues
     end
 
+    # Resolved on first use, not in the constructor (that needed Redis
+    # just to build a Manager). First use is normally the Heartbeat's first
+    # beat; if Redis can't be reached then, ReliableFetch.supported? raises
+    # and nothing is memoized — the beat fails and is retried, and this
+    # resolves again next time — rather than memoizing a guess.
+    def fetch_class
+      @fetch_mutex.synchronize { @fetch_class ||= resolve_fetch_class }
+    end
+
+    # The server turned out not to support ReliableFetch after all
+    # (ReliableFetch::UnsupportedError on a real fetch): every processor
+    # switches to BasicFetch from its next fetch on. Safe, since no job can
+    # be on the in-progress list — no LMOVE ever succeeded.
+    def fall_back_to_basic_fetch!(error)
+      @fetch_mutex.synchronize do
+        next if @fetch_class == BasicFetch
+
+        Cogworker.logger.warn do
+          "config.fetch = :reliable isn't supported by this Redis (#{error.message[0, 120]}); " \
+            'falling back to :basic (a job is lost if its process dies mid-job)'
+        end
+        @fetch_class = BasicFetch
+      end
+    end
+
     def start!
       @processors.each(&:start!)
     end
@@ -87,7 +110,7 @@ module Cogworker
     end
 
     def requeue_unfinished
-      moved = @fetch_class.requeue_in_progress(Cogworker.identity)
+      moved = fetch_class.requeue_in_progress(Cogworker.identity)
       Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
     rescue StandardError => e
       Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }

data/lib/cogworker/owned_key.rb

@@ -32,9 +32,9 @@ module Cogworker
     end
 
     def run(conn, script, key, argv)
-      return conn.eval(script, keys: [key], argv: argv) if conn
+      return LuaScript.run(conn, script, keys: [key], argv: argv) if conn
 
-      Cogworker.config.redis { |c| c.eval(script, keys: [key], argv: argv) }
+      Cogworker.config.redis { |c| LuaScript.run(c, script, keys: [key], argv: argv) }
     end
   end
 end

data/lib/cogworker/periodic/claim.lua

@@ -7,8 +7,10 @@
 -- ARGV[4] = jid the caller will push this slot's job under
 -- ARGV[5] = running-lock TTL (seconds)
 --
--- Returns 1 if this call won the claim for this slot (the caller should
--- enqueue the job), 0 otherwise. Composes two guards: `last_slot` stops a
+-- Returns {1, <previous last_slot, or "" if none>} if this call won the
+-- claim for this slot (the caller should enqueue the job — and, if that
+-- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
+-- value), 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
@@ -20,7 +22,8 @@ if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
   return 0
 end
 
-local last = tonumber(redis.call("GET", KEYS[2]) or "0")
+local previous = redis.call("GET", KEYS[2])
+local last = tonumber(previous or "0")
 if tonumber(ARGV[1]) <= last then
   return 0
 end
@@ -33,4 +36,4 @@ redis.call("SET", KEYS[2], ARGV[1])
 if ARGV[2] == "until_executed" then
   redis.call("SET", KEYS[1], ARGV[4], "EX", ARGV[5])
 end
-return 1
+return {1, previous or ""}

data/lib/cogworker/periodic/release_middleware.rb

@@ -4,7 +4,7 @@ module Cogworker
   module Periodic
     # Drives the server side of the `periodic:running:<pjid>` lock (used
     # only for `unique: :until_executed` entries; see RunningLock for the
-    # whole lifecycle): shortens it to RunningLock::ACTIVE_TTL when the run
+    # whole lifecycle): shortens it to `RunningLock.active_ttl` when the run
     # starts, and releases it on success or on the terminal failed attempt
     # (a failure that will be retried is `Processor#route_failure`'s to
     # extend — only it knows the retry delay).
@@ -36,7 +36,7 @@ module Cogworker
       # the lock just keeps its longer queued TTL until this process's next
       # heartbeat shortens it — within Heartbeat::INTERVAL.
       def shorten_lock(pjid, jid)
-        RunningLock.touch(pjid, jid, RunningLock::ACTIVE_TTL)
+        RunningLock.touch(pjid, jid, RunningLock.active_ttl)
       rescue StandardError => e
         Cogworker.logger.warn { "couldn't shorten periodic lock for #{pjid}: #{e.class}: #{e.message}" }
       end

data/lib/cogworker/periodic/running_lock.rb

@@ -12,22 +12,47 @@ module Cogworker
     #   (and release a lock that didn't exist yet) before the lock was
     #   written, wedging the entry forever. TTL = `queued_ttl`, covering
     #   however long the job may wait in its queue.
-    # - shortened to ACTIVE_TTL once a processor starts it, then kept alive
+    # - shortened to `active_ttl` once a processor starts it, then kept alive
     #   by that process's Heartbeat every beat — so a run whose process dies
-    #   (OOM, SIGKILL, shutdown past the drain timeout) frees the entry
-    #   within ACTIVE_TTL instead of never.
+    #   (OOM, SIGKILL, shutdown past the drain timeout) can't hold the entry
+    #   forever. `active_ttl` deliberately outlives the time it takes for
+    #   that dead process's job to be recovered (`config.orphan_threshold`
+    #   plus one orphan check), so the lock is still there, still owned by
+    #   that job, when the job is requeued — at which point `retake` (or ReliableFetch's
+    #   REQUEUE_SCRIPT) extends it again for the wait in the queue. A shorter
+    #   TTL let the lock lapse first: the ticker would enqueue the next slot
+    #   while the recovered run was still to come, i.e. two runs at once.
     # - extended on a non-terminal failure to cover the retry backoff plus
     #   `queued_ttl` (`Processor#extend_locks_for_retry` — the job now waits
     #   in `cogworker:retry`, not on a process).
     # - deleted on success or on the terminal failure.
     module RunningLock
-      ACTIVE_TTL = 60
+      # Lua, prepended to every script that puts a job back on its queue
+      # (ReliableFetch::REQUEUE_SCRIPT, BasicFetch::REQUEUE_ENTRY_SCRIPT), so
+      # the requeue and the lock move together in one atomic step. Extends
+      # the lock if still that job's, takes it if it lapsed, leaves it if a
+      # newer run holds it.
+      RETAKE_FUNCTION = <<~LUA
+        local function retake_running_lock(lock, jid, ttl)
+          local owner = redis.call("GET", lock)
+          if not owner then
+            redis.call("SET", lock, jid, "EX", ttl)
+          elseif owner == jid then
+            redis.call("EXPIRE", lock, ttl)
+          end
+        end
+      LUA
 
       module_function
 
       # Whole seconds: it goes straight into `SET ... EX` in claim.lua, which
       # rejects a non-integer (a fractional `unique_lock_ttl` used to make
       # every claim — and so every tick — fail).
+      # A method, not a constant: it follows `config.orphan_threshold`.
+      def active_ttl
+        (Cogworker.config.orphan_threshold + Scheduled::ORPHAN_CHECK_INTERVAL + 60).to_f.ceil
+      end
+
       def queued_ttl
         Cogworker.config.unique_lock_ttl.to_f.ceil
       end