cogworker 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 680a06059bb5853e9d7843ff6d5313d7cd9431b222b846b496c9ca965d6b8844
-  data.tar.gz: fd688cdc23fa56e86e6d4c50a7f319e98698d4c6166d8213359c38eb0f94aa71
+  metadata.gz: e908112cee4c6a9f98588b6cca08c292437d1713ec351ac6287e1f5ef605c452
+  data.tar.gz: 0c271ea997d7ddf51df5aa5a16b45cb08c0b3231f71dcdb253956160c589bde3
 SHA512:
-  metadata.gz: fc5d3f1a358e0c826b2801ddd73f2b039b31b1e29616a38947da6df0cd85edfdf3b9b8c877e7123fc6283617f7f34317b9503c76db4eeb89a881677e335c5cfb
-  data.tar.gz: 486aff800ab67a15ce9c815f659a70ee95abcfd28218e7c862329a26a0f3643dce3a3c24dc79c539531a9b29b50f745e81482e617ba908c7d80ea131fe7b6d60
+  metadata.gz: fb759fcff852bfa57315e4428250daeac0a0ae27771dc108e1879f3847616c073ce5d2fa8e079a77d73a51d7db3f18154d5cd6c7ea45c69b1282163cf817aa75
+  data.tar.gz: caea49e5286f9bf98e93c2931c2e8a77e18b9cce59e88c7ee7854f80bfe30b7b45649a148ed08c605ae87faeeb5ce87ae7f6fe23a523625db1728db1f215b120

data/lib/cogworker/basic_fetch.rb

@@ -68,7 +68,8 @@ module Cogworker
     # that's what gets pushed back — onto the end its queue is popped from,
     # so it runs next. Returns how many. One entry that can't be read is
     # logged and skipped, never allowed to cost the others their requeue.
-    def self.requeue_in_progress(identity)
+    # (`except:` is ReliableFetch's — there's no in-progress list here.)
+    def self.requeue_in_progress(identity, except: [], keep: []) # rubocop:disable Lint/UnusedMethodArgument
       Cogworker.config.redis do |c|
         c.hvals(RedisKeys.workers(identity)).count do |raw|
           entry = JSON.parse(raw)

data/lib/cogworker/config.rb

@@ -9,7 +9,7 @@ module Cogworker
     FETCH_MODES = %i[reliable basic].freeze
 
     attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
-                :orphan_threshold
+                :orphan_threshold, :max_orphanings
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -51,9 +51,21 @@ module Cogworker
       # failover, a long GVL-holding call) — each of which runs its jobs
       # twice; lower: a really crashed process's jobs come back sooner.
       @orphan_threshold = 300
+      # How many times a job may be recovered from a process that died
+      # while running it before it goes to dead instead (ReliableFetch). The
+      # count can't single out the culprit: every job running in a process
+      # that crashed is counted, so keep this above the number of crashes an
+      # innocent job might plausibly sit through.
+      @max_orphanings = 3
       register_default_middleware
     end
 
+    def max_orphanings=(count)
+      raise ArgumentError, "max_orphanings must be a positive Integer, got #{count.inspect}" unless count.is_a?(Integer) && count.positive?
+
+      @max_orphanings = count
+    end
+
     def orphan_threshold=(seconds)
       unless seconds.is_a?(Numeric) && seconds.positive?
         raise ArgumentError, "orphan_threshold must be a positive number of seconds, got #{seconds.inspect}"

data/lib/cogworker/heartbeat.rb

@@ -117,22 +117,35 @@ module Cogworker
         'rss_kb' => current_rss_kb
       }
 
+      # The presence key is what makes this process alive to everyone else
+      # (and what Heartbeat#start! waits for) — the one write a beat can't
+      # do without. Everything else is written on its own, best-effort: one
+      # shared key of the wrong type used to fail every beat of every
+      # process, so no new process could start anywhere in the fleet.
       Cogworker.config.redis do |c|
-        c.sadd?(RedisKeys::PROCESSES, identity)
         c.hset(RedisKeys.process(identity),
                'info', JSON.generate(info),
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
-        c.expire(RedisKeys.workers(identity), TTL)
+        secondary('process registry') { c.sadd?(RedisKeys::PROCESSES, identity) }
+        secondary('workers expiry') { c.expire(RedisKeys.workers(identity), TTL) }
         if reliable
-          c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
-          c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c))
+          secondary('in-progress registry') { c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
+          secondary('last-beat stamp') { c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c)) }
         end
         touch_running_periodic_locks(c, identity)
       end
     end
 
+    def secondary(what)
+      yield
+    rescue Redis::BaseConnectionError
+      raise # Redis itself is gone: the beat as a whole failed
+    rescue StandardError => e
+      Cogworker.logger.error { "Heartbeat: #{what} not written: #{e.class}: #{e.message}" }
+    end
+
     # Redis's clock, not this host's (so host clock skew doesn't matter to
     # orphan detection) — falling back to this host's where `TIME` isn't
     # allowed (some proxies/managed services): better a skewed stamp than a
@@ -145,12 +158,12 @@ module Cogworker
 
     # Keeps every in-flight `until_executed` periodic run's
     # `periodic:running:<pjid>` lock alive for as long as this process is —
-    # see Periodic::RunningLock. Reads the same `cogworker:workers:<identity>`
-    # Hash the Workers tab does, so it covers exactly the jobs actually
-    # executing here right now.
-    def touch_running_periodic_locks(conn, identity)
-      conn.hvals(RedisKeys.workers(identity)).each do |raw|
-        job = JSON.parse(raw)['payload']
+    # see Periodic::RunningLock. From the Manager's own in-memory record of
+    # what's running here, not `cogworker:workers` in Redis: a failover that
+    # lost that Hash would otherwise let a long run's lock lapse mid-run.
+    def touch_running_periodic_locks(conn, _identity)
+      @manager.running_jobs.each do |raw|
+        job = JSON.parse(raw)
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
         Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)

data/lib/cogworker/job_util.rb

@@ -37,11 +37,18 @@ module Cogworker
     # define `#to_i` (`nil.to_i` happens to be `0` — coincidentally correct
     # for absent — but `false.to_i` raises `NoMethodError` outright, which is
     # exactly the bug this method exists to not have).
+    #
+    # Anything else (a Hash, a non-numeric String…) means no retries — the
+    # cautious reading for a job with side effects — rather than raising
+    # from the middle of recording its failure.
     def max_retries(job)
-      case job['retry']
+      case (value = job['retry'])
       when false then 0
       when true, nil then DEFAULT_MAX_RETRY_ATTEMPTS
-      else job['retry'].to_i
+      when Integer then value
+      when Float then value.to_i
+      when String then Integer(value, exception: false) || 0
+      else 0
       end
     end
 
@@ -75,12 +82,17 @@ module Cogworker
     # commands, a crash or dropped connection between the ZREM and the
     # LPUSH lost the job. The new payload is built in Ruby, not re-encoded
     # by Lua's cjson (which would round large integers in args).
+    # Present? Push, then remove — never remove first: a script stopped by
+    # an error keeps the writes made before it, so a ZREM followed by a
+    # failing LPUSH (the queue key holding the wrong type) lost the entry.
+    # The `cogworker:queues` listing is best-effort.
     CLAIM_AND_REQUEUE_SCRIPT = <<~LUA
-      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+      if not redis.call("ZSCORE", KEYS[1], ARGV[1]) then
         return 0
       end
-      redis.call("SADD", KEYS[2], ARGV[2])
       redis.call("LPUSH", KEYS[3], ARGV[3])
+      redis.call("ZREM", KEYS[1], ARGV[1])
+      redis.pcall("SADD", KEYS[2], ARGV[2])
       return 1
     LUA
 
@@ -120,8 +132,14 @@ module Cogworker
       limit ? str[0, limit] : str
     end
 
+    # Never raises: an exception whose own `#message` raises (a broken
+    # custom exception class) used to escape from the middle of recording
+    # its failure — and, the job then counting as not-yet-run, it was handed
+    # back and re-run about once a second, forever.
     def error_message(error)
       safe_string(error.message, ERROR_MESSAGE_LIMIT)
+    rescue Exception # rubocop:disable Lint/RescueException
+      "#<#{error.class}>"
     end
 
     UNPARSEABLE_CLASS = '(unparseable)'

data/lib/cogworker/key_guard.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Cogworker
+  # Finds this gem's Redis keys holding the wrong type of value — which
+  # only a manual write or data corruption produces, but which then breaks
+  # every command on that key, deterministically: a queue that can't be
+  # pushed to, a `dead` set nothing can be filed in, a registry that fails
+  # every beat — and moves each such key out of the way, to
+  # `cogworker:quarantine:<key>:<time>:<random>` (kept QUARANTINE_TTL for a human to
+  # look at), with an error in the log. Everything then carries on with a
+  # fresh, empty key of the right type. Run by every process's Scheduled
+  # poller at boot and every CHECK_INTERVAL.
+  module KeyGuard
+    CHECK_INTERVAL = 60
+    QUARANTINE_TTL = 30 * 24 * 60 * 60
+    QUARANTINE_PREFIX = 'cogworker:quarantine:'
+
+    FIXED = {
+      RedisKeys::SCHEDULE => 'zset', RedisKeys::RETRY => 'zset', RedisKeys::DEAD => 'zset',
+      RedisKeys::QUEUES => 'set', RedisKeys::PROCESSES => 'set', RedisKeys::PAUSED_QUEUES => 'set',
+      RedisKeys::IN_PROGRESS_IDENTITIES => 'set', RedisKeys::LAST_BEAT => 'hash',
+      RedisKeys::REPEAT_ORPHANS => 'list', RedisKeys::UNSETTLED => 'list', RedisKeys::PERIODIC_SCHEDULE => 'hash',
+      RedisKeys::PERIODIC_DISABLED => 'set', RedisKeys::STATS_PROCESSED => 'string',
+      RedisKeys::STATS_FAILED => 'string'
+    }.freeze
+
+    # KEYS[1] = the key, KEYS[2] = its quarantine name; ARGV[1] = expected
+    # type, ARGV[2] = quarantine TTL. Re-checked here, atomically with the
+    # move, so a key fixed (or recreated) in between is left alone.
+    QUARANTINE_SCRIPT = <<~LUA
+      local actual = redis.call("TYPE", KEYS[1])["ok"]
+      if actual == "none" or actual == ARGV[1] then
+        return 0
+      end
+      redis.call("RENAME", KEYS[1], KEYS[2])
+      redis.call("EXPIRE", KEYS[2], ARGV[2])
+      return 1
+    LUA
+
+    module_function
+
+    # Returns the keys quarantined.
+    def check(queues: Cogworker.config.queues)
+      expected = expected_types(queues)
+      actual = Cogworker.config.redis do |c|
+        c.pipelined { |p| expected.each_key { |key| p.type(key) } }
+      end
+      expected.keys.zip(actual).filter_map do |key, type|
+        next if type == 'none' || type == expected[key]
+
+        key if quarantine(key, expected[key], type)
+      end
+    end
+
+    # The fixed keys, plus the per-queue and per-process ones currently
+    # listed in them (only where the listing itself is readable).
+    def expected_types(queues)
+      types = FIXED.dup
+      queue_names = Array(queues) | readable_members(RedisKeys::QUEUES)
+      queue_names.each { |q| types[RedisKeys.queue(q)] = 'list' }
+      readable_members(RedisKeys::IN_PROGRESS_IDENTITIES).each { |id| types[RedisKeys.in_progress(id)] = 'list' }
+      readable_members(RedisKeys::PROCESSES).each do |id|
+        types[RedisKeys.process(id)] = 'hash'
+        types[RedisKeys.workers(id)] = 'hash'
+      end
+      types
+    end
+
+    def readable_members(set)
+      Cogworker.config.redis { |c| c.smembers(set) }
+    rescue Redis::CommandError
+      [] # itself of the wrong type: quarantined this round, listed again next
+    end
+
+    def quarantine(key, expected, actual)
+      # Unique per move: a second quarantine of the same key within the same
+      # second must not overwrite (RENAME) the first one's copy.
+      target = "#{QUARANTINE_PREFIX}#{key}:#{Time.now.to_i}:#{SecureRandom.hex(4)}"
+      moved = Cogworker.config.redis do |c|
+        LuaScript.run(c, QUARANTINE_SCRIPT, keys: [key, target], argv: [expected, QUARANTINE_TTL])
+      end
+      return false unless moved == 1
+
+      Cogworker.logger.error do
+        "#{key} held a #{actual}, not a #{expected}: moved to #{target} (kept #{QUARANTINE_TTL / 86_400} days)"
+      end
+      true
+    end
+  end
+end

data/lib/cogworker/launcher.rb

@@ -13,14 +13,15 @@ module Cogworker
       @manager = Manager.new(config)
       @scheduled = Scheduled.new(@manager)
       @heartbeat = Heartbeat.new(@manager)
-      # The first tick waits for the first orphan check: after the whole
+      # The first tick waits for the first orphan check that really
+      # completed (Scheduled#ready_for_ticker?): after the whole
       # fleet was down longer than a running `until_executed` job's lock
       # lives, the ticker would otherwise claim that entry's next slot before
       # recovery had requeued the interrupted run (and re-taken its lock) —
       # two runs at once.
       @ticker = Periodic::Ticker.new(
         @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
-                                                   ready: -> { @scheduled.recovered_once? }
+                                                   ready: -> { @scheduled.ready_for_ticker? }
       )
       @signal_queue = ::Queue.new
     end

data/lib/cogworker/manager.rb

@@ -7,6 +7,16 @@ module Cogworker
     def initialize(config = Cogworker.config)
       @config = config
       @fetch_mutex = Mutex.new
+      # This process's own record of its jobs (raw payloads, as fetched):
+      # running right now (a count — two copies of one payload can run at
+      # once), and finished but not yet settled in Redis — `:ack` (the ack
+      # failed) or `[set, score, payload]` (the job ran and failed, and
+      # filing it in retry/dead failed too). In memory on purpose:
+      # ReliableFetch.reconcile trusts it over anything in Redis, which may
+      # be what's failing (or what lost its last writes in a failover).
+      @jobs_mutex = Mutex.new
+      @running_jobs = Hash.new(0)
+      @pending_settlements = {}
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
       @stopping = false
@@ -63,6 +73,34 @@ module Cogworker
       @stopping
     end
 
+    def job_started(raw)
+      @jobs_mutex.synchronize { @running_jobs[raw] += 1 }
+    end
+
+    def job_finished(raw)
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] -= 1
+        @running_jobs.delete(raw) unless @running_jobs[raw].positive?
+      end
+    end
+
+    def running_jobs
+      @jobs_mutex.synchronize { @running_jobs.keys }
+    end
+
+    # `how`: :ack, or [set, score, payload] — see ReliableFetch.settle.
+    def settle_later(raw, how)
+      @jobs_mutex.synchronize { @pending_settlements[raw] = how }
+    end
+
+    def settled(raw)
+      @jobs_mutex.synchronize { @pending_settlements.delete(raw) }
+    end
+
+    def pending_settlements
+      @jobs_mutex.synchronize { @pending_settlements.dup }
+    end
+
     def busy_count
       @busy_mutex.synchronize { @busy_count }
     end
@@ -109,8 +147,27 @@ module Cogworker
       end
     end
 
+    # Jobs that finished but aren't settled in Redis yet are still on the
+    # in-progress list. One more try at settling them first; then `except:`
+    # drops the ones only waiting for their ack (which is what dropping
+    # them is), in the same atomic step as requeueing the rest — a separate
+    # ack pass that failed used to leave them to be requeued and run again.
+    # Ones that couldn't be filed in retry/dead even now are handed over to
+    # UNSETTLED for any live process to file later (ReliableFetch.park_unsettled)
+    # — never requeued: they already ran. Any that can't even be handed
+    # over are left on the list (`keep:`): orphan recovery requeues them
+    # once this process is gone, at most `max_orphanings` times.
     def requeue_unfinished
-      moved = fetch_class.requeue_in_progress(Cogworker.identity)
+      keep = []
+      if fetch_class == ReliableFetch
+        ReliableFetch.settle_pending(Cogworker.identity, pending_settlements) { |raw| settled(raw) }
+        unfiled = pending_settlements.reject { |_raw, how| how == :ack }
+        keep = ReliableFetch.park_unsettled(Cogworker.identity, unfiled)
+        (unfiled.keys - keep).each { |raw| settled(raw) }
+      end
+      to_ack = pending_settlements.select { |_raw, how| how == :ack }.keys
+      moved = fetch_class.requeue_in_progress(Cogworker.identity, except: to_ack, keep: keep)
+      to_ack.each { |raw| settled(raw) }
       Cogworker.logger.warn { "requeued #{moved} job(s) unfinished at shutdown" } if moved.positive?
     rescue StandardError => e
       Cogworker.logger.error { "Requeueing unfinished jobs failed: #{e.class}: #{e.message}" }

data/lib/cogworker/periodic/ticker.rb

@@ -20,10 +20,14 @@ module Cogworker
       # Undoes a won claim whose push then failed, so the slot isn't lost:
       # puts `last_slot` back (only if it still is this slot — a later slot
       # may have been claimed since) and drops the per-slot NX lock.
-      # KEYS[1] = last_slot, KEYS[2] = slot lock; ARGV[1] = slot, ARGV[2] =
-      # previous last_slot ("" if none: then slot - 1, which still lets this
-      # slot fire, unlike deleting the key — with catch_up off, a missing
-      # last_slot would be re-primed to this very slot instead).
+      # KEYS[1] = last_slot, KEYS[2] = slot lock, KEYS[3] = running lock;
+      # ARGV[1] = slot, ARGV[2] = previous last_slot ("" if none: then
+      # slot - 1, which still lets this slot fire, unlike deleting the key —
+      # with catch_up off, a missing last_slot would be re-primed to this
+      # very slot instead), ARGV[3] = the jid the claim took the running lock
+      # under. Releasing that lock happens here too, in the same step: as a
+      # separate call made first, its failure (Redis down — the usual reason
+      # the push failed) skipped the rollback altogether.
       ROLLBACK_SCRIPT = <<~LUA
         if redis.call("GET", KEYS[1]) == ARGV[1] then
           if ARGV[2] == "" then
@@ -33,6 +37,12 @@ module Cogworker
           end
         end
         redis.call("DEL", KEYS[2])
+        -- Best-effort (an entry without until_executed has no lock; one of
+        -- the wrong type mustn't make an applied rollback count as failed
+        -- and be retried forever).
+        if redis.pcall("GET", KEYS[3]) == ARGV[3] then
+          redis.pcall("DEL", KEYS[3])
+        end
         return 1
       LUA
 
@@ -44,10 +54,19 @@ module Cogworker
         @last_checked_slot = {}
         @cron_cache = {}
         @pending_rollbacks = []
+        @failures = Hash.new(0) # pjid => consecutive failed ticks
+        @retry_at = {}          # pjid => monotonic time it may be tried again
       end
 
+      # Publishing is only for restart survival and the Web UI: its failure
+      # (Redis down at boot, the key holding the wrong type) is logged, and
+      # must not take the process down with it.
       def start!
-        publish_schedule!
+        begin
+          publish_schedule!
+        rescue StandardError => e
+          Cogworker.logger.error { "Periodic schedule not published: #{e.class}: #{e.message}" }
+        end
         @thread = Thread.new { run }
       end
 
@@ -96,53 +115,114 @@ module Cogworker
         retry_pending_rollbacks
         now = Time.now
         @entries.each do |entry|
-          slot = cron_for(entry).previous_time(now).to_i
-          next if @last_checked_slot[entry.pjid] == slot
+          next if backing_off?(entry.pjid)
 
-          jid = SecureRandom.hex(12)
-          previous = claim(entry, slot, jid) unless disabled?(entry)
-          enqueue(entry, slot, jid, previous) if previous
-          @last_checked_slot[entry.pjid] = slot
+          tick_entry(entry, now)
+          @failures.delete(entry.pjid)
+        rescue StandardError => e
+          # Per entry: one entry failing (its keys of the wrong type, its
+          # push failing) used to skip every entry after it, on every tick.
+          # Its slot isn't recorded as checked, so it's retried — after an
+          # exponential pause (`back_off`), not on every 5s tick, which made a
+          # deterministically failing entry hammer Redis and the log. Not for
+          # a lost connection, though: that's Redis being away, not the
+          # entry failing — backing off every entry on it left cron idle for
+          # minutes after Redis came back.
+          delay = e.is_a?(Redis::BaseConnectionError) ? TICK_INTERVAL : back_off(entry.pjid)
+          Cogworker.logger.error { "Periodic tick failed for #{entry.pjid} (next try in #{delay}s): #{e.class}: #{e.message}" }
         end
       end
 
-      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
-      # step entirely, short-circuiting before `claim` even runs, so
-      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
-      # while disabled: the Web UI's own "LastRun" column keeps showing the
-      # last time it *actually* ran, not a due-but-skipped slot, and
-      # re-enabling doesn't trigger a catch-up burst for every slot that
-      # was silently skipped in between. `@last_checked_slot` (this one
-      # Ticker instance's own in-memory dedup, unrelated to the Redis-
-      # persisted last_slot) still advances either way, exactly as it
-      # already did before this entry ever had a disabled state — it only
-      # stops this same tick loop from re-evaluating the same slot twice.
+      BACKOFF_MAX = 300
+
+      def backing_off?(key)
+        (at = @retry_at[key]) && ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < at
+      end
+
+      # 5s, 10s, 20s … up to BACKOFF_MAX; reset by the next success.
+      def back_off(key)
+        @failures[key] += 1
+        delay = [TICK_INTERVAL * (2**(@failures[key] - 1)), BACKOFF_MAX].min
+        @retry_at[key] = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + delay
+        delay
+      end
+
+      def tick_entry(entry, now)
+        slot = cron_for(entry).previous_time(now).to_i
+        return if @last_checked_slot[entry.pjid] == slot
+
+        jid = SecureRandom.hex(12)
+        previous = claim(entry, slot, jid) unless disabled?(entry)
+        enqueue(entry, slot, jid, previous) if previous
+        @last_checked_slot[entry.pjid] = slot
+      end
+
       # The usual reason a push fails — Redis unreachable — makes this fail
       # too; it's then kept and retried at the start of every tick until it
       # goes through (before that tick looks at the slot again), rather than
-      # just logged and the slot lost. (A push that timed out *after* Redis
-      # had applied it is the one case this turns into a second run of the
-      # slot: indistinguishable from one that never arrived.)
-      def rollback_claim(entry, slot, previous_slot)
+      # just logged and the slot lost. Returns true, or the error. (A push
+      # that timed out *after* Redis had applied it is the one case this
+      # turns into a second run of the slot: indistinguishable from one that
+      # never arrived.) Pending
+      # rollbacks live in this process's memory only: one still pending when
+      # the process restarts is lost — that slot is skipped, and an
+      # `until_executed` entry stays locked until its lock's TTL.
+      def rollback_claim(entry, slot, previous_slot, jid)
         Cogworker.config.redis do |c|
-          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot)]
-          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot])
+          keys = [RedisKeys.periodic_last_slot(entry.pjid), RedisKeys.periodic_lock(entry.pjid, slot),
+                  RedisKeys.periodic_running(entry.pjid)]
+          LuaScript.run(c, ROLLBACK_SCRIPT, keys: keys, argv: [slot, previous_slot, jid])
         end
         true
       rescue StandardError => e
         Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
-        @pending_rollbacks << [entry, slot, previous_slot]
-        false
+        @pending_rollbacks << [entry, slot, previous_slot, jid]
+        e # (truthy, but not `true`: the caller tells a lost connection from a failing rollback)
       end
 
+      # With the same backoff as a failing entry (keyed by the rollback): a
+      # rollback that keeps failing is retried less and less often, but
+      # always before that entry's slot is looked at again.
       def retry_pending_rollbacks
         pending = @pending_rollbacks
         @pending_rollbacks = []
-        pending.each { |args| rollback_claim(*args) }
+        pending.each do |args|
+          key = "rollback:#{args[0].pjid}:#{args[1]}"
+          next @pending_rollbacks << args if backing_off?(key)
+
+          result = rollback_claim(*args)
+          if result == true
+            @failures.delete(key)
+            @retry_at.delete(key)
+          elsif !result.is_a?(Redis::BaseConnectionError)
+            back_off(key) # only a rollback failing on its own backs off; see #tick
+          end
+        end
       end
 
+      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
+      # step entirely, short-circuiting before `claim` even runs, so
+      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
+      # while disabled: the Web UI's own "LastRun" column keeps showing the
+      # last time it *actually* ran, not a due-but-skipped slot, and
+      # re-enabling doesn't trigger a catch-up burst for every slot that
+      # was silently skipped in between. `@last_checked_slot` (this one
+      # Ticker instance's own in-memory dedup, unrelated to the Redis-
+      # persisted last_slot) still advances either way, exactly as it
+      # already did before this entry ever had a disabled state — it only
+      # stops this same tick loop from re-evaluating the same slot twice.
+      #
+      # A disabled-set that can't be read (the key holding the wrong type)
+      # counts as "not disabled", logged: it used to fail every entry's tick,
+      # stopping all cron. Losing a Disable is the lesser evil. A dropped
+      # connection still raises (the tick is then retried as a whole).
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
+      rescue Redis::BaseConnectionError
+        raise
+      rescue StandardError => e
+        Cogworker.logger.error { "Periodic: can't read #{RedisKeys::PERIODIC_DISABLED} (#{e.class}: #{e.message}); treating #{entry.pjid} as enabled" }
+        false
       end
 
       def cron_for(entry)
@@ -201,8 +281,7 @@ module Cogworker
         pushed = begin
           Client.push(job)
         rescue StandardError
-          RunningLock.release(entry.pjid, jid) if entry.until_executed?
-          rollback_claim(entry, slot, previous_slot)
+          rollback_claim(entry, slot, previous_slot, jid)
           raise
         end
         return unless entry.until_executed?