cogworker 0.5.0 → 0.6.0

data/lib/cogworker/client.rb

@@ -43,16 +43,29 @@ module Cogworker
         end
       end
 
+      # The queue listing only feeds the Web UI/stats; failing to update it
+      # is logged, never raised — whatever the error, Redis being away
+      # included: the job is already queued by then, and raising would make
+      # the caller push it a second time (see raw_push).
+      def register_queue(conn, queue)
+        conn.sadd?(RedisKeys::QUEUES, queue)
+      rescue StandardError => e
+        RedisErrors.report("#{RedisKeys::QUEUES} not updated", e) { queue }
+      end
+
       def raw_push(job)
         Cogworker.config.redis do |conn|
           if job['at']
             conn.zadd(RedisKeys::SCHEDULE, job['at'].to_f, JSON.generate(job))
           else
             job['enqueued_at'] = Time.now.to_f
-            conn.multi do |pipeline|
-              pipeline.sadd?(RedisKeys::QUEUES, job['queue'])
-              pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
-            end
+            # The push is the one write that matters, and goes first, on its
+            # own. They used to share a MULTI — which doesn't roll back: with
+            # `cogworker:queues` unwritable the job was queued anyway but the
+            # whole push raised, so callers retried it (the cron ticker did,
+            # every tick) and queued it again and again.
+            conn.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
+            register_queue(conn, job['queue'])
           end
         end
       end

data/lib/cogworker/config.rb

@@ -9,7 +9,7 @@ module Cogworker
     FETCH_MODES = %i[reliable basic].freeze
 
     attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options, :fetch, :fetch_idle_max_interval,
-                :orphan_threshold, :max_orphanings
+                :orphan_threshold, :max_orphanings, :periodic_replace
     attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
 
     def initialize
@@ -61,7 +61,10 @@ module Cogworker
     end
 
     def max_orphanings=(count)
-      raise ArgumentError, "max_orphanings must be a positive Integer, got #{count.inspect}" unless count.is_a?(Integer) && count.positive?
+      unless count.is_a?(Integer) && count.positive?
+        raise ArgumentError,
+              "max_orphanings must be a positive Integer, got #{count.inspect}"
+      end
 
       @max_orphanings = count
     end
@@ -130,7 +133,18 @@ module Cogworker
     # registers cron entries synchronously as it runs. No-op-safe: an app
     # that never calls this simply has an empty periodic_manager, and the
     # Ticker (started later, per process) has nothing to do.
-    def periodic(&block)
+    #
+    # `replace: true` clears what earlier calls registered first, and makes
+    # the Ticker replace the whole published `periodic:schedule` rather than
+    # adding to it — so an entry deleted from (or changed in) the schedule
+    # file disappears from the Web UI instead of lingering there forever.
+    # Opt-in: with processes that register *different* schedules against one
+    # Redis, each would wipe the others' entries off the Schedules tab.
+    def periodic(replace: false, &block)
+      if replace
+        periodic_manager.clear!
+        @periodic_replace = true
+      end
       block&.call(periodic_manager)
       periodic_manager
     end

data/lib/cogworker/deferred_releases.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Lock releases that failed (Redis away when a job finished) — kept here
+  # and retried on every heartbeat beat (Heartbeat#beat) until they go
+  # through, owner-checked like the original release. A failed release is
+  # best-effort so it can't fail the job, but left alone it held the lock
+  # until its TTL: an `until_executed` cron entry skipped slots for up to
+  # `RunningLock.active_ttl`, a unique job blocked its duplicates for up to
+  # `unique_lock_ttl`. In this process's memory only: one still pending when
+  # the process exits falls back to the lock's TTL.
+  module DeferredReleases
+    @mutex = Mutex.new
+    @pending = {} # lock key => owner jid
+
+    class << self
+      def add(key, owner)
+        @mutex.synchronize { @pending[key] = owner }
+      end
+
+      # Retries every pending release; stops at the first one Redis can't
+      # serve (no point hammering it), keeps whatever's left.
+      def retry_all(conn)
+        pending.each do |key, owner|
+          OwnedKey.delete(key, owner, conn) # (a no-op if it lapsed or changed hands meanwhile)
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+        rescue StandardError => e
+          raise if RedisErrors.unavailable?(e)
+
+          # Anything else (the key holding the wrong type, say) won't get
+          # better by retrying every 5s: dropped, logged once.
+          @mutex.synchronize { @pending.delete(key) if @pending[key] == owner }
+          Cogworker.logger.error { "deferred release of #{key} dropped: #{e.class}: #{e.message}" }
+        end
+      end
+
+      def pending
+        @mutex.synchronize { @pending.dup }
+      end
+
+      def reset!
+        @mutex.synchronize { @pending.clear }
+      end
+    end
+  end
+end

data/lib/cogworker/heartbeat.rb

@@ -99,9 +99,12 @@ module Cogworker
 
     def beat_safely
       beat
+      @beat_failed = false
+      RedisErrors.recovered('Heartbeat')
       true
     rescue StandardError => e
-      Cogworker.logger.error { "Heartbeat failed: #{e.class}: #{e.message}" }
+      @beat_failed = true
+      RedisErrors.report('Heartbeat', e)
       false
     end
 
@@ -128,21 +131,43 @@ module Cogworker
                'busy', @manager.busy_count.to_s,
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
-        secondary('process registry') { c.sadd?(RedisKeys::PROCESSES, identity) }
-        secondary('workers expiry') { c.expire(RedisKeys.workers(identity), TTL) }
-        if reliable
-          secondary('in-progress registry') { c.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
-          secondary('last-beat stamp') { c.hset(RedisKeys::LAST_BEAT, identity, redis_now(c)) }
-        end
+        write_registries(c, identity, reliable)
         touch_running_periodic_locks(c, identity)
+        # Not via #secondary: Redis failing here, after the presence key went
+        # through, doesn't make this beat a failed one.
+        BestEffort.call('Deferred lock releases') { DeferredReleases.retry_all(c) }
       end
     end
 
+    # Everything a beat writes besides the presence key — each on its own,
+    # best-effort (see #secondary).
+    def write_registries(conn, identity, reliable)
+      secondary('process registry') { conn.sadd?(RedisKeys::PROCESSES, identity) }
+      # The queues this process reads are listed even while empty: the Web
+      # UI shows them, and recovery can tell a queue nobody reads from one
+      # that's merely had nothing pushed yet.
+      secondary('queue registry') { conn.sadd(RedisKeys::QUEUES, @manager.queues.uniq) }
+      secondary('live queue registry') { mark_queues_live(conn) }
+      secondary('workers expiry') { conn.expire(RedisKeys.workers(identity), TTL) }
+      return unless reliable
+
+      secondary('in-progress registry') { conn.sadd?(RedisKeys::IN_PROGRESS_IDENTITIES, identity) }
+      secondary('last-beat stamp') { conn.hset(RedisKeys::LAST_BEAT, identity, redis_now(conn)) }
+    end
+
+    # Stamps this process's queues as read right now, and drops ones no
+    # process has stamped for a day (only ever a listing, never data).
+    def mark_queues_live(conn)
+      now = redis_now(conn)
+      conn.zadd(RedisKeys::LIVE_QUEUES, @manager.queues.uniq.map { |q| [now, q] })
+      conn.zremrangebyscore(RedisKeys::LIVE_QUEUES, '-inf', now - 86_400)
+    end
+
     def secondary(what)
       yield
-    rescue Redis::BaseConnectionError
-      raise # Redis itself is gone: the beat as a whole failed
     rescue StandardError => e
+      raise if RedisErrors.unavailable?(e) # Redis itself is away: the beat as a whole failed
+
       Cogworker.logger.error { "Heartbeat: #{what} not written: #{e.class}: #{e.message}" }
     end
 
@@ -161,12 +186,30 @@ module Cogworker
     # see Periodic::RunningLock. From the Manager's own in-memory record of
     # what's running here, not `cogworker:workers` in Redis: a failover that
     # lost that Hash would otherwise let a long run's lock lapse mid-run.
+    # (One narrow race remains: a release landing between this beat's
+    # snapshot of `performing_jobs` and its Lua call, on the first beat
+    # after a failed one, can be undone — the lock then lapses on
+    # `active_ttl`. Not worth a cross-thread lock on every beat.)
+    # Only jobs still inside `perform` (`performing_jobs`): once it returns,
+    # Periodic::ReleaseMiddleware has released the lock, and refreshing it
+    # again would bring it back. A missing lock is only *re-taken* on the
+    # first beat after a failed one — i.e. after Redis was away, possibly
+    # long enough for the lock to lapse; otherwise a missing lock means it
+    # was released, and stays released.
     def touch_running_periodic_locks(conn, _identity)
-      @manager.running_jobs.each do |raw|
+      recovering = @beat_failed
+      @manager.performing_jobs.each do |raw|
         job = JSON.parse(raw)
         next unless job.is_a?(Hash) && job['periodic_pjid']
 
-        Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        if recovering && job['periodic_until_executed']
+          # Re-taken, not just extended: after Redis was away longer than
+          # `active_ttl`, the lock is gone while the run is still going, and
+          # the ticker could start the next slot alongside it.
+          Periodic::RunningLock.retake(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        else
+          Periodic::RunningLock.touch(job['periodic_pjid'], job['jid'], Periodic::RunningLock.active_ttl, conn)
+        end
       rescue StandardError => e
         # Per entry: one bad entry mustn't leave the others' locks to expire.
         Cogworker.logger.error { "couldn't refresh periodic lock for #{raw.to_s[0, 200]}: #{e.class}: #{e.message}" }
@@ -199,6 +242,7 @@ module Cogworker
     def subscribe_loop
       @subscribe_client = ::Redis.new(RedisConnection.client_options(Cogworker.config.redis_options))
       @subscribe_client.subscribe(RedisKeys.signal(Cogworker.identity)) do |on|
+        on.subscribe { |*| RedisErrors.recovered('Signal subscriber') }
         on.message do |_channel, message|
           dispatch(message)
         end
@@ -206,7 +250,7 @@ module Cogworker
     rescue StandardError => e
       return if @stopping
 
-      Cogworker.logger.error { "Signal subscriber died: #{e.class}: #{e.message}" }
+      RedisErrors.report('Signal subscriber', e)
       sleep(1)
       retry
     end

data/lib/cogworker/history/middleware.rb

@@ -5,13 +5,17 @@ module Cogworker
     # Server middleware: times every run and records it via Storage,
     # success or failure, regardless of what other middleware/worker code does.
     class Middleware
+      # Recording is best-effort (BestEffort): only the job's own exception
+      # is re-raised, never one from writing its history.
       def call(_worker, job, queue)
         started_at = Time.now.to_f
-        yield
-        Storage.record(job, queue, started_at, Time.now.to_f, 'success')
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e)
-        raise e
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e) }
+          raise e
+        end
+        BestEffort.call('History') { Storage.record(job, queue, started_at, Time.now.to_f, 'success') }
       end
     end
   end

data/lib/cogworker/job_util.rb

@@ -142,6 +142,23 @@ module Cogworker
       "#<#{error.class}>"
     end
 
+    # A job on its way to dead outside the usual path (no server middleware
+    # around it to do this): releases its `until_executed` locks, owner-
+    # checked, each on its own — so the entry / the unique key isn't blocked
+    # until the locks' TTL. Best-effort.
+    def release_terminal_locks(conn, job)
+      keys = []
+      keys << RedisKeys.unique_lock(UniqueJobs.digest(job)) if UniqueJobs.until_executed?(job)
+      keys << RedisKeys.periodic_running(job['periodic_pjid']) if job['periodic_pjid']
+      keys.each do |key|
+        released = BestEffort.call('Lock release') do
+          OwnedKey.delete(key, job['jid'], conn)
+          true
+        end
+        DeferredReleases.add(key, job['jid']) unless released
+      end
+    end
+
     UNPARSEABLE_CLASS = '(unparseable)'
 
     # A payload that isn't a JSON object can't be run, retried or even

data/lib/cogworker/key_guard.rb

@@ -22,6 +22,7 @@ module Cogworker
       RedisKeys::QUEUES => 'set', RedisKeys::PROCESSES => 'set', RedisKeys::PAUSED_QUEUES => 'set',
       RedisKeys::IN_PROGRESS_IDENTITIES => 'set', RedisKeys::LAST_BEAT => 'hash',
       RedisKeys::REPEAT_ORPHANS => 'list', RedisKeys::UNSETTLED => 'list', RedisKeys::PERIODIC_SCHEDULE => 'hash',
+      RedisKeys::LIVE_QUEUES => 'zset',
       RedisKeys::PERIODIC_DISABLED => 'set', RedisKeys::STATS_PROCESSED => 'string',
       RedisKeys::STATS_FAILED => 'string'
     }.freeze
@@ -65,9 +66,41 @@ module Cogworker
         types[RedisKeys.process(id)] = 'hash'
         types[RedisKeys.workers(id)] = 'hash'
       end
+      readable_fields(RedisKeys::PERIODIC_SCHEDULE).each do |pjid|
+        types[RedisKeys.periodic_running(pjid)] = 'string'
+        types[RedisKeys.periodic_last_slot(pjid)] = 'string'
+      end
       types
     end
 
+    def readable_fields(hash)
+      Cogworker.config.redis { |c| c.hkeys(hash) }
+    rescue Redis::CommandError
+      []
+    end
+
+    MAX_COPIES = 5
+
+    # Keeps the newest MAX_COPIES quarantined copies of one key: something
+    # that keeps writing the wrong type would otherwise leave one more every
+    # CHECK_INTERVAL, for QUARANTINE_TTL. (A SCAN, but only on the rare
+    # occasion a key actually gets quarantined.)
+    def prune_copies(key)
+      prefix = "#{QUARANTINE_PREFIX}#{key}:"
+      exact = /\A#{Regexp.escape(prefix)}(\d+):\h{8}\z/ # this key's copies only — not `<key>:<more>:...`'s
+      Cogworker.config.redis do |c|
+        copies = c.scan_each(match: "#{glob_escape(prefix)}*").select { |name| name.match?(exact) }
+        stale = copies.sort_by { |name| name[exact, 1].to_i }.reverse.drop(MAX_COPIES)
+        c.del(*stale) unless stale.empty?
+      end
+    rescue StandardError => e
+      Cogworker.logger.error { "pruning quarantined copies of #{key} failed: #{e.class}: #{e.message}" }
+    end
+
+    def glob_escape(text)
+      text.gsub(/[*?\[\]\\]/) { |ch| "\\#{ch}" }
+    end
+
     def readable_members(set)
       Cogworker.config.redis { |c| c.smembers(set) }
     rescue Redis::CommandError
@@ -77,12 +110,14 @@ module Cogworker
     def quarantine(key, expected, actual)
       # Unique per move: a second quarantine of the same key within the same
       # second must not overwrite (RENAME) the first one's copy.
-      target = "#{QUARANTINE_PREFIX}#{key}:#{Time.now.to_i}:#{SecureRandom.hex(4)}"
+      # Milliseconds, so copies made within one second still sort by age.
+      target = "#{QUARANTINE_PREFIX}#{key}:#{(Time.now.to_f * 1000).to_i}:#{SecureRandom.hex(4)}"
       moved = Cogworker.config.redis do |c|
         LuaScript.run(c, QUARANTINE_SCRIPT, keys: [key, target], argv: [expected, QUARANTINE_TTL])
       end
       return false unless moved == 1
 
+      prune_copies(key)
       Cogworker.logger.error do
         "#{key} held a #{actual}, not a #{expected}: moved to #{target} (kept #{QUARANTINE_TTL / 86_400} days)"
       end

data/lib/cogworker/launcher.rb

@@ -21,6 +21,7 @@ module Cogworker
       # two runs at once.
       @ticker = Periodic::Ticker.new(
         @manager, config.periodic_manager.entries, catch_up: config.periodic_catch_up,
+                                                   replace: config.periodic_replace,
                                                    ready: -> { @scheduled.ready_for_ticker? }
       )
       @signal_queue = ::Queue.new
@@ -45,6 +46,9 @@ module Cogworker
 
     def stop!
       @manager.stop!
+      # Last chance for releases deferred by a recent outage — they live in
+      # this process's memory and would otherwise wait out their locks' TTL.
+      BestEffort.call('Deferred lock releases') { @config.redis { |c| DeferredReleases.retry_all(c) } }
       @scheduled.stop!
       @ticker.stop!
       @heartbeat.stop!

data/lib/cogworker/lua_script.rb

@@ -9,8 +9,8 @@ module Cogworker
   # restarted or failed-over server, SCRIPT FLUSH). Every script in this
   # gem goes through here.
   module LuaScript
-    SHAS = {} # source => sha1; memoized, the source strings are frozen constants
-    SHAS_MUTEX = Mutex.new
+    @shas = {} # source => sha1; memoized, the sources are frozen constants
+    @shas_mutex = Mutex.new
 
     module_function
 
@@ -24,7 +24,7 @@ module Cogworker
     end
 
     def sha(source)
-      SHAS_MUTEX.synchronize { SHAS[source] ||= Digest::SHA1.hexdigest(source) }
+      @shas_mutex.synchronize { @shas[source] ||= Digest::SHA1.hexdigest(source) }
     end
   end
 end

data/lib/cogworker/manager.rb

@@ -16,6 +16,7 @@ module Cogworker
       # be what's failing (or what lost its last writes in a failover).
       @jobs_mutex = Mutex.new
       @running_jobs = Hash.new(0)
+      @performing_jobs = Hash.new(0) # the subset still inside `perform`
       @pending_settlements = {}
       @processors = Array.new(config.concurrency) { Processor.new(self) }
       @quiet = false
@@ -74,7 +75,24 @@ module Cogworker
     end
 
     def job_started(raw)
-      @jobs_mutex.synchronize { @running_jobs[raw] += 1 }
+      @jobs_mutex.synchronize do
+        @running_jobs[raw] += 1
+        @performing_jobs[raw] += 1
+      end
+    end
+
+    # `perform` (with its middleware) has returned: the job is still this
+    # process's until settled, but its locks are no longer its to keep
+    # alive — Heartbeat stops refreshing them (see `performing_jobs`).
+    def job_performed(raw)
+      @jobs_mutex.synchronize do
+        @performing_jobs[raw] -= 1
+        @performing_jobs.delete(raw) unless @performing_jobs[raw].positive?
+      end
+    end
+
+    def performing_jobs
+      @jobs_mutex.synchronize { @performing_jobs.keys }
     end
 
     def job_finished(raw)

data/lib/cogworker/periodic/claim.lua

@@ -10,7 +10,7 @@
 -- Returns {1, <previous last_slot, or "" if none>} if this call won the
 -- claim for this slot (the caller should enqueue the job — and, if that
 -- fails, roll the claim back with ROLLBACK_SCRIPT using that previous
--- value), 0 otherwise. Composes two guards: `last_slot` stops a
+-- value), 2 if blocked by the entry's running lock, 0 otherwise. Composes two guards: `last_slot` stops a
 -- process re-firing the same (or an earlier) slot on a later tick, and the
 -- NX lock breaks the narrow race where two processes both pass the
 -- `last_slot` check before either has written it back.
@@ -18,8 +18,11 @@
 -- For `until_executed`, the running lock is written here, in the same
 -- atomic step as the claim and before the job exists anywhere — see
 -- Periodic::RunningLock for why it must never be written after the push.
+-- Blocked by a run still holding the entry's lock: a distinct 2, so the
+-- caller can retry the slot briefly — the lock may only be waiting for a
+-- release delayed by an outage (see Ticker::BLOCKED_GRACE).
 if ARGV[2] == "until_executed" and redis.call("GET", KEYS[1]) then
-  return 0
+  return 2
 end
 
 local previous = redis.call("GET", KEYS[2])

data/lib/cogworker/periodic/manager.rb

@@ -23,6 +23,11 @@ module Cogworker
         @entries << entry
         entry
       end
+
+      # In place: Launcher hands this same `entries` array to the Ticker.
+      def clear!
+        @entries.clear
+      end
     end
   end
 end

data/lib/cogworker/periodic/release_middleware.rb

@@ -23,14 +23,25 @@ module Cogworker
         begin
           yield
         rescue Exception => e # rubocop:disable Lint/RescueException
-          RunningLock.release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
+          release(pjid, job['jid']) if JobUtil.terminal_failure?(job)
           raise e
         end
-        RunningLock.release(pjid, job['jid'])
+        release(pjid, job['jid'])
       end
 
       private
 
+      # Best-effort — a release that fails (Redis away) never turns the
+      # finished run into a failure — and retried later (DeferredReleases)
+      # rather than left to the lock's TTL.
+      def release(pjid, jid)
+        released = BestEffort.call('Periodic lock') do
+          RunningLock.release(pjid, jid)
+          true
+        end
+        DeferredReleases.add(RedisKeys.periodic_running(pjid), jid) unless released
+      end
+
       # Best-effort: failing here must not stop the job from running (it
       # would be routed to retry/dead as if it had failed). If it does fail,
       # the lock just keeps its longer queued TTL until this process's next

data/lib/cogworker/periodic/running_lock.rb

@@ -61,6 +61,18 @@ module Cogworker
         OwnedKey.expire(RedisKeys.periodic_running(pjid), jid, ttl, conn)
       end
 
+      RETAKE_SCRIPT = RETAKE_FUNCTION + <<~LUA
+        retake_running_lock(KEYS[1], ARGV[1], ARGV[2])
+        return 1
+      LUA
+
+      # Extends the lock if still `jid`'s, takes it if it's gone, leaves it
+      # if another run holds it — for a run still going whose lock may have
+      # lapsed (Redis away longer than `active_ttl`).
+      def retake(pjid, jid, ttl, conn)
+        LuaScript.run(conn, RETAKE_SCRIPT, keys: [RedisKeys.periodic_running(pjid)], argv: [jid, ttl.to_f.ceil])
+      end
+
       def release(pjid, jid, conn = nil)
         OwnedKey.delete(RedisKeys.periodic_running(pjid), jid, conn)
       end