cogworker 0.5.0 → 0.6.0

data/lib/cogworker/periodic/ticker.rb

@@ -29,7 +29,7 @@ module Cogworker
       # separate call made first, its failure (Redis down — the usual reason
       # the push failed) skipped the rollback altogether.
       ROLLBACK_SCRIPT = <<~LUA
-        if redis.call("GET", KEYS[1]) == ARGV[1] then
+        if redis.pcall("GET", KEYS[1]) == ARGV[1] then
           if ARGV[2] == "" then
             redis.call("SET", KEYS[1], tostring(tonumber(ARGV[1]) - 1))
           else
@@ -46,14 +46,16 @@ module Cogworker
         return 1
       LUA
 
-      def initialize(manager, entries, catch_up: true, ready: -> { true })
+      def initialize(manager, entries, catch_up: true, replace: false, ready: -> { true })
         @manager = manager
         @entries = entries
         @catch_up = catch_up
+        @replace = replace
         @ready = ready
         @last_checked_slot = {}
         @cron_cache = {}
         @pending_rollbacks = []
+        @blocked_since = {} # pjid => [slot, monotonic time it was first found blocked]
         @failures = Hash.new(0) # pjid => consecutive failed ticks
         @retry_at = {}          # pjid => monotonic time it may be tried again
       end
@@ -83,8 +85,12 @@ module Cogworker
       # here (not at DSL-registration time in Config#periodic) so it never
       # depends on `config.redis =` having already run — every process that
       # boots re-writes the same idempotent entries regardless of ordering.
+      # With `replace` (Config#periodic(replace: true)) the hash is swapped
+      # out wholesale in one MULTI, dropping entries no longer registered
+      # (their `last_slot`/`disabled` state is left alone: an old process
+      # still ticking them mid-deploy must not re-fire an already-run slot).
       def publish_schedule!
-        return if @entries.empty?
+        return if @entries.empty? && !@replace
 
         payloads = @entries.each_with_object({}) do |entry, h|
           h[entry.pjid] = JSON.generate(
@@ -92,7 +98,14 @@ module Cogworker
             'unique' => entry.unique, 'args' => entry.args
           )
         end
-        Cogworker.config.redis { |c| c.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) }
+        Cogworker.config.redis do |c|
+          next c.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) unless @replace
+
+          c.multi do |tx|
+            tx.del(RedisKeys::PERIODIC_SCHEDULE)
+            tx.hset(RedisKeys::PERIODIC_SCHEDULE, *payloads.to_a.flatten) unless payloads.empty?
+          end
+        end
       end
 
       # A failed tick is logged and retried next interval rather than ending
@@ -113,12 +126,16 @@ module Cogworker
 
       def tick
         retry_pending_rollbacks
+        # This process's own delayed lock releases first (see BLOCKED_GRACE).
+        BestEffort.call('Deferred lock releases') { Cogworker.config.redis { |c| DeferredReleases.retry_all(c) } }
         now = Time.now
         @entries.each do |entry|
           next if backing_off?(entry.pjid)
 
-          tick_entry(entry, now)
+          next if tick_entry(entry, now) == :skipped # (no Redis call made: proves nothing)
+
           @failures.delete(entry.pjid)
+          RedisErrors.recovered('Periodic tick')
         rescue StandardError => e
           # Per entry: one entry failing (its keys of the wrong type, its
           # push failing) used to skip every entry after it, on every tick.
@@ -128,8 +145,8 @@ module Cogworker
           # a lost connection, though: that's Redis being away, not the
           # entry failing — backing off every entry on it left cron idle for
           # minutes after Redis came back.
-          delay = e.is_a?(Redis::BaseConnectionError) ? TICK_INTERVAL : back_off(entry.pjid)
-          Cogworker.logger.error { "Periodic tick failed for #{entry.pjid} (next try in #{delay}s): #{e.class}: #{e.message}" }
+          delay = RedisErrors.unavailable?(e) ? TICK_INTERVAL : back_off(entry.pjid)
+          RedisErrors.report('Periodic tick', e) { "#{entry.pjid} failed (next try in #{delay}s)" }
         end
       end
 
@@ -149,12 +166,32 @@ module Cogworker
 
       def tick_entry(entry, now)
         slot = cron_for(entry).previous_time(now).to_i
-        return if @last_checked_slot[entry.pjid] == slot
+        return :skipped if @last_checked_slot[entry.pjid] == slot
 
         jid = SecureRandom.hex(12)
         previous = claim(entry, slot, jid) unless disabled?(entry)
-        enqueue(entry, slot, jid, previous) if previous
+        return if previous == :blocked && still_blocked_in_grace?(entry.pjid, slot)
+
+        enqueue(entry, slot, jid, previous) if previous && previous != :blocked
         @last_checked_slot[entry.pjid] = slot
+        @blocked_since.delete(entry.pjid)
+      end
+
+      # A slot blocked by the entry's running lock is retried for
+      # BLOCKED_GRACE before it counts as skipped. After an outage, the lock
+      # may still be held only because its finished run's release was
+      # deferred (DeferredReleases — retried on the next heartbeat, in
+      # whichever process ran it); marking the slot checked straight away
+      # lost it, in every process, until the entry's next slot — an hour,
+      # for an hourly entry. A run genuinely still going keeps the lock past
+      # the grace, and its slot is skipped as `until_executed` intends.
+      BLOCKED_GRACE = 3 * Heartbeat::INTERVAL
+
+      def still_blocked_in_grace?(pjid, slot)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        since_slot, since = @blocked_since[pjid]
+        @blocked_since[pjid] = [slot, since = now] unless since_slot == slot
+        now - since < BLOCKED_GRACE
       end
 
       # The usual reason a push fails — Redis unreachable — makes this fail
@@ -175,7 +212,7 @@ module Cogworker
         end
         true
       rescue StandardError => e
-        Cogworker.logger.error { "periodic #{entry.pjid}: couldn't roll back slot #{slot} (will retry): #{e.class}: #{e.message}" }
+        RedisErrors.report('Periodic rollback', e) { "#{entry.pjid}: couldn't roll back slot #{slot} (will retry)" }
         @pending_rollbacks << [entry, slot, previous_slot, jid]
         e # (truthy, but not `true`: the caller tells a lost connection from a failing rollback)
       end
@@ -194,7 +231,8 @@ module Cogworker
           if result == true
             @failures.delete(key)
             @retry_at.delete(key)
-          elsif !result.is_a?(Redis::BaseConnectionError)
+            RedisErrors.recovered('Periodic rollback')
+          elsif !RedisErrors.unavailable?(result)
             back_off(key) # only a rollback failing on its own backs off; see #tick
           end
         end
@@ -218,9 +256,9 @@ module Cogworker
       # connection still raises (the tick is then retried as a whole).
       def disabled?(entry)
         Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
-      rescue Redis::BaseConnectionError
-        raise
       rescue StandardError => e
+        raise if RedisErrors.unavailable?(e)
+
         Cogworker.logger.error { "Periodic: can't read #{RedisKeys::PERIODIC_DISABLED} (#{e.class}: #{e.message}); treating #{entry.pjid} as enabled" }
         false
       end
@@ -229,17 +267,20 @@ module Cogworker
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end
 
-      # nil if this process didn't win the slot; otherwise the previous
-      # `last_slot` value ("" if none), for #rollback_claim.
+      # nil if this process didn't win the slot, :blocked if the entry's
+      # running lock stopped it; otherwise the previous `last_slot` value
+      # ("" if none), for #rollback_claim.
       def claim(entry, slot, jid)
         return nil if !@catch_up && priming_first_slot?(entry, slot)
 
         result = Cogworker.config.redis do |c|
           LuaScript.run(c, CLAIM_SCRIPT,
-                 keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
-                        RedisKeys.periodic_lock(entry.pjid, slot)],
-                 argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
+                        keys: [RedisKeys.periodic_running(entry.pjid), RedisKeys.periodic_last_slot(entry.pjid),
+                               RedisKeys.periodic_lock(entry.pjid, slot)],
+                        argv: [slot, entry.unique.to_s, LOCK_TTL, jid, RunningLock.queued_ttl])
         end
+        return :blocked if result == 2
+
         result.is_a?(Array) ? result[1].to_s : nil
       end
 

data/lib/cogworker/process_set.rb

@@ -3,6 +3,9 @@
 require 'json'
 
 module Cogworker
+  # Every live worker process, as published by its Heartbeat — one
+  # Cogworker::Process per `cogworker:processes` member whose presence key
+  # still exists (stale members are pruned while iterating).
   class ProcessSet
     include Enumerable
 

data/lib/cogworker/processor.rb

@@ -56,9 +56,9 @@ module Cogworker
     def run
       until @manager.stopping?
         begin
-          process_one
+          RedisErrors.recovered('Processor') unless process_one == :idle
         rescue StandardError => e
-          Cogworker.logger.error { "Processor error: #{e.class}: #{e.message}" }
+          RedisErrors.report('Processor', e)
           sleep(ERROR_BACKOFF)
         end
       end
@@ -67,7 +67,7 @@ module Cogworker
     def process_one
       if @manager.quiet?
         sleep(0.5)
-        return
+        return :idle # (no Redis call made)
       end
 
       work = begin
@@ -89,11 +89,13 @@ module Cogworker
       @manager.job_started(work.raw_job)
       @busy = true
       @ran = nil
+      @performed = false
       disposition = nil
       begin
         disposition = execute(work)
       ensure
         @busy = false
+        @manager.job_performed(work.raw_job) unless @performed
         @manager.processor_idle!
         # `execute` raising leaves `disposition` unset: before the job ran
         # that's a hand-back; once it ran, whatever its outcome dictates
@@ -155,7 +157,11 @@ module Cogworker
         @manager.settle_later(work.raw_job, [set, score, encoded]) if fetcher.is_a?(ReliableFetch)
         raise
       end
-      best_effort('lock extension') { extend_locks_for_retry(job, delay) } if set == RedisKeys::RETRY
+      if set == RedisKeys::RETRY
+        best_effort('lock extension') { extend_locks_for_retry(job, delay) }
+      else
+        best_effort('lock release') { release_locks(payload) }
+      end
       reason = if set == RedisKeys::RETRY
                  "retrying in #{delay}s (interruption #{count} of at most #{MAX_INTERRUPTS})"
                elsif out_of_retries
@@ -176,9 +182,11 @@ module Cogworker
     end
 
     def remember_for_dead(work, error)
-      entry = JSON.generate('jid' => jid_of(work), 'error_class' => error.class.name.to_s,
-                            'error_message' => JobUtil.error_message(error), 'failed_at' => Time.now.to_f,
-                            'raw_payload' => JobUtil.safe_string(work.raw_job, 100_000))
+      # The original job's own fields where readable (so filing it releases
+      # its locks); just enough to show in Dead otherwise.
+      base = ReliableFetch.parse_hash(work.raw_job) || { 'raw_payload' => JobUtil.safe_string(work.raw_job, 100_000) }
+      entry = JSON.generate(base.merge('jid' => jid_of(work), 'error_class' => error.class.name.to_s,
+                                       'error_message' => JobUtil.error_message(error), 'failed_at' => Time.now.to_f))
       @manager.settle_later(work.raw_job, [RedisKeys::DEAD, Time.now.to_f, entry])
     rescue StandardError
       nil
@@ -230,6 +238,8 @@ module Cogworker
       Cogworker.logger.info { "start: #{job['class']} jid=#{job['jid']}" }
       begin
         error = run_job(work, job)
+        @manager.job_performed(work.raw_job)
+        @performed = true
         @ran = error ? [:interrupt, job, error] : :acknowledge
         error ? finish_failure(job, error) : finish_success(job)
       ensure
@@ -286,7 +296,9 @@ module Cogworker
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
         Throughput.record('failed')
       end
-      Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}" }
+      Cogworker.logger.warn do
+        "fail: #{job['class']} jid=#{job['jid']}: #{error.class}: #{JobUtil.error_message(error)}"
+      end
       route_failure(job, error) ? :acknowledge : [:interrupt, job, error]
     end
 
@@ -350,7 +362,12 @@ module Cogworker
       begin
         payload = JSON.generate(job)
         Cogworker.config.redis do |c|
-          retrying ? c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, payload) : c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+          if retrying
+            c.zadd(RedisKeys::RETRY, Time.now.to_f + delay,
+                   payload)
+          else
+            c.zadd(RedisKeys::DEAD, Time.now.to_f, payload)
+          end
         end
       rescue StandardError => e
         Cogworker.logger.error { "couldn't record failure of jid=#{job['jid']}: #{e.class}: #{e.message}" }
@@ -380,6 +397,13 @@ module Cogworker
       end
     end
 
+    # Into dead by way of `interrupt` — possibly with retries still left,
+    # so the middleware that normally releases a terminal failure's locks
+    # didn't: release them here (owner-checked).
+    def release_locks(job)
+      Cogworker.config.redis { |c| JobUtil.release_terminal_locks(c, job) }
+    end
+
     # Grows with the retry count, with jitter to avoid a thundering herd of
     # retries all landing on the same second.
     def retry_delay(count)

data/lib/cogworker/redis_errors.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Telling "Redis is unavailable right now" from every other error, and
+  # keeping the log readable while it is.
+  #
+  # Unavailable means a lost connection, or one of the replies a server
+  # gives while it can't serve writes for a while — mid-failover (READONLY,
+  # MASTERDOWN), still loading its data (LOADING), or busy with a script
+  # (BUSY, TRYAGAIN). None of those say anything about the job, entry or key
+  # being worked on, so nothing should back off, count a failure, or give up
+  # on them; they just mean "try again shortly".
+  #
+  # `report` logs such errors once per `site` (a short name of the place
+  # they come from) as they start, then a one-line summary at most every
+  # SUMMARY_INTERVAL, and `recovered(site)` logs once that it's over. A 6
+  # minute outage used to produce thousands of identical lines. Any other
+  # error is logged in full, every time.
+  module RedisErrors
+    UNAVAILABLE_REPLIES = %w[READONLY LOADING MASTERDOWN TRYAGAIN BUSY].freeze
+    SUMMARY_INTERVAL = 60
+    SCRIPT_WRAPPED = /-(?:#{UNAVAILABLE_REPLIES.join('|')})(?=\s|\z)/
+
+    @mutex = Mutex.new
+    @outages = {} # site => { count:, since:, summarized_at:, error: }
+
+    class << self
+      def unavailable?(error)
+        return true if error.is_a?(Redis::BaseConnectionError)
+        # redis-rb 5's pub/sub path can surface redis-client's own error class.
+        return true if defined?(RedisClient::ConnectionError) && error.is_a?(RedisClient::ConnectionError)
+
+        return false unless error.is_a?(Redis::CommandError)
+
+        message = error.message.to_s
+        # The reply's code is its whole first word — BUSYKEY or BUSYGROUP
+        # aren't BUSY. Inside a script, Redis 6.x wraps the code instead:
+        # "ERR Error running script (...): ... -READONLY You can't ...".
+        UNAVAILABLE_REPLIES.include?(message[/\A\S+/]) ||
+          (message.include?('Error running script') && message.match?(SCRIPT_WRAPPED))
+      end
+
+      # Logs `error` at `site`; the block, if given, adds context to the
+      # first line (e.g. what was being done).
+      def report(site, error, &context)
+        detail = "#{error.class}: #{error.message}"
+        detail = "#{context.call} — #{detail}" if context
+        return Cogworker.logger.error { "#{site}: #{detail}" } unless unavailable?(error)
+
+        line = @mutex.synchronize { track(site, error, detail) }
+        Cogworker.logger.error { line } if line
+      end
+
+      # Call wherever `site` has just done its job: ends an outage there.
+      def recovered(site)
+        outage = @mutex.synchronize { @outages.delete(site) }
+        return unless outage
+
+        Cogworker.logger.warn do
+          "#{site}: Redis available again after #{outage[:count]} error(s) over #{(now - outage[:since]).round}s"
+        end
+      end
+
+      def reset!
+        @mutex.synchronize { @outages.clear }
+      end
+
+      private
+
+      def track(site, error, detail)
+        outage = @outages[site]
+        unless outage
+          @outages[site] = { count: 1, since: now, summarized_at: now, suppressed: 0 }
+          return "#{site}: Redis unavailable — #{detail} (repeats summarized every #{SUMMARY_INTERVAL}s)"
+        end
+
+        outage[:count] += 1
+        outage[:suppressed] += 1
+        return unless now - outage[:summarized_at] >= SUMMARY_INTERVAL
+
+        suppressed = outage[:suppressed]
+        outage[:summarized_at] = now
+        outage[:suppressed] = 0
+        "#{site}: Redis still unavailable — #{suppressed} more error(s) in the last #{SUMMARY_INTERVAL}s " \
+          "(latest: #{error.class}: #{error.message})"
+      end
+
+      def now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/cogworker/redis_keys.rb

@@ -26,6 +26,10 @@ module Cogworker
     # over by a process shutting down (JSON {set, score, payload} each) —
     # see ReliableFetch.park_unsettled / file_unsettled.
     UNSETTLED = 'cogworker:unsettled'
+    # queue => Redis time a live process last said it reads it (every
+    # heartbeat). Unlike QUEUES (the Web UI's listing, never pruned), this
+    # tells "nobody reads this queue any more" apart from "somebody does".
+    LIVE_QUEUES = 'cogworker:live_queues'
     QUEUES = 'cogworker:queues'
     SCHEDULE = 'cogworker:schedule'
     RETRY = 'cogworker:retry'