cogworker 0.4.1 → 0.5.1

data/lib/cogworker/scheduled.rb

@@ -21,17 +21,21 @@ module Cogworker
     # is limited to processes actually using ReliableFetch. The requeue is
     # atomic, so any number of processes doing it at once is safe.
     ORPHAN_CHECK_INTERVAL = 60
+    DEFER_DELAY = 60
 
     # KEYS[1] = the ZSET, KEYS[2] = cogworker:dead, KEYS[3] = stats:failed;
     # ARGV[1] = the entry, ARGV[2] = its dead-set score, ARGV[3] = the
     # wrapper to store. Same claim-and-move-atomically shape as
     # JobUtil::CLAIM_AND_REQUEUE_SCRIPT.
+    # (Write, then remove — see JobUtil::CLAIM_AND_REQUEUE_SCRIPT; the stats
+    # counter is best-effort.)
     BURY_SCRIPT = <<~LUA
-      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+      if not redis.call("ZSCORE", KEYS[1], ARGV[1]) then
         return 0
       end
       redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
-      redis.call("INCR", KEYS[3])
+      redis.call("ZREM", KEYS[1], ARGV[1])
+      redis.pcall("INCR", KEYS[3])
       return 1
     LUA
 
@@ -61,41 +65,120 @@ module Cogworker
       @recovered_once == true
     end
 
+    # What the periodic Ticker waits for before its first tick: the first
+    # orphan check that really completed. No timeout: without Redis the
+    # Ticker couldn't do anything anyway, and a key that would keep the check
+    # failing for good is quarantined (KeyGuard) within a minute — whereas
+    # starting early after a long outage could run an `until_executed` slot
+    # alongside the recovered run of the previous one.
+    def ready_for_ticker?
+      recovered_once?
+    end
+
     private
 
     # A failed poll is logged and retried next interval rather than ending
     # the thread — a single Redis blip used to stop every scheduled/retry
     # job in this process from ever reaching its queue again.
+    # Recovery and graduating due jobs fail independently: a recovery that
+    # keeps failing used to skip `enqueue_due_jobs` on every poll, so no
+    # scheduled or retried job reached its queue anywhere in the fleet.
     def run
       until @manager.stopping?
-        begin
-          recover_orphans_if_due
-          enqueue_due_jobs unless @manager.quiet?
-        rescue StandardError => e
-          Cogworker.logger.error { "Scheduled poll failed: #{e.class}: #{e.message}" }
-        end
+        guarded('Key check') { check_keys_if_due }
+        guarded('Orphan check') { recover_orphans_if_due }
+        guarded('Scheduled poll') { enqueue_due_jobs unless @manager.quiet? }
         sleep(POLL_INTERVAL)
       end
     end
 
-    # Next check scheduled only once this one has completed: a failed one
-    # (Redis down) is retried on the next poll rather than a minute later.
+    # An entry that can't be graduated right now (its queue key of the wrong
+    # type, say) is pushed DEFER_DELAY into the future: left due, it would be
+    # one of the same oldest 50 picked again on every poll, and enough of
+    # them would starve everything behind them for good.
+    def defer(set, raw, error)
+      Cogworker.logger.error do
+        "Graduating an entry of #{set} failed (deferred #{DEFER_DELAY}s): #{error.class}: #{error.message}"
+      end
+      Cogworker.config.redis { |c| c.zadd(set, Time.now.to_f + DEFER_DELAY, raw, xx: true) }
+    rescue StandardError => e
+      Cogworker.logger.error { "Deferring it failed too: #{e.class}: #{e.message}" }
+    end
+
+    # A block returning :skipped did nothing (not due yet): that proves
+    # nothing about Redis, so it doesn't end an outage report either.
+    def guarded(what)
+      result = yield
+      RedisErrors.recovered(what) unless result == :skipped
+      result
+    rescue StandardError => e
+      RedisErrors.report(what, e)
+    end
+
+    # The next check is scheduled before this one runs: one that keeps
+    # failing (everything inside it is isolated, so that means Redis itself
+    # is failing) is retried every ORPHAN_CHECK_INTERVAL — or on the next
+    # poll if it lost the connection — never on every poll with a full SCAN.
+    # The one-off SCAN only counts as done once it has succeeded.
     def recover_orphans_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      return if @next_orphan_check && now < @next_orphan_check
+      return :skipped if @next_orphan_check && now < @next_orphan_check
 
-      reliable = @manager.fetch_class == ReliableFetch
-      ReliableFetch.recover_orphans(scan: !@recovered_once && reliable)
-      @strays = ReliableFetch.reconcile(Cogworker.identity, @strays || []) if reliable
-      @recovered_once = true
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
+      reliable = @manager.fetch_class == ReliableFetch
+      report = {}
+      # Lost the connection (anywhere in it): check again on the next poll,
+      # not a minute later — Redis coming back is when it matters most.
+      begin
+        ReliableFetch.recover_orphans(scan: reliable && !@scanned, report: report)
+      rescue StandardError => e
+        @next_orphan_check = now + POLL_INTERVAL if RedisErrors.unavailable?(e)
+        raise
+      end
+      @next_orphan_check = now + POLL_INTERVAL if report[:connection_lost]
+      guarded('Reconcile') { reconcile } if reliable
+      # Only once that actually looked: a pass that merely survived every
+      # error (recovery isolates them), or lost the connection part-way,
+      # mustn't let the Ticker start, nor count the one-off SCAN as done.
+      return if report[:connection_lost]
+
+      @scanned = true if report[:scan]
+      @recovered_once = true if report[:registry] || report[:scan]
+      nil
+    end
+
+    # See KeyGuard. First thing on the first poll, then every
+    # KeyGuard::CHECK_INTERVAL — before recovery and polling, so a key that
+    # would break them is already out of the way.
+    def check_keys_if_due
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return :skipped if @next_key_check && now < @next_key_check
+
+      @next_key_check = now + KeyGuard::CHECK_INTERVAL
+      KeyGuard.check(queues: @manager.queues)
     end
 
+    def reconcile
+      @strays = ReliableFetch.reconcile(Cogworker.identity, @strays || [],
+                                        running: @manager.running_jobs, pending: @manager.pending_settlements) do |raw|
+        @manager.settled(raw)
+      end
+    end
+
+    # Each set, and each entry, on its own: `schedule` unreadable mustn't
+    # stop `retry` from being polled, nor one entry whose graduation fails
+    # stop the rest of the batch.
     def enqueue_due_jobs
       now = Time.now.to_f
       SETS.each do |set|
-        candidates = Cogworker.config.redis { |c| c.zrangebyscore(set, '-inf', now, limit: [0, 50]) }
-        candidates.each { |raw| graduate(set, raw) }
+        guarded("Polling #{set}") do
+          candidates = Cogworker.config.redis { |c| c.zrangebyscore(set, '-inf', now, limit: [0, 50]) }
+          candidates.each do |raw|
+            graduate(set, raw)
+          rescue StandardError => e
+            defer(set, raw, e)
+          end
+        end
       end
     end
 
@@ -115,7 +198,7 @@ module Cogworker
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
       buried = LuaScript.run(conn, BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
-                                               argv: [raw, job['failed_at'], JSON.generate(job)])
+                                                argv: [raw, job['failed_at'], JSON.generate(job)])
       return unless buried == 1
 
       Throughput.record('failed') # counted like Processor#bury_unparseable

data/lib/cogworker/status/server_middleware.rb

@@ -9,14 +9,20 @@ module Cogworker
         @expiration = expiration
       end
 
+      # Every status write is best-effort (BestEffort) — before `perform`
+      # as much as after: one failing used to count as the job failing.
       def call(_worker, job, queue)
-        write(job, queue, 'working')
-        yield
-        write(job, queue, 'complete')
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
-              error_class: e.class.name, error_message: JobUtil.error_message(e))
-        raise e
+        BestEffort.call('Status') { write(job, queue, 'working') }
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          BestEffort.call('Status') do
+            write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
+                  error_class: e.class.name, error_message: JobUtil.error_message(e))
+          end
+          raise e
+        end
+        BestEffort.call('Status') { write(job, queue, 'complete') }
       end
 
       private

data/lib/cogworker/swarm.rb

@@ -59,6 +59,8 @@ module Cogworker
     def fork_child(slot)
       pid = ::Process.fork do
         Cogworker.reset_identity!
+        RedisErrors.reset! # the parent's outage state isn't this child's
+        DeferredReleases.reset! # nor are its pending releases
         CLI.new.run(@argv.dup)
         # `exit!`, not plain `exit`: `Launcher#run` only returns once
         # `Manager#stop!` has had its bounded (default 25s) chance to join

data/lib/cogworker/unique_jobs/release_middleware.rb

@@ -12,12 +12,16 @@ module Cogworker
     # `unique: :until_executed`. Registered unconditionally in every
     # `Config`, same as `Periodic::ReleaseMiddleware`.
     class ReleaseMiddleware
+      # Releasing is best-effort (BestEffort): a failed release leaves the
+      # lock to its TTL, and only the job's own exception is re-raised.
       def call(_worker, job, _queue)
-        yield
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          release(job) if UniqueJobs.until_executed?(job) && JobUtil.terminal_failure?(job)
+          raise e
+        end
         release(job) if UniqueJobs.until_executed?(job)
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        release(job) if UniqueJobs.until_executed?(job) && JobUtil.terminal_failure?(job)
-        raise e
       end
 
       private
@@ -25,8 +29,15 @@ module Cogworker
       # Owner-checked: if this job's lock already expired and passed to a
       # newer copy, this must not release *that* one's lock (which would let
       # a third copy in while the newer one is still running).
+      # Owner-checked; retried later (DeferredReleases) if it fails, rather
+      # than leaving duplicates blocked for the lock's whole TTL.
       def release(job)
-        OwnedKey.delete(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'])
+        key = RedisKeys.unique_lock(UniqueJobs.digest(job))
+        released = BestEffort.call('Unique lock') do
+          OwnedKey.delete(key, job['jid'])
+          true
+        end
+        DeferredReleases.add(key, job['jid']) unless released
       end
     end
   end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.4.1'
+  VERSION = '0.5.1'
 end

data/lib/cogworker/web/layout.rb

@@ -380,7 +380,9 @@ module Cogworker
       # Redis cards), where the default wrapper would just nest one bordered
       # box inside another.
       def table(headers, rows, empty_message: 'Nothing here.', wrapped: true)
-        return %(<p class="text-muted" style="font-size: 13px; font-style: italic;">#{h(empty_message)}</p>) if rows.empty?
+        if rows.empty?
+          return %(<p class="text-muted" style="font-size: 13px; font-style: italic;">#{h(empty_message)}</p>)
+        end
 
         head = headers.map { |c| %(<th>#{h(c)}</th>) }.join
         body = rows.map do |row|

data/lib/cogworker/web/routes/jobs.rb

@@ -288,13 +288,13 @@ module Cogworker
         def retry_now_button(bucket, script_name, row, params)
           action = Layout.path(script_name, "jobs/#{bucket}/retry_now")
           form_with_extra(action, 'raw', row[:raw], 'retry now', params, variant: :primary, icon: 'arrow-clockwise',
-                                                                          extra: {})
+                                                                         extra: {})
         end
 
         def retry_button(bucket, script_name, row, params)
           action = Layout.path(script_name, "jobs/#{bucket}/retry")
           form_with_extra(action, 'raw', row[:raw], 'retry', params, variant: :primary, icon: 'arrow-clockwise',
-                                                                      extra: {})
+                                                                     extra: {})
         end
 
         # `Layout.form_button` only carries one hidden field. Every action
@@ -304,9 +304,9 @@ module Cogworker
         # since it isn't in the URL a plain form submits to) plus, for an
         # enqueued-job delete, `queue` (`Queue#delete` isn't keyed off the
         # raw payload alone the way the ZSETs are).
-        def form_with_extra(action, hidden_name, hidden_value, label, params, variant:, icon: nil, extra:)
+        def form_with_extra(action, hidden_name, hidden_value, label, params, variant:, extra:, icon: nil)
           hidden = extra.merge('status' => params['status'].to_s, 'q' => params['q'].to_s,
-                                'selected' => params['selected'].to_s)
+                               'selected' => params['selected'].to_s)
           hidden_inputs = hidden.map { |k, v| %(<input type="hidden" name="#{k}" value="#{Layout.h(v)}">) }.join
           classes = "btn #{Layout::BUTTON_VARIANTS.fetch(variant)}"
           <<~HTML

data/lib/cogworker/web/routes/overview.rb

@@ -613,7 +613,9 @@ module Cogworker
         def render_layout_b(script_name, selected_name)
           names = queue_names
           selected_name = names.first if (selected_name.nil? || selected_name.empty?) && !names.empty?
-          return %(<p class="text-muted">No queues yet — push a job to create one.</p>) if selected_name.nil? || selected_name.empty?
+          if selected_name.nil? || selected_name.empty?
+            return %(<p class="text-muted">No queues yet — push a job to create one.</p>)
+          end
 
           <<~HTML
             <div style="display: grid; grid-template-columns: minmax(240px, 320px) minmax(0, 1fr); gap: 16px; align-items: start;">
@@ -676,7 +678,7 @@ module Cogworker
           end.join
           paused_tag = q.paused? ? " #{Layout.badge('paused', variant: :warning)}" : ''
           header_actions = queue_pause_button(q, script_name, 'b') + retry_all_button(name, script_name,
-                                                                                       retrying_count)
+                                                                                      retrying_count)
           <<~HTML
             <section style="display: flex; flex-direction: column; gap: 16px; min-width: 0;">
               <div style="background: var(--color-surface); border-radius: var(--radius-md); box-shadow: var(--shadow-sm); padding: 18px;">

data/lib/cogworker/work_set.rb

@@ -3,6 +3,9 @@
 require 'json'
 
 module Cogworker
+  # Every job running right now, across all live processes: yields
+  # `[identity, thread id, Cogworker::Work]` from each process's
+  # `cogworker:workers:<identity>` Hash (only for processes ProcessSet knows).
   class WorkSet
     include Enumerable
 

data/lib/cogworker.rb

@@ -101,7 +101,7 @@ module Cogworker
     # No-op: exists so calling code doesn't need to change, but this gem
     # simply never coerces job args, so there is nothing to toggle — see
     # README "Job arguments".
-    def strict_args!(_value = true)
+    def strict_args!(_value = true) # rubocop:disable Style/OptionalBooleanParameter -- kept positional for calling-code compatibility
       true
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.1
 platform: ruby
 authors:
 - vdonec
@@ -203,16 +203,20 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
+- README.md
 - exe/cogworker
 - exe/cogworkerswarm
 - lib/cogworker.rb
 - lib/cogworker/attempts.rb
 - lib/cogworker/basic_fetch.rb
+- lib/cogworker/best_effort.rb
 - lib/cogworker/cli.rb
 - lib/cogworker/client.rb
 - lib/cogworker/component.rb
 - lib/cogworker/config.rb
 - lib/cogworker/config_loader.rb
+- lib/cogworker/deferred_releases.rb
 - lib/cogworker/heartbeat.rb
 - lib/cogworker/history.rb
 - lib/cogworker/history/middleware.rb
@@ -220,6 +224,7 @@ files:
 - lib/cogworker/job.rb
 - lib/cogworker/job_record.rb
 - lib/cogworker/job_util.rb
+- lib/cogworker/key_guard.rb
 - lib/cogworker/launcher.rb
 - lib/cogworker/logging.rb
 - lib/cogworker/lua_script.rb
@@ -238,6 +243,7 @@ files:
 - lib/cogworker/prometheus/exporter.rb
 - lib/cogworker/queue.rb
 - lib/cogworker/redis_connection.rb
+- lib/cogworker/redis_errors.rb
 - lib/cogworker/redis_keys.rb
 - lib/cogworker/reliable_fetch.rb
 - lib/cogworker/reliable_fetch/unsupported_error.rb