cogworker 0.5.0 → 0.5.1

data/lib/cogworker/scheduled.rb

@@ -97,16 +97,22 @@ module Cogworker
     # one of the same oldest 50 picked again on every poll, and enough of
     # them would starve everything behind them for good.
     def defer(set, raw, error)
-      Cogworker.logger.error { "Graduating an entry of #{set} failed (deferred #{DEFER_DELAY}s): #{error.class}: #{error.message}" }
+      Cogworker.logger.error do
+        "Graduating an entry of #{set} failed (deferred #{DEFER_DELAY}s): #{error.class}: #{error.message}"
+      end
       Cogworker.config.redis { |c| c.zadd(set, Time.now.to_f + DEFER_DELAY, raw, xx: true) }
     rescue StandardError => e
       Cogworker.logger.error { "Deferring it failed too: #{e.class}: #{e.message}" }
     end
 
+    # A block returning :skipped did nothing (not due yet): that proves
+    # nothing about Redis, so it doesn't end an outage report either.
     def guarded(what)
-      yield
+      result = yield
+      RedisErrors.recovered(what) unless result == :skipped
+      result
     rescue StandardError => e
-      Cogworker.logger.error { "#{what} failed: #{e.class}: #{e.message}" }
+      RedisErrors.report(what, e)
     end
 
     # The next check is scheduled before this one runs: one that keeps
@@ -116,7 +122,7 @@ module Cogworker
     # The one-off SCAN only counts as done once it has succeeded.
     def recover_orphans_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      return if @next_orphan_check && now < @next_orphan_check
+      return :skipped if @next_orphan_check && now < @next_orphan_check
 
       @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
       reliable = @manager.fetch_class == ReliableFetch
@@ -125,8 +131,8 @@ module Cogworker
       # not a minute later — Redis coming back is when it matters most.
       begin
         ReliableFetch.recover_orphans(scan: reliable && !@scanned, report: report)
-      rescue Redis::BaseConnectionError
-        @next_orphan_check = now + POLL_INTERVAL
+      rescue StandardError => e
+        @next_orphan_check = now + POLL_INTERVAL if RedisErrors.unavailable?(e)
         raise
       end
       @next_orphan_check = now + POLL_INTERVAL if report[:connection_lost]
@@ -138,6 +144,7 @@ module Cogworker
 
       @scanned = true if report[:scan]
       @recovered_once = true if report[:registry] || report[:scan]
+      nil
     end
 
     # See KeyGuard. First thing on the first poll, then every
@@ -145,7 +152,7 @@ module Cogworker
     # would break them is already out of the way.
     def check_keys_if_due
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      return if @next_key_check && now < @next_key_check
+      return :skipped if @next_key_check && now < @next_key_check
 
       @next_key_check = now + KeyGuard::CHECK_INTERVAL
       KeyGuard.check(queues: @manager.queues)
@@ -191,7 +198,7 @@ module Cogworker
       error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
       job = JobUtil.unparseable_job(raw, queue: nil, error: error)
       buried = LuaScript.run(conn, BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
-                                               argv: [raw, job['failed_at'], JSON.generate(job)])
+                                                argv: [raw, job['failed_at'], JSON.generate(job)])
       return unless buried == 1
 
       Throughput.record('failed') # counted like Processor#bury_unparseable

data/lib/cogworker/status/server_middleware.rb

@@ -9,14 +9,20 @@ module Cogworker
         @expiration = expiration
       end
 
+      # Every status write is best-effort (BestEffort) — before `perform`
+      # as much as after: one failing used to count as the job failing.
       def call(_worker, job, queue)
-        write(job, queue, 'working')
-        yield
-        write(job, queue, 'complete')
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
-              error_class: e.class.name, error_message: JobUtil.error_message(e))
-        raise e
+        BestEffort.call('Status') { write(job, queue, 'working') }
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          BestEffort.call('Status') do
+            write(job, queue, JobUtil.terminal_failure?(job) ? 'failed' : 'retrying',
+                  error_class: e.class.name, error_message: JobUtil.error_message(e))
+          end
+          raise e
+        end
+        BestEffort.call('Status') { write(job, queue, 'complete') }
       end
 
       private

data/lib/cogworker/swarm.rb

@@ -59,6 +59,8 @@ module Cogworker
     def fork_child(slot)
       pid = ::Process.fork do
         Cogworker.reset_identity!
+        RedisErrors.reset! # the parent's outage state isn't this child's
+        DeferredReleases.reset! # nor are its pending releases
         CLI.new.run(@argv.dup)
         # `exit!`, not plain `exit`: `Launcher#run` only returns once
         # `Manager#stop!` has had its bounded (default 25s) chance to join

data/lib/cogworker/unique_jobs/release_middleware.rb

@@ -12,12 +12,16 @@ module Cogworker
     # `unique: :until_executed`. Registered unconditionally in every
     # `Config`, same as `Periodic::ReleaseMiddleware`.
     class ReleaseMiddleware
+      # Releasing is best-effort (BestEffort): a failed release leaves the
+      # lock to its TTL, and only the job's own exception is re-raised.
       def call(_worker, job, _queue)
-        yield
+        begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          release(job) if UniqueJobs.until_executed?(job) && JobUtil.terminal_failure?(job)
+          raise e
+        end
         release(job) if UniqueJobs.until_executed?(job)
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        release(job) if UniqueJobs.until_executed?(job) && JobUtil.terminal_failure?(job)
-        raise e
       end
 
       private
@@ -25,8 +29,15 @@ module Cogworker
       # Owner-checked: if this job's lock already expired and passed to a
       # newer copy, this must not release *that* one's lock (which would let
       # a third copy in while the newer one is still running).
+      # Owner-checked; retried later (DeferredReleases) if it fails, rather
+      # than leaving duplicates blocked for the lock's whole TTL.
       def release(job)
-        OwnedKey.delete(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'])
+        key = RedisKeys.unique_lock(UniqueJobs.digest(job))
+        released = BestEffort.call('Unique lock') do
+          OwnedKey.delete(key, job['jid'])
+          true
+        end
+        DeferredReleases.add(key, job['jid']) unless released
       end
     end
   end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.5.0'
+  VERSION = '0.5.1'
 end

data/lib/cogworker/web/layout.rb

@@ -380,7 +380,9 @@ module Cogworker
       # Redis cards), where the default wrapper would just nest one bordered
       # box inside another.
       def table(headers, rows, empty_message: 'Nothing here.', wrapped: true)
-        return %(<p class="text-muted" style="font-size: 13px; font-style: italic;">#{h(empty_message)}</p>) if rows.empty?
+        if rows.empty?
+          return %(<p class="text-muted" style="font-size: 13px; font-style: italic;">#{h(empty_message)}</p>)
+        end
 
         head = headers.map { |c| %(<th>#{h(c)}</th>) }.join
         body = rows.map do |row|

data/lib/cogworker/web/routes/jobs.rb

@@ -288,13 +288,13 @@ module Cogworker
         def retry_now_button(bucket, script_name, row, params)
           action = Layout.path(script_name, "jobs/#{bucket}/retry_now")
           form_with_extra(action, 'raw', row[:raw], 'retry now', params, variant: :primary, icon: 'arrow-clockwise',
-                                                                          extra: {})
+                                                                         extra: {})
         end
 
         def retry_button(bucket, script_name, row, params)
           action = Layout.path(script_name, "jobs/#{bucket}/retry")
           form_with_extra(action, 'raw', row[:raw], 'retry', params, variant: :primary, icon: 'arrow-clockwise',
-                                                                      extra: {})
+                                                                     extra: {})
         end
 
         # `Layout.form_button` only carries one hidden field. Every action
@@ -304,9 +304,9 @@ module Cogworker
         # since it isn't in the URL a plain form submits to) plus, for an
         # enqueued-job delete, `queue` (`Queue#delete` isn't keyed off the
         # raw payload alone the way the ZSETs are).
-        def form_with_extra(action, hidden_name, hidden_value, label, params, variant:, icon: nil, extra:)
+        def form_with_extra(action, hidden_name, hidden_value, label, params, variant:, extra:, icon: nil)
           hidden = extra.merge('status' => params['status'].to_s, 'q' => params['q'].to_s,
-                                'selected' => params['selected'].to_s)
+                               'selected' => params['selected'].to_s)
           hidden_inputs = hidden.map { |k, v| %(<input type="hidden" name="#{k}" value="#{Layout.h(v)}">) }.join
           classes = "btn #{Layout::BUTTON_VARIANTS.fetch(variant)}"
           <<~HTML

data/lib/cogworker/web/routes/overview.rb

@@ -613,7 +613,9 @@ module Cogworker
         def render_layout_b(script_name, selected_name)
           names = queue_names
           selected_name = names.first if (selected_name.nil? || selected_name.empty?) && !names.empty?
-          return %(<p class="text-muted">No queues yet — push a job to create one.</p>) if selected_name.nil? || selected_name.empty?
+          if selected_name.nil? || selected_name.empty?
+            return %(<p class="text-muted">No queues yet — push a job to create one.</p>)
+          end
 
           <<~HTML
             <div style="display: grid; grid-template-columns: minmax(240px, 320px) minmax(0, 1fr); gap: 16px; align-items: start;">
@@ -676,7 +678,7 @@ module Cogworker
           end.join
           paused_tag = q.paused? ? " #{Layout.badge('paused', variant: :warning)}" : ''
           header_actions = queue_pause_button(q, script_name, 'b') + retry_all_button(name, script_name,
-                                                                                       retrying_count)
+                                                                                      retrying_count)
           <<~HTML
             <section style="display: flex; flex-direction: column; gap: 16px; min-width: 0;">
               <div style="background: var(--color-surface); border-radius: var(--radius-md); box-shadow: var(--shadow-sm); padding: 18px;">

data/lib/cogworker/work_set.rb

@@ -3,6 +3,9 @@
 require 'json'
 
 module Cogworker
+  # Every job running right now, across all live processes: yields
+  # `[identity, thread id, Cogworker::Work]` from each process's
+  # `cogworker:workers:<identity>` Hash (only for processes ProcessSet knows).
   class WorkSet
     include Enumerable
 

data/lib/cogworker.rb

@@ -101,7 +101,7 @@ module Cogworker
     # No-op: exists so calling code doesn't need to change, but this gem
     # simply never coerces job args, so there is nothing to toggle — see
     # README "Job arguments".
-    def strict_args!(_value = true)
+    def strict_args!(_value = true) # rubocop:disable Style/OptionalBooleanParameter -- kept positional for calling-code compatibility
       true
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - vdonec
@@ -203,16 +203,20 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
+- README.md
 - exe/cogworker
 - exe/cogworkerswarm
 - lib/cogworker.rb
 - lib/cogworker/attempts.rb
 - lib/cogworker/basic_fetch.rb
+- lib/cogworker/best_effort.rb
 - lib/cogworker/cli.rb
 - lib/cogworker/client.rb
 - lib/cogworker/component.rb
 - lib/cogworker/config.rb
 - lib/cogworker/config_loader.rb
+- lib/cogworker/deferred_releases.rb
 - lib/cogworker/heartbeat.rb
 - lib/cogworker/history.rb
 - lib/cogworker/history/middleware.rb
@@ -239,6 +243,7 @@ files:
 - lib/cogworker/prometheus/exporter.rb
 - lib/cogworker/queue.rb
 - lib/cogworker/redis_connection.rb
+- lib/cogworker/redis_errors.rb
 - lib/cogworker/redis_keys.rb
 - lib/cogworker/reliable_fetch.rb
 - lib/cogworker/reliable_fetch/unsupported_error.rb