cogworker 0.2.2 → 0.3.0

data/lib/cogworker/swarm.rb

@@ -20,14 +20,30 @@ module Cogworker
     # that never exits (a hung Redis call outliving even Manager#stop!'s
     # own 25s join, or anything else) must not be able to wedge the whole
     # restart cycle, and the supervisor's entire signal-handling loop with
-    # it, forever.
-    GRACEFUL_STOP_TIMEOUT = 20
+    # it, forever. Kept well above a child's own worst-case clean shutdown
+    # (Manager#stop!'s 25s drain + up to BasicFetch::TIMEOUT + 1 waiting out
+    # fetching threads + ~1s for Heartbeat#stop!) so a child is never killed
+    # mid-`requeue_unfinished` — for `:basic`, that would lose the jobs.
+    GRACEFUL_STOP_TIMEOUT = 35
+
+    # Respawn backoff for a child that exits unexpectedly: none for the
+    # first crash after a healthy run, then doubling from
+    # RESPAWN_BASE_DELAY up to RESPAWN_MAX_DELAY while the slot keeps dying
+    # within HEALTHY_UPTIME of starting. Without it, a child crashing during
+    # boot (bad deploy, Redis unreachable, a typo in the init file) made the
+    # supervisor fork/crash in a tight loop forever.
+    HEALTHY_UPTIME = 30
+    RESPAWN_BASE_DELAY = 1
+    RESPAWN_MAX_DELAY = 60
 
     def initialize(argv, count: ENV.fetch('COGWORKER_COUNT', 1).to_i, phased: ENV['PHASED_RESTART'] == 'true')
       @argv = argv
       @count = [count, 1].max
       @phased = phased
       @children = {} # pid => slot index
+      @started_at = {} # slot => monotonic time of its current child's fork
+      @crash_streak = Hash.new(0) # slot => consecutive quick crashes
+      @pending_respawns = {} # slot => monotonic time it may be forked again
       @signal_queue = ::Queue.new
       @stopping = false
     end
@@ -56,6 +72,7 @@ module Cogworker
         ::Process.exit!(true)
       end
       @children[pid] = slot
+      @started_at[slot] = monotonic_now
       Cogworker.logger.info { "swarm: started child pid=#{pid} slot=#{slot}" }
     end
 
@@ -70,13 +87,16 @@ module Cogworker
       until @stopping && @children.empty?
         drain_signals
         reap_children
+        respawn_due_children
+        kill_stragglers
         sleep 0.2
       end
     end
 
     def drain_signals
       until @signal_queue.empty?
-        case @signal_queue.pop
+        signal = @signal_queue.pop
+        case signal
         when :quiet then relay(Signals::QUIET)
         when :stop then initiate_stop
         when :restart
@@ -93,9 +113,25 @@ module Cogworker
 
     def initiate_stop
       @stopping = true
+      @stop_deadline = monotonic_now + GRACEFUL_STOP_TIMEOUT
+      @pending_respawns.clear
       relay(Signals::STOP)
     end
 
+    # Same bound as a phased restart's `wait_for_exit`: a child that hasn't
+    # exited GRACEFUL_STOP_TIMEOUT after the stop was relayed is
+    # force-killed (once), so one hung child can't keep the supervisor —
+    # and whatever is waiting on it — alive forever.
+    def kill_stragglers
+      return unless @stop_deadline && monotonic_now > @stop_deadline
+
+      @stop_deadline = nil
+      @children.each_key do |pid|
+        Cogworker.logger.warn { "swarm: child pid=#{pid} didn't stop within #{GRACEFUL_STOP_TIMEOUT}s, killing" }
+        safe_kill(pid, 'KILL')
+      end
+    end
+
     # Non-phased: signal every child at once; the normal reap/respawn path
     # below brings each one back as it exits, so there's a brief capacity
     # dip but no explicit special-casing needed. Phased: replace children
@@ -155,7 +191,31 @@ module Cogworker
       if @stopping
         Cogworker.logger.info { "swarm: child pid=#{pid} slot=#{slot} exited" }
       else
-        Cogworker.logger.warn { "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly, respawning" }
+        schedule_respawn(pid, slot)
+      end
+    end
+
+    def schedule_respawn(pid, slot)
+      uptime = monotonic_now - @started_at.fetch(slot, monotonic_now)
+      @crash_streak[slot] = uptime < HEALTHY_UPTIME ? @crash_streak[slot] + 1 : 0
+      delay = respawn_delay(@crash_streak[slot])
+      Cogworker.logger.warn do
+        "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly after #{uptime.round(1)}s, " \
+          "respawning in #{delay}s"
+      end
+      @pending_respawns[slot] = monotonic_now + delay
+    end
+
+    def respawn_delay(streak)
+      return 0 if streak <= 1
+
+      [RESPAWN_BASE_DELAY * (2**(streak - 2)), RESPAWN_MAX_DELAY].min
+    end
+
+    def respawn_due_children
+      now = monotonic_now
+      @pending_respawns.select { |_slot, at| at <= now }.each_key do |slot|
+        @pending_respawns.delete(slot)
         fork_child(slot)
       end
     end

data/lib/cogworker/testing/empty_queue_error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Testing
+    # Raised by `Testing.perform_one`/`SomeJob.perform_one` when fake mode
+    # has nothing recorded for that class.
+    class EmptyQueueError < StandardError; end
+  end
+end

data/lib/cogworker/testing.rb

@@ -14,8 +14,9 @@ module Cogworker
   #   but the normalized job hash is appended to `jobs_for(klass_name)`
   #   instead of touching Redis — nothing is scheduled or executed. Use
   #   `SomeJob.jobs` (added to `Job::ClassMethods`) to assert what got
-  #   pushed, `SomeJob.clear`/`Testing.clear_jobs!` to reset between
-  #   examples.
+  #   pushed, `SomeJob.perform_one`/`SomeJob.drain`/`Testing.drain_all` to
+  #   then actually run them, `SomeJob.clear`/`Testing.clear_jobs!` to reset
+  #   between examples.
   # - `:inline` — the job runs synchronously, right here in the calling
   #   thread, through the real server middleware chain
   #   (`Cogworker.config.server_chain`) — same as `Processor#execute`, minus
@@ -75,8 +76,31 @@ module Cogworker
         registry.clear
       end
 
+      # Fake mode's "now run them": `perform_one` takes the oldest recorded
+      # job for a class off its fake queue and runs it via `perform_inline`
+      # (real server middleware chain, errors propagate); `drain` repeats
+      # until that class's queue is empty, `drain_all` until every class's
+      # is — both also running whatever those jobs push in turn. Also
+      # reachable as `SomeJob.perform_one`/`SomeJob.drain`.
+      def perform_one(klass_name)
+        job = jobs_for(klass_name).shift
+        raise EmptyQueueError, "no #{klass_name} jobs recorded" unless job
+
+        perform_inline(job)
+      end
+
+      def drain(klass_name)
+        perform_one(klass_name) until jobs_for(klass_name).empty?
+      end
+
+      def drain_all
+        while (klass_name = registry.find { |_name, jobs| jobs.any? }&.first)
+          drain(klass_name)
+        end
+      end
+
       # Runs one job synchronously through the real server middleware chain.
-      # Only ever called from `Client.push` while `inline?`.
+      # Called from `Client.push` while `inline?`, and by `perform_one`.
       def perform_inline(job)
         klass = Object.const_get(job['class'])
         worker = klass.new

data/lib/cogworker/unique_jobs/client_middleware.rb

@@ -9,15 +9,14 @@ module Cogworker
     # continue down the chain to the real push. A job that loses the claim
     # (another copy is already enqueued/scheduled/running) never reaches
     # `dispatch` at all — it sets `job['unique_skipped']` instead, which
-    # `Client.push` reads to return `nil` rather than a jid, the same
-    # "nothing was actually pushed" signal `sidekiq-unique-jobs` gives.
+    # `Client.push` reads to return `nil` rather than a jid — the
+    # "nothing was actually pushed" signal to the caller.
     class ClientMiddleware
       def call(_worker_class, job, _queue, redis_pool = Cogworker.config.redis_pool)
         return yield unless UniqueJobs.until_executed?(job)
 
         acquired = redis_pool.with do |c|
-          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'],
-                nx: true, ex: Cogworker.config.unique_lock_ttl)
+          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'], nx: true, ex: lock_ttl(job))
         end
 
         if acquired
@@ -26,6 +25,16 @@ module Cogworker
           job['unique_skipped'] = true
         end
       end
+
+      private
+
+      # Counted from when the job is due, not from now: a `perform_in`/
+      # `perform_at` further out than `unique_lock_ttl` used to lose its
+      # lock before it even ran, letting a duplicate in.
+      def lock_ttl(job)
+        wait = job['at'] ? [job['at'].to_f - Time.now.to_f, 0].max : 0
+        (Cogworker.config.unique_lock_ttl + wait).ceil
+      end
     end
   end
 end

data/lib/cogworker/unique_jobs/release_middleware.rb

@@ -22,8 +22,11 @@ module Cogworker
 
       private
 
+      # Owner-checked: if this job's lock already expired and passed to a
+      # newer copy, this must not release *that* one's lock (which would let
+      # a third copy in while the newer one is still running).
       def release(job)
-        Cogworker.config.redis { |c| c.del(RedisKeys.unique_lock(UniqueJobs.digest(job))) }
+        OwnedKey.delete(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'])
       end
     end
   end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.2.2'
+  VERSION = '0.3.0'
 end

data/lib/cogworker/web/layout.rb

@@ -178,8 +178,8 @@ module Cogworker
       end
 
       # Cluster-wide "quiet every process" (`Routes::Workers`'s own `POST
-      # /workers/pause_all`) — the mock's "Pause intake" button. Unlike real
-      # Sidekiq OSS (where quiet is a one-way trip back to a fresh process),
+      # /workers/pause_all`) — the mock's "Pause intake" button. Quiet isn't
+      # a one-way trip that only a fresh process can undo:
       # `Manager#quiet` here is a plain in-memory flag, so `resume_intake_
       # button` below is a genuine, symmetric undo rather than a full
       # process restart.
@@ -237,7 +237,7 @@ module Cogworker
       # each one's visible text to the *browser's local* time, in
       # `Web.time_format`. Registered on `htmx:load`, which htmx fires once
       # for the initial page **and** again after every fragment swap — so
-      # newly-swapped-in timestamps (an auto-refreshed Busy/Scheduled/Dead
+      # newly-swapped-in timestamps (an auto-refreshed Workers/Jobs/Schedules
       # tab) get the same treatment without a full reload.
       #
       # `window.cogworkerFormatTime` is the same formatter exposed globally

data/lib/cogworker/web/routes/jobs.rb

@@ -14,9 +14,9 @@ module Cogworker
       # retry timeline, last error, actions) in place of `Routes::Dead`'s
       # flat rows. (*`Routes::Queues`' own queue-level browsing survives
       # separately, as `Routes::Overview` layout B — this tab is the
-      # cross-queue, cross-status view; the two overlap a little on purpose,
-      # the same way a real Sidekiq-style admin wants both "what's in this
-      # queue" and "find this one job wherever it is".)
+      # cross-queue, cross-status view; the two overlap a little on purpose:
+      # an operator wants both "what's in this queue" and "find this one job
+      # wherever it is".)
       #
       # `?status=`/`?q=`/`?selected=` are plain query params, read fresh on
       # every request and carried through the self-poll and every action's
@@ -67,7 +67,7 @@ module Cogworker
             Jobs.respond(self, params)
           end
 
-          # Same "graduate back onto its queue" move `Routes::Dead#retry`/
+          # Same "graduate back onto its queue" move `/jobs/dead/retry` below/
           # `Cogworker::Scheduled#graduate` make — `zrem` winning (not
           # losing) gates it so two tabs retrying the same entry at once
           # can't both requeue it; the `redis` gem's `#zrem` returns a
@@ -76,11 +76,7 @@ module Cogworker
           app.post('/jobs/retrying/retry_now') do
             raw = params['raw']
             Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::RETRY, raw)
-                job = JSON.parse(raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
+              JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw)
             end
             Jobs.respond(self, params)
           end
@@ -102,11 +98,7 @@ module Cogworker
           app.post('/jobs/dead/retry') do
             raw = params['raw']
             Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::DEAD, raw)
-                job = JSON.parse(raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
+              JobUtil.claim_and_requeue(c, RedisKeys::DEAD, raw)
             end
             Jobs.respond(self, params)
           end
@@ -254,15 +246,23 @@ module Cogworker
           when :scheduled
             delete_button('scheduled', script_name, row, params)
           when :retrying
-            retry_now_button('retrying', script_name, row, params) + delete_button('retrying', script_name, row,
-                                                                                    params)
+            retry_html = requeueable?(row) ? retry_now_button('retrying', script_name, row, params) : ''
+            retry_html + delete_button('retrying', script_name, row, params)
           when :dead
-            retry_button('dead', script_name, row, params) + delete_button('dead', script_name, row, params)
+            retry_html = requeueable?(row) ? retry_button('dead', script_name, row, params) : ''
+            retry_html + delete_button('dead', script_name, row, params)
           else
             ''
           end
         end
 
+        # No Retry for an entry that can't go back on a queue (no queue of
+        # its own) or that would only fail straight back into Dead (an
+        # unparseable payload's wrapper) — Delete is all that makes sense.
+        def requeueable?(row)
+          row[:klass] != JobUtil::UNPARSEABLE_CLASS && !JobUtil.requeueable(row[:raw].to_s).nil?
+        end
+
         def delete_button(bucket, script_name, row, params, extra: {})
           action = Layout.path(script_name, "jobs/#{bucket}/delete")
           form_with_extra(action, 'raw', row[:raw], 'delete', params, variant: :danger, icon: 'trash', extra: extra)

data/lib/cogworker/web/routes/overview.rb

@@ -125,8 +125,8 @@ module Cogworker
           end
 
           # `raw` is the exact JSON string the job entry was pushed with —
-          # same "raw" identity `Routes::Dead`/`Routes::Retries` already key
-          # their own per-row delete off — so `Queue#delete` can `LREM` it
+          # same "raw" identity `Routes::Jobs`' dead/retrying actions already
+          # key their own per-row delete off — so `Queue#delete` can `LREM` it
           # back out of the list.
           app.post('/overview/:name/delete') do
             name = url_params('name')
@@ -167,15 +167,9 @@ module Cogworker
         def retry_all(name)
           entries = Cogworker.config.redis { |c| c.zrange(RedisKeys::RETRY, 0, -1) }
           entries.each do |raw|
-            job = JSON.parse(raw)
-            next unless job['queue'] == name
+            next unless JobUtil.requeueable(raw)&.fetch('queue') == name
 
-            Cogworker.config.redis do |c|
-              if c.zrem(RedisKeys::RETRY, raw)
-                c.sadd(RedisKeys::QUEUES, job['queue'])
-                c.lpush(RedisKeys.queue(job['queue']), raw)
-              end
-            end
+            Cogworker.config.redis { |c| JobUtil.claim_and_requeue(c, RedisKeys::RETRY, raw) }
           end
         end
 
@@ -447,8 +441,8 @@ module Cogworker
         # `processed`/`failed`. Shared between the initial render
         # (`throughput_section`, baked into the page) and `/overview/
         # throughput_data` (what the chart's own poll re-fetches from then
-        # on), so the two can never drift apart — same pattern `Routes::
-        # Stats#chart_data_payload` already uses for its own chart.
+        # on), so the two can never drift apart — same pattern
+        # `runs_per_day_payload` above uses for the Runs-per-day chart.
         def throughput_payload
           series = Cogworker::Throughput.series
           {

data/lib/cogworker/web/routes/schedules.rb

@@ -52,12 +52,12 @@ module Cogworker
           end
 
           app.post('/schedules/:pjid/disable') do
-            Cogworker.config.redis { |c| c.sadd(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
+            Cogworker.config.redis { |c| c.sadd?(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
             Routes::Schedules.respond(self)
           end
 
           app.post('/schedules/:pjid/enable') do
-            Cogworker.config.redis { |c| c.srem(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
+            Cogworker.config.redis { |c| c.srem?(RedisKeys::PERIODIC_DISABLED, url_params('pjid')) }
             Routes::Schedules.respond(self)
           end
         end

data/lib/cogworker/web.rb

@@ -20,7 +20,7 @@ module Cogworker
     # Every built-in tab lives under here; touching each constant once
     # forces Zeitwerk to load that file (and run its
     # `Cogworker::Web.register(...)` bottom-of-file side effect) — nothing
-    # else ever references `Routes::Queues` etc. by name, so without this
+    # else ever references `Routes::Overview` etc. by name, so without this
     # they would simply never load.
     BUILT_IN_ROUTE_NAMES = %i[Overview Jobs Schedules Workers History SaveSession].freeze
     DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
@@ -94,8 +94,20 @@ module Cogworker
         @tabs ||= {}
       end
 
+      # The built-in session cookie's signing secret. The gem itself never
+      # reads the session — it's there for an auth middleware added via
+      # `.use` — but a random per-process default means each Puma worker
+      # signs with its own key, so that middleware's session doesn't survive
+      # a request landing on a different worker. Set it explicitly
+      # (`Cogworker::Web.session_secret = ...`, at least 64 characters) or
+      # via `COGWORKER_SESSION_SECRET` for any multi-process deployment.
       def session_secret
-        @session_secret ||= SecureRandom.hex(32)
+        @session_secret ||= ENV.fetch('COGWORKER_SESSION_SECRET') { SecureRandom.hex(32) }
+      end
+
+      def session_secret=(secret)
+        @session_secret = secret
+        @app = nil
       end
 
       # How every `Layout.time_tag` timestamp is rendered — a Ruby
@@ -132,10 +144,11 @@ module Cogworker
       end
 
       # How often (in seconds) every auto-refreshing tab polls while the
-      # global live-update toggle is on — both the htmx-polled Busy/Stats/
-      # Queues tabs (`Layout.poll_div`'s `hx-trigger="every Ns [...]"`) and
-      # History's own AG Grid `refreshRows()` JS poll share this one value,
-      # so there's a single knob rather than one per tab. Set this from your
+      # global live-update toggle is on — the htmx-polled Overview/Jobs/
+      # Schedules/Workers tabs (`Layout.poll_div`'s `hx-trigger="every Ns
+      # [...]"`), History's AG Grid poll and Overview's Chart.js polls all
+      # share this one value, so there's a single knob rather than one per
+      # tab. Set this from your
       # own init file/config.ru, e.g. `Cogworker::Web.live_update_interval = 10`.
       def live_update_interval
         @live_update_interval ||= DEFAULT_LIVE_UPDATE_INTERVAL
@@ -237,10 +250,10 @@ module Cogworker
         builder.use(Rack::Session::Cookie, secret: session_secret, key: 'cogworker.session')
         # No `cache_control:` (no `immutable`/long `max-age`): these files
         # are plain, unfingerprinted paths that *do* change — every time
-        # this gem's Tailwind bundle gets rebuilt, or on any gem upgrade —
+        # a vendored stylesheet/script gets edited, or on any gem upgrade —
         # and `immutable` previously told browsers to keep serving a stale
         # cached copy under the old URL for up to a year with no
-        # revalidation at all (caught by hand: a real edit to tailwind.css
+        # revalidation at all (caught by hand: a real edit to a vendored stylesheet
         # didn't show up in an already-open tab until a hard reload).
         # `Rack::Static`/`Rack::Files` already sends `Last-Modified` and
         # honors conditional GETs by default, which is all that's needed

data/lib/cogworker.rb

@@ -2,6 +2,9 @@
 
 require 'socket'
 require 'securerandom'
+# `Time#iso8601` (Logging, Web::Layout.time_tag) only moved into core in
+# Ruby 3.4; on 3.1-3.3 it needs the stdlib extension loaded explicitly.
+require 'time'
 require 'zeitwerk'
 
 # Redis-backed background job processing: worker DSL, periodic (cron)
@@ -85,7 +88,8 @@ module Cogworker
     end
 
     # No-op: exists so calling code doesn't need to change, but this gem
-    # simply never coerces job args, so there is nothing to toggle.
+    # simply never coerces job args, so there is nothing to toggle — see
+    # README "Job arguments".
     def strict_args!(_value = true)
       true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - vdonec
@@ -224,10 +224,12 @@ files:
 - lib/cogworker/logging.rb
 - lib/cogworker/manager.rb
 - lib/cogworker/middleware/chain.rb
+- lib/cogworker/owned_key.rb
 - lib/cogworker/periodic/claim.lua
 - lib/cogworker/periodic/entry.rb
 - lib/cogworker/periodic/manager.rb
 - lib/cogworker/periodic/release_middleware.rb
+- lib/cogworker/periodic/running_lock.rb
 - lib/cogworker/periodic/ticker.rb
 - lib/cogworker/process.rb
 - lib/cogworker/process_set.rb
@@ -236,6 +238,7 @@ files:
 - lib/cogworker/queue.rb
 - lib/cogworker/redis_connection.rb
 - lib/cogworker/redis_keys.rb
+- lib/cogworker/reliable_fetch.rb
 - lib/cogworker/scheduled.rb
 - lib/cogworker/signals.rb
 - lib/cogworker/stats.rb
@@ -246,6 +249,7 @@ files:
 - lib/cogworker/status/worker.rb
 - lib/cogworker/swarm.rb
 - lib/cogworker/testing.rb
+- lib/cogworker/testing/empty_queue_error.rb
 - lib/cogworker/throughput.rb
 - lib/cogworker/unique_jobs.rb
 - lib/cogworker/unique_jobs/client_middleware.rb