cogworker 0.2.1 → 0.3.0

data/lib/cogworker/swarm.rb

@@ -20,14 +20,30 @@ module Cogworker
     # that never exits (a hung Redis call outliving even Manager#stop!'s
     # own 25s join, or anything else) must not be able to wedge the whole
     # restart cycle, and the supervisor's entire signal-handling loop with
-    # it, forever.
-    GRACEFUL_STOP_TIMEOUT = 20
+    # it, forever. Kept well above a child's own worst-case clean shutdown
+    # (Manager#stop!'s 25s drain + up to BasicFetch::TIMEOUT + 1 waiting out
+    # fetching threads + ~1s for Heartbeat#stop!) so a child is never killed
+    # mid-`requeue_unfinished` — for `:basic`, that would lose the jobs.
+    GRACEFUL_STOP_TIMEOUT = 35
+
+    # Respawn backoff for a child that exits unexpectedly: none for the
+    # first crash after a healthy run, then doubling from
+    # RESPAWN_BASE_DELAY up to RESPAWN_MAX_DELAY while the slot keeps dying
+    # within HEALTHY_UPTIME of starting. Without it, a child crashing during
+    # boot (bad deploy, Redis unreachable, a typo in the init file) made the
+    # supervisor fork/crash in a tight loop forever.
+    HEALTHY_UPTIME = 30
+    RESPAWN_BASE_DELAY = 1
+    RESPAWN_MAX_DELAY = 60
 
     def initialize(argv, count: ENV.fetch('COGWORKER_COUNT', 1).to_i, phased: ENV['PHASED_RESTART'] == 'true')
       @argv = argv
       @count = [count, 1].max
       @phased = phased
       @children = {} # pid => slot index
+      @started_at = {} # slot => monotonic time of its current child's fork
+      @crash_streak = Hash.new(0) # slot => consecutive quick crashes
+      @pending_respawns = {} # slot => monotonic time it may be forked again
       @signal_queue = ::Queue.new
       @stopping = false
     end
@@ -56,6 +72,7 @@ module Cogworker
         ::Process.exit!(true)
       end
       @children[pid] = slot
+      @started_at[slot] = monotonic_now
       Cogworker.logger.info { "swarm: started child pid=#{pid} slot=#{slot}" }
     end
 
@@ -70,13 +87,16 @@ module Cogworker
       until @stopping && @children.empty?
         drain_signals
         reap_children
+        respawn_due_children
+        kill_stragglers
         sleep 0.2
       end
     end
 
     def drain_signals
       until @signal_queue.empty?
-        case @signal_queue.pop
+        signal = @signal_queue.pop
+        case signal
         when :quiet then relay(Signals::QUIET)
         when :stop then initiate_stop
         when :restart
@@ -93,9 +113,25 @@ module Cogworker
 
     def initiate_stop
       @stopping = true
+      @stop_deadline = monotonic_now + GRACEFUL_STOP_TIMEOUT
+      @pending_respawns.clear
       relay(Signals::STOP)
     end
 
+    # Same bound as a phased restart's `wait_for_exit`: a child that hasn't
+    # exited GRACEFUL_STOP_TIMEOUT after the stop was relayed is
+    # force-killed (once), so one hung child can't keep the supervisor —
+    # and whatever is waiting on it — alive forever.
+    def kill_stragglers
+      return unless @stop_deadline && monotonic_now > @stop_deadline
+
+      @stop_deadline = nil
+      @children.each_key do |pid|
+        Cogworker.logger.warn { "swarm: child pid=#{pid} didn't stop within #{GRACEFUL_STOP_TIMEOUT}s, killing" }
+        safe_kill(pid, 'KILL')
+      end
+    end
+
     # Non-phased: signal every child at once; the normal reap/respawn path
     # below brings each one back as it exits, so there's a brief capacity
     # dip but no explicit special-casing needed. Phased: replace children
@@ -155,7 +191,31 @@ module Cogworker
       if @stopping
         Cogworker.logger.info { "swarm: child pid=#{pid} slot=#{slot} exited" }
       else
-        Cogworker.logger.warn { "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly, respawning" }
+        schedule_respawn(pid, slot)
+      end
+    end
+
+    def schedule_respawn(pid, slot)
+      uptime = monotonic_now - @started_at.fetch(slot, monotonic_now)
+      @crash_streak[slot] = uptime < HEALTHY_UPTIME ? @crash_streak[slot] + 1 : 0
+      delay = respawn_delay(@crash_streak[slot])
+      Cogworker.logger.warn do
+        "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly after #{uptime.round(1)}s, " \
+          "respawning in #{delay}s"
+      end
+      @pending_respawns[slot] = monotonic_now + delay
+    end
+
+    def respawn_delay(streak)
+      return 0 if streak <= 1
+
+      [RESPAWN_BASE_DELAY * (2**(streak - 2)), RESPAWN_MAX_DELAY].min
+    end
+
+    def respawn_due_children
+      now = monotonic_now
+      @pending_respawns.select { |_slot, at| at <= now }.each_key do |slot|
+        @pending_respawns.delete(slot)
         fork_child(slot)
       end
     end

data/lib/cogworker/testing/empty_queue_error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Testing
+    # Raised by `Testing.perform_one`/`SomeJob.perform_one` when fake mode
+    # has nothing recorded for that class.
+    class EmptyQueueError < StandardError; end
+  end
+end

data/lib/cogworker/testing.rb

@@ -14,8 +14,9 @@ module Cogworker
   #   but the normalized job hash is appended to `jobs_for(klass_name)`
   #   instead of touching Redis — nothing is scheduled or executed. Use
   #   `SomeJob.jobs` (added to `Job::ClassMethods`) to assert what got
-  #   pushed, `SomeJob.clear`/`Testing.clear_jobs!` to reset between
-  #   examples.
+  #   pushed, `SomeJob.perform_one`/`SomeJob.drain`/`Testing.drain_all` to
+  #   then actually run them, `SomeJob.clear`/`Testing.clear_jobs!` to reset
+  #   between examples.
   # - `:inline` — the job runs synchronously, right here in the calling
   #   thread, through the real server middleware chain
   #   (`Cogworker.config.server_chain`) — same as `Processor#execute`, minus
@@ -75,8 +76,31 @@ module Cogworker
         registry.clear
       end
 
+      # Fake mode's "now run them": `perform_one` takes the oldest recorded
+      # job for a class off its fake queue and runs it via `perform_inline`
+      # (real server middleware chain, errors propagate); `drain` repeats
+      # until that class's queue is empty, `drain_all` until every class's
+      # is — both also running whatever those jobs push in turn. Also
+      # reachable as `SomeJob.perform_one`/`SomeJob.drain`.
+      def perform_one(klass_name)
+        job = jobs_for(klass_name).shift
+        raise EmptyQueueError, "no #{klass_name} jobs recorded" unless job
+
+        perform_inline(job)
+      end
+
+      def drain(klass_name)
+        perform_one(klass_name) until jobs_for(klass_name).empty?
+      end
+
+      def drain_all
+        while (klass_name = registry.find { |_name, jobs| jobs.any? }&.first)
+          drain(klass_name)
+        end
+      end
+
       # Runs one job synchronously through the real server middleware chain.
-      # Only ever called from `Client.push` while `inline?`.
+      # Called from `Client.push` while `inline?`, and by `perform_one`.
       def perform_inline(job)
         klass = Object.const_get(job['class'])
         worker = klass.new

data/lib/cogworker/unique_jobs/client_middleware.rb

@@ -9,15 +9,14 @@ module Cogworker
     # continue down the chain to the real push. A job that loses the claim
     # (another copy is already enqueued/scheduled/running) never reaches
     # `dispatch` at all — it sets `job['unique_skipped']` instead, which
-    # `Client.push` reads to return `nil` rather than a jid, the same
-    # "nothing was actually pushed" signal `sidekiq-unique-jobs` gives.
+    # `Client.push` reads to return `nil` rather than a jid — the
+    # "nothing was actually pushed" signal to the caller.
     class ClientMiddleware
       def call(_worker_class, job, _queue, redis_pool = Cogworker.config.redis_pool)
         return yield unless UniqueJobs.until_executed?(job)
 
         acquired = redis_pool.with do |c|
-          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'],
-                nx: true, ex: Cogworker.config.unique_lock_ttl)
+          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'], nx: true, ex: lock_ttl(job))
         end
 
         if acquired
@@ -26,6 +25,16 @@ module Cogworker
           job['unique_skipped'] = true
         end
       end
+
+      private
+
+      # Counted from when the job is due, not from now: a `perform_in`/
+      # `perform_at` further out than `unique_lock_ttl` used to lose its
+      # lock before it even ran, letting a duplicate in.
+      def lock_ttl(job)
+        wait = job['at'] ? [job['at'].to_f - Time.now.to_f, 0].max : 0
+        (Cogworker.config.unique_lock_ttl + wait).ceil
+      end
     end
   end
 end

data/lib/cogworker/unique_jobs/release_middleware.rb

@@ -22,8 +22,11 @@ module Cogworker
 
       private
 
+      # Owner-checked: if this job's lock already expired and passed to a
+      # newer copy, this must not release *that* one's lock (which would let
+      # a third copy in while the newer one is still running).
       def release(job)
-        Cogworker.config.redis { |c| c.del(RedisKeys.unique_lock(UniqueJobs.digest(job))) }
+        OwnedKey.delete(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'])
       end
     end
   end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 end

data/lib/cogworker/web/history_query.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  class Web
+    # Server side of the History tab's AG Grid (Infinite Row Model): one
+    # block of rows at a time — `[rows, total]` for `offset`/`limit` —
+    # instead of handing the browser every retained entry up front.
+    #
+    # `sort`/`filters` arrive in AG Grid's own `sortModel`/`filterModel`
+    # shapes (`[{ 'colId' => ..., 'sort' => 'asc'|'desc' }]` /
+    # `{ colId => { 'filterType' => 'text'|'number', 'type' => ..., ... } }`,
+    # optionally combined via `operator`/`conditions`), and `COLUMNS` below
+    # computes each column's value the same way `Routes::History#grid_script`
+    # displays it, so a server-side filter matches what the viewer sees.
+    #
+    # Two paths:
+    # - no filters, sorted by Finished (the default): a plain `ZREVRANGE`/
+    #   `ZRANGE` of just the requested slice, total from `ZCARD` — the
+    #   everyday case stays O(limit), whatever `History.max_entries` is.
+    # - anything else (a column filter, `?jid=`, sorting by another column):
+    #   a scan of the whole status ZSET in chunks, filtered/sorted in Ruby,
+    #   then sliced. Bounded by retention (`History.retention_days`/
+    #   `max_entries`), and only the requested slice ever leaves the server.
+    class HistoryQuery
+      SCAN_CHUNK = 1000
+      MAX_LIMIT = 1000
+      DEFAULT_SORT = 'finished_at'
+
+      COLUMNS = {
+        'finished_at' => ->(e) { e['finished_at'].to_f },
+        'class' => ->(e) { e['class'].to_s },
+        'queue' => ->(e) { e['queue'].to_s },
+        'jid' => ->(e) { e['jid'].to_s },
+        'args' => ->(e) { JSON.generate(e['args']) },
+        'status' => ->(e) { e['status'].to_s },
+        'duration' => ->(e) { ((e['finished_at'].to_f - e['started_at'].to_f) * 1000).round },
+        'error' => ->(e) { e['error_class'] ? "#{e['error_class']}: #{e['error_message']}" : '' }
+      }.freeze
+
+      def initialize(status:, jid: '', sort: [], filters: {})
+        @key = Cogworker::History::Storage::LIST_KEYS.fetch(status, Cogworker::History::Storage::LIST_KEYS.fetch('all'))
+        @jid = jid.to_s
+        @sort = Array(sort).find { |s| s.is_a?(Hash) && COLUMNS.key?(s['colId']) }
+        @filters = filters.is_a?(Hash) ? filters.select { |col, _| COLUMNS.key?(col) } : {}
+      end
+
+      def fetch(offset, limit)
+        offset = [offset.to_i, 0].max
+        limit = limit.to_i.clamp(0, MAX_LIMIT)
+        fast_path? ? fetch_slice(offset, limit) : fetch_scanned(offset, limit)
+      end
+
+      private
+
+      def sort_col
+        @sort ? @sort['colId'] : DEFAULT_SORT
+      end
+
+      def ascending?
+        @sort && @sort['sort'] == 'asc'
+      end
+
+      def fast_path?
+        @jid.empty? && @filters.empty? && sort_col == DEFAULT_SORT
+      end
+
+      def fetch_slice(offset, limit)
+        return [[], Cogworker.config.redis { |c| c.zcard(@key) }] if limit.zero?
+
+        stop = offset + limit - 1
+        total, raw = Cogworker.config.redis do |c|
+          [c.zcard(@key), ascending? ? c.zrange(@key, offset, stop) : c.zrevrange(@key, offset, stop)]
+        end
+        [raw.map { |r| JSON.parse(r) }, total]
+      end
+
+      def fetch_scanned(offset, limit)
+        matched = scan.select { |entry| matches?(entry) }
+        [sort(matched)[offset, limit] || [], matched.size]
+      end
+
+      # Ties stay newest-first (the scan's own order) either direction —
+      # the index tiebreaker also makes Ruby's unstable `sort_by` stable.
+      def sort(entries)
+        getter = COLUMNS.fetch(sort_col)
+        keyed = entries.each_with_index.map { |entry, i| [getter.call(entry), i, entry] }
+        if ascending?
+          keyed.sort_by { |value, i, _| [value, i] }.map(&:last)
+        else
+          keyed.sort_by { |value, i, _| [value, -i] }.reverse.map(&:last)
+        end
+      end
+
+      def scan
+        needle = @jid.empty? ? nil : JSON.generate('jid' => @jid)[1..-2]
+        entries = []
+        Cogworker.config.redis do |c|
+          start = 0
+          loop do
+            chunk = c.zrevrange(@key, start, start + SCAN_CHUNK - 1)
+            chunk.each do |raw|
+              next if needle && !raw.include?(needle) # cheap pre-check before parsing
+
+              entry = JSON.parse(raw)
+              entries << entry if @jid.empty? || entry['jid'] == @jid
+            end
+            break if chunk.size < SCAN_CHUNK
+
+            start += SCAN_CHUNK
+          end
+        end
+        entries
+      end
+
+      def matches?(entry)
+        @filters.all? { |col, model| model_matches?(model, COLUMNS.fetch(col).call(entry)) }
+      end
+
+      def model_matches?(model, value)
+        return true unless model.is_a?(Hash)
+
+        if model['conditions'].is_a?(Array)
+          results = model['conditions'].map { |cond| condition_matches?(model['filterType'], cond, value) }
+          model['operator'] == 'OR' ? results.any? : results.all?
+        else
+          condition_matches?(model['filterType'], model, value)
+        end
+      end
+
+      def condition_matches?(filter_type, cond, value)
+        return true unless cond.is_a?(Hash)
+
+        case cond['filterType'] || filter_type
+        when 'number' then number_matches?(cond, value)
+        else text_matches?(cond, value.to_s)
+        end
+      end
+
+      # AG Grid's default text filter semantics: case-insensitive.
+      def text_matches?(cond, value)
+        value = value.downcase
+        needle = cond['filter'].to_s.downcase
+        case cond['type']
+        when 'contains' then value.include?(needle)
+        when 'notContains' then !value.include?(needle)
+        when 'equals' then value == needle
+        when 'notEqual' then value != needle
+        when 'startsWith' then value.start_with?(needle)
+        when 'endsWith' then value.end_with?(needle)
+        when 'blank' then value.empty?
+        when 'notBlank' then !value.empty?
+        else true
+        end
+      end
+
+      # AG Grid's default number filter semantics (`inRange` exclusive).
+      def number_matches?(cond, value)
+        return value.nil? if cond['type'] == 'blank'
+        return !value.nil? if cond['type'] == 'notBlank'
+
+        target = Float(cond['filter'], exception: false)
+        return true if target.nil?
+
+        case cond['type']
+        when 'equals' then value == target
+        when 'notEqual' then value != target
+        when 'lessThan' then value < target
+        when 'lessThanOrEqual' then value <= target
+        when 'greaterThan' then value > target
+        when 'greaterThanOrEqual' then value >= target
+        when 'inRange'
+          to = Float(cond['filterTo'], exception: false)
+          to.nil? || (value > target && value < to)
+        else true
+        end
+      end
+    end
+  end
+end

data/lib/cogworker/web/layout.rb

@@ -178,8 +178,8 @@ module Cogworker
       end
 
       # Cluster-wide "quiet every process" (`Routes::Workers`'s own `POST
-      # /workers/pause_all`) — the mock's "Pause intake" button. Unlike real
-      # Sidekiq OSS (where quiet is a one-way trip back to a fresh process),
+      # /workers/pause_all`) — the mock's "Pause intake" button. Quiet isn't
+      # a one-way trip that only a fresh process can undo:
       # `Manager#quiet` here is a plain in-memory flag, so `resume_intake_
       # button` below is a genuine, symmetric undo rather than a full
       # process restart.
@@ -237,7 +237,7 @@ module Cogworker
       # each one's visible text to the *browser's local* time, in
       # `Web.time_format`. Registered on `htmx:load`, which htmx fires once
       # for the initial page **and** again after every fragment swap — so
-      # newly-swapped-in timestamps (an auto-refreshed Busy/Scheduled/Dead
+      # newly-swapped-in timestamps (an auto-refreshed Workers/Jobs/Schedules
       # tab) get the same treatment without a full reload.
       #
       # `window.cogworkerFormatTime` is the same formatter exposed globally