cogworker 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aad05a1deaf85505c91f0b6ebe11199fc918b6b3e3831efa5ddb753381cf824
-  data.tar.gz: 760b88ec2cc3075f4bbfe152062bc8f1e24b71ce2c71984f9702c0cbdb7da81f
+  metadata.gz: 1eb0a27dc233c6009a4a78103adb88ddc76aa813dd53e1f5905e419f028f4d11
+  data.tar.gz: 511c286faf085250e00bf5fb7ce4f45bfaac44731a60d1a1c92779095e758147
 SHA512:
-  metadata.gz: fc050c6eff33d2fcc5f61629d46e88452912d3f798c49b49fdc53d21ca525403a575e89e13c9e34307cba312eab7307baebc80a187661a032592fb31eb797854
-  data.tar.gz: a9ef2e674633abe4085b6e6e21eb74c3ec735b3ab7c4f6305f274172049e41058477b03e5a5e8105cc6b0afdeadcafc9babba07bbdd81855b54532585a9a946e
+  metadata.gz: 1663b0475259d7bd30b5dd5718dbccc38bd53e1b4494a6574d4159229943f9a789754549171bee406134def631524733f4bb9e01e901ff552f3ad778f15f5599
+  data.tar.gz: 8a32c328155045a7c19049bba52478dbd2b806b6cf875af3d83cf7a05dfc63d1eda59ae36d66dcd19a4d5494ca8613b13a62620bddbf13fd57d08f6d297927d3

data/lib/cogworker/attempts.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  # Per-job append-only log of failed attempts (attempt number, when, error
+  # class/message, and whether that failure routed the job to retry or to
+  # dead) — what `Processor#route_failure` appends to on every failure, and
+  # what the Web UI's Jobs tab reads to render a retry timeline in a job's
+  # detail panel.
+  #
+  # Distinct from `Cogworker::History`: History is every *completed* run
+  # (success or failure) across every job, capped and trimmed independently
+  # of any one job's fate. This is the *in-progress* failure trail for one
+  # still-retrying-or-dead job, keyed by JID, growing one entry per attempt
+  # until that job is deleted from Retries/Dead (`.clear`) or the key
+  # expires on its own (`TTL_SECONDS`) if nothing ever does.
+  module Attempts
+    MAX_ENTRIES = 25
+    TTL_SECONDS = 30 * 24 * 60 * 60 # 30 days — bounds an abandoned dead entry's log even if nothing ever calls `.clear`
+
+    module_function
+
+    def record(jid, attempt:, error:, outcome:)
+      entry = JSON.generate(
+        'attempt' => attempt, 'failed_at' => Time.now.to_f, 'outcome' => outcome,
+        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000]
+      )
+      key = RedisKeys.job_attempts(jid)
+      Cogworker.config.redis do |c|
+        c.rpush(key, entry)
+        c.ltrim(key, -MAX_ENTRIES, -1)
+        c.expire(key, TTL_SECONDS)
+      end
+    end
+
+    # Oldest attempt first — the order a timeline reads top-to-bottom.
+    def for(jid)
+      raw = Cogworker.config.redis { |c| c.lrange(RedisKeys.job_attempts(jid), 0, -1) }
+      raw.map { |r| JSON.parse(r) }
+    end
+
+    def clear(jid)
+      Cogworker.config.redis { |c| c.del(RedisKeys.job_attempts(jid)) }
+    end
+  end
+end

data/lib/cogworker/basic_fetch.rb

@@ -13,15 +13,38 @@ module Cogworker
       @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
     end
 
+    # Re-reads `cogworker:paused_queues` on every single fetch (not once at
+    # `initialize`, and not cached) — a `Queue#pause!`/`#resume!` from the
+    # Web UI takes effect on this processor's very next cycle, not just for
+    # ones started after the toggle. If every one of this processor's
+    # queues is currently paused, there's nothing left to `BRPOP` at all
+    # (an empty key list is invalid) — sleep out one `TIMEOUT` instead of
+    # returning immediately, same pause `BRPOP` itself would have caused,
+    # so `Processor#run`'s loop doesn't spin hot polling Redis in a tight
+    # loop with no rate limit.
     def retrieve_work
-      keys = @queue_keys.shuffle
-      result = Cogworker.config.redis { |c| c.brpop(*keys, timeout: TIMEOUT) }
+      keys = active_keys
+      if keys.empty?
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      result = Cogworker.config.redis { |c| c.brpop(*keys.shuffle, timeout: TIMEOUT) }
       return nil unless result
 
       queue_key, raw_job = result
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
 
+    private
+
+    def active_keys
+      paused = Cogworker.config.redis { |c| c.smembers(RedisKeys::PAUSED_QUEUES) }
+      return @queue_keys if paused.empty?
+
+      @queue_keys.reject { |k| paused.include?(k.delete_prefix(RedisKeys::QUEUE_PREFIX)) }
+    end
+
     UnitOfWork = Struct.new(:queue, :raw_job)
   end
 end

data/lib/cogworker/heartbeat.rb

@@ -5,8 +5,9 @@ require 'redis'
 
 module Cogworker
   # Publishes this process's presence (for ProcessSet) and listens for
-  # remote quiet!/stop! requests (for Process#quiet!/#stop! issued by another
-  # process, e.g. a self-targeting WorkerKiller or the Web UI). Both the
+  # remote quiet!/resume!/stop! requests (for Process#quiet!/#resume!/#stop!
+  # issued by another process, e.g. a self-targeting WorkerKiller or the Web
+  # UI). Both the
   # heartbeat loop and the pub/sub subscriber must only start *after* a
   # cogworkerswarm fork, never before, in the parent — otherwise the thread
   # simply doesn't exist in the child, and a lock that thread held could
@@ -119,6 +120,7 @@ module Cogworker
     def dispatch(message)
       case message
       when 'quiet' then @manager.quiet!
+      when 'resume' then @manager.unquiet!
       when 'stop' then remote_stop!
       else Cogworker.logger.warn { "Unknown signal message: #{message}" }
       end

data/lib/cogworker/manager.rb

@@ -25,6 +25,10 @@ module Cogworker
       @quiet = true
     end
 
+    def unquiet!
+      @quiet = false
+    end
+
     def quiet?
       @quiet
     end

data/lib/cogworker/periodic/ticker.rb

@@ -69,11 +69,26 @@ module Cogworker
           slot = cron_for(entry).previous_time(now).to_i
           next if @last_checked_slot[entry.pjid] == slot
 
-          enqueue(entry, slot) if claim?(entry, slot)
+          enqueue(entry, slot) if !disabled?(entry) && claim?(entry, slot)
           @last_checked_slot[entry.pjid] = slot
         end
       end
 
+      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
+      # step entirely, short-circuiting before `claim?` even runs, so
+      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
+      # while disabled: the Web UI's own "LastRun" column keeps showing the
+      # last time it *actually* ran, not a due-but-skipped slot, and
+      # re-enabling doesn't trigger a catch-up burst for every slot that
+      # was silently skipped in between. `@last_checked_slot` (this one
+      # Ticker instance's own in-memory dedup, unrelated to the Redis-
+      # persisted last_slot) still advances either way, exactly as it
+      # already did before this entry ever had a disabled state — it only
+      # stops this same tick loop from re-evaluating the same slot twice.
+      def disabled?(entry)
+        Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
+      end
+
       def cron_for(entry)
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end

data/lib/cogworker/process.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 module Cogworker
-  # One live worker process, looked up by heartbeat key. `#quiet!`/`#stop!`
-  # publish to that process's signal channel; the process (whether that's
-  # this same process — e.g. WorkerKiller finding itself via ProcessSet — or
-  # a genuinely remote one) is subscribed to it and reacts identically to a
-  # real `kill -TSTP`/`TERM`.
+  # One live worker process, looked up by heartbeat key. `#quiet!`/`#resume!`/
+  # `#stop!` publish to that process's signal channel; the process (whether
+  # that's this same process — e.g. WorkerKiller finding itself via
+  # ProcessSet — or a genuinely remote one) is subscribed to it and reacts.
+  # `#quiet!`/`#stop!` mirror a real `kill -TSTP`/`TERM`; `#resume!` has no
+  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet in real
+  # Sidekiq-alike tooling) — it's Cogworker-specific, made possible by
+  # `Manager#quiet` being a plain in-memory flag rather than a one-way state
+  # transition like `stopping?`.
   #
   # Named `Cogworker::Process`, not `::Process` — inside this namespace a
   # bare `Process` resolves to this class, not the Kernel module, so any
@@ -23,6 +27,10 @@ module Cogworker
       publish('quiet')
     end
 
+    def resume!
+      publish('resume')
+    end
+
     def stop!
       publish('stop')
     end

data/lib/cogworker/processor.rb

@@ -77,9 +77,11 @@ module Cogworker
           worker.perform(*job['args'])
         end
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
+        Throughput.record('processed')
         Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
       rescue Exception => e # rubocop:disable Lint/RescueException
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
+        Throughput.record('failed')
         route_failure(job, e)
         Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
       end
@@ -106,8 +108,10 @@ module Cogworker
       if new_count <= max_retries
         delay = retry_delay(new_count)
         Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
       else
         Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'dead')
       end
     end
 

data/lib/cogworker/queue.rb

@@ -43,6 +43,23 @@ module Cogworker
       Cogworker.config.redis { |c| c.del(@key) }
     end
 
+    # Paused queues are skipped by every processor's own weighted fetch
+    # (`BasicFetch#retrieve_work`, re-read fresh every fetch cycle, not
+    # cached at process start) — jobs already sitting in this queue's list
+    # are untouched and keep arriving via `Client.push`, they just stop
+    # being picked up until `resume!`.
+    def pause!
+      Cogworker.config.redis { |c| c.sadd(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
+    def resume!
+      Cogworker.config.redis { |c| c.srem(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
+    def paused?
+      Cogworker.config.redis { |c| c.sismember(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
     def each
       page = 0
       per = 50

data/lib/cogworker/redis_keys.rb

@@ -14,13 +14,16 @@ module Cogworker
     RETRY = 'cogworker:retry'
     DEAD = 'cogworker:dead'
     PROCESSES = 'cogworker:processes'
+    PAUSED_QUEUES = 'cogworker:paused_queues'
     STATS_PROCESSED = 'cogworker:stats:processed'
     STATS_FAILED = 'cogworker:stats:failed'
     PERIODIC_SCHEDULE = 'periodic:schedule'
+    PERIODIC_DISABLED = 'periodic:disabled'
 
     module_function
 
     def queue(name) = "#{QUEUE_PREFIX}#{name}"
+    def job_attempts(jid) = "cogworker:job_attempts:#{jid}"
     def process(identity) = "cogworker:process:#{identity}"
     def workers(identity) = "cogworker:workers:#{identity}"
     def signal(identity) = "cogworker:signal:#{identity}"
@@ -28,5 +31,6 @@ module Cogworker
     def periodic_last_slot(pjid) = "periodic:last_slot:#{pjid}"
     def periodic_lock(pjid, slot) = "periodic:lock:#{pjid}:#{slot}"
     def unique_lock(digest) = "cogworker:unique:#{digest}"
+    def throughput_bucket(hour) = "cogworker:throughput:#{hour}"
   end
 end

data/lib/cogworker/throughput.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Per-hour processed/failed counters, feeding Overview's own "Throughput"
+  # chart (the "Relay" concept mock's own sparkline — dropped in the
+  # initial nocturne migration for lack of real data; this is that data).
+  # Distinct from `Cogworker::Stats`' plain running totals (`cogworker:
+  # stats:processed`/`:failed`, incremented forever, no time dimension) and
+  # from `Cogworker::History` (a capped log of individual runs, one entry
+  # per job) — this is neither a total nor a per-job record, just how many
+  # finished in each one-hour bucket, bounded to the last `WINDOW_HOURS` by
+  # each bucket's own TTL rather than any trim/cleanup pass.
+  #
+  # Hour buckets, not minute ones: the mock's own "24h" sparkline is 24
+  # sample points (one per hour), not 1440 — matching that resolution
+  # keeps `.series` a 24-key read (cheap enough to poll every few seconds)
+  # instead of 1440.
+  module Throughput
+    WINDOW_HOURS = 24
+    BUCKET_TTL = (WINDOW_HOURS + 1) * 3600 # a bit of slack past the window itself
+
+    module_function
+
+    def record(outcome, at: Time.now)
+      key = RedisKeys.throughput_bucket(bucket_for(at))
+      Cogworker.config.redis do |c|
+        c.hincrby(key, outcome, 1)
+        c.expire(key, BUCKET_TTL)
+      end
+    end
+
+    # One entry per hour, oldest first, covering exactly the last `hours`
+    # hours ending now — including hours with no activity at all (a real
+    # "0 processed" reads as an honest gap, not a jump the chart papers
+    # over by skipping it).
+    def series(hours: WINDOW_HOURS, now: Time.now)
+      end_bucket = bucket_for(now)
+      buckets = ((end_bucket - hours + 1)..end_bucket).to_a
+      raw = Cogworker.config.redis do |c|
+        c.pipelined { |pipe| buckets.each { |b| pipe.hmget(RedisKeys.throughput_bucket(b), 'processed', 'failed') } }
+      end
+      buckets.zip(raw).map do |bucket, (processed, failed)|
+        { 'time' => Time.at(bucket * 3600).utc, 'processed' => processed.to_i, 'failed' => failed.to_i }
+      end
+    end
+
+    def bucket_for(time)
+      time.to_i / 3600
+    end
+  end
+end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end