cogworker 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5aad05a1deaf85505c91f0b6ebe11199fc918b6b3e3831efa5ddb753381cf824
-  data.tar.gz: 760b88ec2cc3075f4bbfe152062bc8f1e24b71ce2c71984f9702c0cbdb7da81f
+  metadata.gz: ef260933f908739609343cb6b72e5fbbe75a01d0c93dd8fba5d1c4983e807f26
+  data.tar.gz: 22b576f3c302378c1ca2e8075893c3c4b9404329f6dacc22c281ad4b7c31f688
 SHA512:
-  metadata.gz: fc050c6eff33d2fcc5f61629d46e88452912d3f798c49b49fdc53d21ca525403a575e89e13c9e34307cba312eab7307baebc80a187661a032592fb31eb797854
-  data.tar.gz: a9ef2e674633abe4085b6e6e21eb74c3ec735b3ab7c4f6305f274172049e41058477b03e5a5e8105cc6b0afdeadcafc9babba07bbdd81855b54532585a9a946e
+  metadata.gz: a48c5530445af6a80846ef5313202f88a3638ecdb33614e7f86baba87c33c4e214add29febf257e025cdf1921ad76bc35140cd91828101aa556a4fb0a461e662
+  data.tar.gz: f6b2cdaef9cf94adb3823d29cffca99d50697a7553dc6178357fb6cf435ba8fb82320be291e1b04bd5075ab05ad2cf86cb5b2ed67a4f6420e20283360ec399f0

data/lib/cogworker/attempts.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Cogworker
+  # Per-job append-only log of failed attempts (attempt number, when, error
+  # class/message, and whether that failure routed the job to retry or to
+  # dead) — what `Processor#route_failure` appends to on every failure, and
+  # what the Web UI's Jobs tab reads to render a retry timeline in a job's
+  # detail panel.
+  #
+  # Distinct from `Cogworker::History`: History is every *completed* run
+  # (success or failure) across every job, capped and trimmed independently
+  # of any one job's fate. This is the *in-progress* failure trail for one
+  # still-retrying-or-dead job, keyed by JID, growing one entry per attempt
+  # until that job is deleted from Retries/Dead (`.clear`) or the key
+  # expires on its own (`TTL_SECONDS`) if nothing ever does.
+  module Attempts
+    MAX_ENTRIES = 25
+    TTL_SECONDS = 30 * 24 * 60 * 60 # 30 days — bounds an abandoned dead entry's log even if nothing ever calls `.clear`
+
+    module_function
+
+    def record(jid, attempt:, error:, outcome:)
+      entry = JSON.generate(
+        'attempt' => attempt, 'failed_at' => Time.now.to_f, 'outcome' => outcome,
+        'error_class' => error.class.name, 'error_message' => error.message.to_s[0, 10_000]
+      )
+      key = RedisKeys.job_attempts(jid)
+      Cogworker.config.redis do |c|
+        c.rpush(key, entry)
+        c.ltrim(key, -MAX_ENTRIES, -1)
+        c.expire(key, TTL_SECONDS)
+      end
+    end
+
+    # Oldest attempt first — the order a timeline reads top-to-bottom.
+    def for(jid)
+      raw = Cogworker.config.redis { |c| c.lrange(RedisKeys.job_attempts(jid), 0, -1) }
+      raw.map { |r| JSON.parse(r) }
+    end
+
+    def clear(jid)
+      Cogworker.config.redis { |c| c.del(RedisKeys.job_attempts(jid)) }
+    end
+  end
+end

data/lib/cogworker/basic_fetch.rb

@@ -13,15 +13,38 @@ module Cogworker
       @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
     end
 
+    # Re-reads `cogworker:paused_queues` on every single fetch (not once at
+    # `initialize`, and not cached) — a `Queue#pause!`/`#resume!` from the
+    # Web UI takes effect on this processor's very next cycle, not just for
+    # ones started after the toggle. If every one of this processor's
+    # queues is currently paused, there's nothing left to `BRPOP` at all
+    # (an empty key list is invalid) — sleep out one `TIMEOUT` instead of
+    # returning immediately, same pause `BRPOP` itself would have caused,
+    # so `Processor#run`'s loop doesn't spin hot polling Redis in a tight
+    # loop with no rate limit.
     def retrieve_work
-      keys = @queue_keys.shuffle
-      result = Cogworker.config.redis { |c| c.brpop(*keys, timeout: TIMEOUT) }
+      keys = active_keys
+      if keys.empty?
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      result = Cogworker.config.redis { |c| c.brpop(*keys.shuffle, timeout: TIMEOUT) }
       return nil unless result
 
       queue_key, raw_job = result
       UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
     end
 
+    private
+
+    def active_keys
+      paused = Cogworker.config.redis { |c| c.smembers(RedisKeys::PAUSED_QUEUES) }
+      return @queue_keys if paused.empty?
+
+      @queue_keys.reject { |k| paused.include?(k.delete_prefix(RedisKeys::QUEUE_PREFIX)) }
+    end
+
     UnitOfWork = Struct.new(:queue, :raw_job)
   end
 end

data/lib/cogworker/heartbeat.rb

@@ -5,8 +5,9 @@ require 'redis'
 
 module Cogworker
   # Publishes this process's presence (for ProcessSet) and listens for
-  # remote quiet!/stop! requests (for Process#quiet!/#stop! issued by another
-  # process, e.g. a self-targeting WorkerKiller or the Web UI). Both the
+  # remote quiet!/resume!/stop! requests (for Process#quiet!/#resume!/#stop!
+  # issued by another process, e.g. a self-targeting WorkerKiller or the Web
+  # UI). Both the
   # heartbeat loop and the pub/sub subscriber must only start *after* a
   # cogworkerswarm fork, never before, in the parent — otherwise the thread
   # simply doesn't exist in the child, and a lock that thread held could
@@ -119,6 +120,7 @@ module Cogworker
     def dispatch(message)
       case message
       when 'quiet' then @manager.quiet!
+      when 'resume' then @manager.unquiet!
       when 'stop' then remote_stop!
       else Cogworker.logger.warn { "Unknown signal message: #{message}" }
       end

data/lib/cogworker/history/storage.rb

@@ -6,8 +6,14 @@ module Cogworker
   module History
     # Three parallel ZSETs (score = finished_at), so any of the three
     # filters (all/success/failed) is a plain `ZREVRANGE` — no per-request
-    # `ZUNIONSTORE`/scan needed. Each is trimmed to `History.max_entries`
-    # independently on every write, oldest first.
+    # `ZUNIONSTORE`/scan needed. Each is trimmed independently on every
+    # write, oldest first, by age (`History.retention_days`, the primary,
+    # expected limit) and then by count (`History.max_entries`, a safety
+    # ceiling only — see both constants' own comments in `history.rb`) —
+    # see `write_and_trim`. Alongside those, `record` also maintains one
+    # small per-UTC-day Hash counter (`daily_counts` reads these back) —
+    # kept deliberately separate from, and unaffected by, either trim above
+    # (see `daily_counts`' own comment).
     module Storage
       LIST_KEYS = {
         'all' => 'cogworker:history:all',
@@ -29,13 +35,21 @@ module Cogworker
         end
 
         raw = JSON.generate(entry)
-        max = Cogworker::History.max_entries
         Cogworker.config.redis do |c|
-          write_and_trim(c, LIST_KEYS.fetch('all'), raw, finished_at, max)
-          write_and_trim(c, LIST_KEYS.fetch(status), raw, finished_at, max)
+          write_and_trim(c, LIST_KEYS.fetch('all'), raw, finished_at)
+          write_and_trim(c, LIST_KEYS.fetch(status), raw, finished_at)
+          # Independent of the two trims above — see `daily_counts`' own
+          # comment for why "Runs per day" can't just read the `all` list.
+          bump_daily_count(c, finished_at, status)
         end
       end
 
+      def bump_daily_count(conn, finished_at, status)
+        daily_key = RedisKeys.history_daily_bucket(Time.at(finished_at).utc.strftime('%Y-%m-%d'))
+        conn.hincrby(daily_key, status, 1)
+        conn.expire(daily_key, Cogworker::History.daily_stats_retention_days * 86_400)
+      end
+
       # Newest-first page of `count`/`status` entries: `[entries, total]`.
       def page(status, page_number, per_page)
         key = LIST_KEYS.fetch(status, LIST_KEYS.fetch('all'))
@@ -49,33 +63,55 @@ module Cogworker
         end
       end
 
-      def write_and_trim(conn, key, raw, score, max)
+      def write_and_trim(conn, key, raw, score)
         conn.zadd(key, score, raw)
-        conn.zremrangebyrank(key, 0, -(max + 1))
+        # Age first — the primary, expected trim in normal operation: gone
+        # once older than `History.retention_days`, regardless of how few
+        # entries that leaves. Count second, as a safety ceiling only — see
+        # `History::DEFAULT_MAX_ENTRIES`'s own comment for why both run on
+        # every write rather than just one. Both read fresh from
+        # `Cogworker::History` here (rather than being passed in) so a
+        # config change takes effect on the very next write, same as
+        # `max_entries` always has.
+        cutoff = Time.now.to_f - (Cogworker::History.retention_days * 86_400)
+        conn.zremrangebyscore(key, '-inf', cutoff)
+        conn.zremrangebyrank(key, 0, -(Cogworker::History.max_entries + 1))
       end
 
       # Success/failed counts per UTC calendar day, for the last `days` days
       # (today included) — `{ 'YYYY-MM-DD' => { 'success' => n, 'failed' => n } }`,
-      # a day with no entries simply absent from the Hash. Reads the shared
-      # "all" list once (rather than "success" and "failed" separately) since
-      # every entry already carries its own `status`. Bucketing uses UTC,
-      # not the viewer's browser timezone (unlike `Layout.time_tag`
+      # a day with no entries simply absent from the Hash. Bucketing uses
+      # UTC, not the viewer's browser timezone (unlike `Layout.time_tag`
       # elsewhere) — this is a server-rendered daily aggregate, not a single
       # instant, so there's no one "browser day" to convert into.
-      # Same caveat as any other read of this list: only entries still
-      # within `History.max_entries` are counted, so a very busy queue's
-      # oldest requested days may already have been trimmed away.
+      #
+      # Reads `RedisKeys.history_daily_bucket(day)` (one small Hash per day,
+      # `record` above keeps them updated) rather than scanning the `all`
+      # list and bucketing entries by hand — deliberately: even now that the
+      # `all`/`success`/`failed` lists are trimmed by age (`History.
+      # retention_days`) rather than purely by count, that's still a
+      # *different*, independently configured window than the daily
+      # buckets' own (`History.daily_stats_retention_days`, TTL-based, same
+      # idea as `Cogworker::Throughput`'s hourly buckets) — a "6 months"
+      # chart request would find nothing if it depended on `all` still
+      # holding 6 months of entries, since `retention_days` defaults to
+      # far less than that. The daily buckets are sized (and meant) to
+      # outlive the raw per-entry lists, so "Runs per day" stays accurate
+      # regardless of how short `retention_days` is configured.
       def daily_counts(days)
-        since = Time.now.to_f - (days * 86_400)
-        Cogworker.config.redis do |c|
-          c.zrangebyscore(LIST_KEYS.fetch('all'), since, '+inf').each_with_object({}) do |raw, counts|
-            entry = JSON.parse(raw)
-            day = Time.at(entry['finished_at']).utc.strftime('%Y-%m-%d')
-            bucket = (counts[day] ||= { 'success' => 0, 'failed' => 0 })
-            bucket[entry['status']] += 1 if bucket.key?(entry['status'])
-          end
+        now = Time.now.utc
+        day_strings = (days - 1).downto(0).map { |offset| (now - (offset * 86_400)).strftime('%Y-%m-%d') }
+        raw = Cogworker.config.redis { |c| c.pipelined { |pipe| day_strings.each { |d| daily_hmget(pipe, d) } } }
+        day_strings.zip(raw).each_with_object({}) do |(day, (success, failed)), counts|
+          next if success.nil? && failed.nil?
+
+          counts[day] = { 'success' => success.to_i, 'failed' => failed.to_i }
         end
       end
+
+      def daily_hmget(pipe, day)
+        pipe.hmget(RedisKeys.history_daily_bucket(day), 'success', 'failed')
+      end
     end
   end
 end

data/lib/cogworker/history.rb

@@ -4,29 +4,79 @@ module Cogworker
   # Execution history: every job run (success and failure alike) is recorded
   # with its full args, timing, and — on failure — error class/message/
   # backtrace. Distinct from the Status layer (`Cogworker::Status`), which
-  # tracks *current* per-jid state with a TTL; History is an append-only,
-  # length-capped log meant for browsing/auditing past runs.
+  # tracks *current* per-jid state with a TTL; History is an append-only log
+  # meant for browsing/auditing past runs, trimmed by age first.
   module History
-    DEFAULT_MAX_ENTRIES = 1000
+    # The primary, expected trim for the "all"/"success"/"failed" lists —
+    # anything older than this many days is gone on the next write,
+    # regardless of how few (or many) entries that leaves. See `Storage.
+    # write_and_trim`.
+    DEFAULT_RETENTION_DAYS = 30
+    # A safety ceiling, not the everyday trim mechanism — `retention_days`
+    # above is. Each entry in the lists it bounds carries its full args and,
+    # on failure, up to 200 backtrace lines, so a burst of traffic under a
+    # purely time-based trim could otherwise grow Redis memory unboundedly
+    # until those entries finally age out; this caps that. Sized well above
+    # what `retention_days`' default window is expected to hold in normal
+    # operation specifically so it stays a rare safety net rather than the
+    # thing actually doing the trimming day to day — if it's the one
+    # regularly cutting entries short of `retention_days`, raise it (or
+    # lower `retention_days`) to match this queue's real volume.
+    DEFAULT_MAX_ENTRIES = 50_000
+    # "Runs per day" (`Storage.daily_counts`) reads a separate, per-UTC-day
+    # counter (`RedisKeys.history_daily_bucket`) rather than the capped
+    # `all`/`success`/`failed` lists above, specifically so those lists'
+    # own trim (age- or count-based) doesn't also blow away the daily
+    # chart's older days — see `Storage`'s own comment. Each day's counter
+    # self-expires via `EXPIRE`, the same TTL-based cleanup `Cogworker::
+    # Throughput` already uses for its hourly buckets, rather than a
+    # count-based trim: 400 days comfortably covers the widest period the
+    # built-in chart offers (6 months / 182 days) with real margin, so a
+    # viewer opening that period on day 1 of a new retention window still
+    # sees its oldest days.
+    DEFAULT_DAILY_STATS_RETENTION_DAYS = 400
 
     class << self
-      # How many entries are retained (oldest trimmed first), independently
-      # for the "all", "success", and "failed" lists — read fresh on every
+      # How many days an entry survives in the "all"/"success"/"failed"
+      # lists before `write_and_trim` removes it — read fresh on every
       # write, so changing it takes effect immediately, no need to rebuild
       # the middleware chain. Configurable via
+      # `configure_server_middleware(config, retention_days: N)`, or
+      # directly: `Cogworker::History.retention_days = 90`.
+      attr_writer :retention_days
+      # The count-based safety ceiling alongside `retention_days` above —
+      # see `DEFAULT_MAX_ENTRIES`. Configurable via
       # `configure_server_middleware(config, max_entries: N)`, or directly:
-      # `Cogworker::History.max_entries = 5000`.
+      # `Cogworker::History.max_entries = 50_000`.
       attr_writer :max_entries
+      # How many days a "Runs per day" daily counter survives before
+      # self-expiring — independent of `retention_days`/`max_entries` above
+      # (see `DEFAULT_DAILY_STATS_RETENTION_DAYS`). Configurable via
+      # `configure_server_middleware(config, daily_stats_retention_days: N)`,
+      # or directly: `Cogworker::History.daily_stats_retention_days = 800`.
+      attr_writer :daily_stats_retention_days
+
+      def retention_days
+        @retention_days ||= DEFAULT_RETENTION_DAYS
+      end
 
       def max_entries
         @max_entries ||= DEFAULT_MAX_ENTRIES
       end
+
+      def daily_stats_retention_days
+        @daily_stats_retention_days ||= DEFAULT_DAILY_STATS_RETENTION_DAYS
+      end
     end
 
     module_function
 
-    def configure_server_middleware(config, max_entries: DEFAULT_MAX_ENTRIES)
+    def configure_server_middleware(config, retention_days: DEFAULT_RETENTION_DAYS,
+                                    max_entries: DEFAULT_MAX_ENTRIES,
+                                    daily_stats_retention_days: DEFAULT_DAILY_STATS_RETENTION_DAYS)
+      self.retention_days = retention_days
       self.max_entries = max_entries
+      self.daily_stats_retention_days = daily_stats_retention_days
       config.server_middleware { |chain| chain.add(Middleware) }
     end
   end

data/lib/cogworker/manager.rb

@@ -25,6 +25,10 @@ module Cogworker
       @quiet = true
     end
 
+    def unquiet!
+      @quiet = false
+    end
+
     def quiet?
       @quiet
     end

data/lib/cogworker/periodic/ticker.rb

@@ -69,11 +69,26 @@ module Cogworker
           slot = cron_for(entry).previous_time(now).to_i
           next if @last_checked_slot[entry.pjid] == slot
 
-          enqueue(entry, slot) if claim?(entry, slot)
+          enqueue(entry, slot) if !disabled?(entry) && claim?(entry, slot)
           @last_checked_slot[entry.pjid] = slot
         end
       end
 
+      # Web UI "Disable" (`Routes::Schedules`) — skips the claim/enqueue
+      # step entirely, short-circuiting before `claim?` even runs, so
+      # neither `periodic:last_slot:<pjid>` nor the per-slot lock advance
+      # while disabled: the Web UI's own "LastRun" column keeps showing the
+      # last time it *actually* ran, not a due-but-skipped slot, and
+      # re-enabling doesn't trigger a catch-up burst for every slot that
+      # was silently skipped in between. `@last_checked_slot` (this one
+      # Ticker instance's own in-memory dedup, unrelated to the Redis-
+      # persisted last_slot) still advances either way, exactly as it
+      # already did before this entry ever had a disabled state — it only
+      # stops this same tick loop from re-evaluating the same slot twice.
+      def disabled?(entry)
+        Cogworker.config.redis { |c| c.sismember(RedisKeys::PERIODIC_DISABLED, entry.pjid) }
+      end
+
       def cron_for(entry)
         @cron_cache[entry.pjid] ||= Fugit::Cron.parse(entry.cron)
       end

data/lib/cogworker/process.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 module Cogworker
-  # One live worker process, looked up by heartbeat key. `#quiet!`/`#stop!`
-  # publish to that process's signal channel; the process (whether that's
-  # this same process — e.g. WorkerKiller finding itself via ProcessSet — or
-  # a genuinely remote one) is subscribed to it and reacts identically to a
-  # real `kill -TSTP`/`TERM`.
+  # One live worker process, looked up by heartbeat key. `#quiet!`/`#resume!`/
+  # `#stop!` publish to that process's signal channel; the process (whether
+  # that's this same process — e.g. WorkerKiller finding itself via
+  # ProcessSet — or a genuinely remote one) is subscribed to it and reacts.
+  # `#quiet!`/`#stop!` mirror a real `kill -TSTP`/`TERM`; `#resume!` has no
+  # OS-signal equivalent (there's no `SIGCONT`-style un-quiet in real
+  # Sidekiq-alike tooling) — it's Cogworker-specific, made possible by
+  # `Manager#quiet` being a plain in-memory flag rather than a one-way state
+  # transition like `stopping?`.
   #
   # Named `Cogworker::Process`, not `::Process` — inside this namespace a
   # bare `Process` resolves to this class, not the Kernel module, so any
@@ -23,6 +27,10 @@ module Cogworker
       publish('quiet')
     end
 
+    def resume!
+      publish('resume')
+    end
+
     def stop!
       publish('stop')
     end

data/lib/cogworker/processor.rb

@@ -77,9 +77,11 @@ module Cogworker
           worker.perform(*job['args'])
         end
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_PROCESSED) }
+        Throughput.record('processed')
         Cogworker.logger.info { "done: #{job['class']} jid=#{job['jid']}" }
       rescue Exception => e # rubocop:disable Lint/RescueException
         Cogworker.config.redis { |c| c.incr(RedisKeys::STATS_FAILED) }
+        Throughput.record('failed')
         route_failure(job, e)
         Cogworker.logger.warn { "fail: #{job['class']} jid=#{job['jid']}: #{e.class}: #{e&.message}" }
       end
@@ -106,8 +108,10 @@ module Cogworker
       if new_count <= max_retries
         delay = retry_delay(new_count)
         Cogworker.config.redis { |c| c.zadd(RedisKeys::RETRY, Time.now.to_f + delay, JSON.generate(job)) }
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'retrying')
       else
         Cogworker.config.redis { |c| c.zadd(RedisKeys::DEAD, Time.now.to_f, JSON.generate(job)) }
+        Attempts.record(job['jid'], attempt: new_count, error: error, outcome: 'dead')
       end
     end
 

data/lib/cogworker/queue.rb

@@ -43,6 +43,23 @@ module Cogworker
       Cogworker.config.redis { |c| c.del(@key) }
     end
 
+    # Paused queues are skipped by every processor's own weighted fetch
+    # (`BasicFetch#retrieve_work`, re-read fresh every fetch cycle, not
+    # cached at process start) — jobs already sitting in this queue's list
+    # are untouched and keep arriving via `Client.push`, they just stop
+    # being picked up until `resume!`.
+    def pause!
+      Cogworker.config.redis { |c| c.sadd(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
+    def resume!
+      Cogworker.config.redis { |c| c.srem(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
+    def paused?
+      Cogworker.config.redis { |c| c.sismember(RedisKeys::PAUSED_QUEUES, name) }
+    end
+
     def each
       page = 0
       per = 50

data/lib/cogworker/redis_keys.rb

@@ -14,13 +14,16 @@ module Cogworker
     RETRY = 'cogworker:retry'
     DEAD = 'cogworker:dead'
     PROCESSES = 'cogworker:processes'
+    PAUSED_QUEUES = 'cogworker:paused_queues'
     STATS_PROCESSED = 'cogworker:stats:processed'
     STATS_FAILED = 'cogworker:stats:failed'
     PERIODIC_SCHEDULE = 'periodic:schedule'
+    PERIODIC_DISABLED = 'periodic:disabled'
 
     module_function
 
     def queue(name) = "#{QUEUE_PREFIX}#{name}"
+    def job_attempts(jid) = "cogworker:job_attempts:#{jid}"
     def process(identity) = "cogworker:process:#{identity}"
     def workers(identity) = "cogworker:workers:#{identity}"
     def signal(identity) = "cogworker:signal:#{identity}"
@@ -28,5 +31,7 @@ module Cogworker
     def periodic_last_slot(pjid) = "periodic:last_slot:#{pjid}"
     def periodic_lock(pjid, slot) = "periodic:lock:#{pjid}:#{slot}"
     def unique_lock(digest) = "cogworker:unique:#{digest}"
+    def throughput_bucket(hour) = "cogworker:throughput:#{hour}"
+    def history_daily_bucket(day) = "cogworker:history:daily:#{day}"
   end
 end

data/lib/cogworker/throughput.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Per-hour processed/failed counters, feeding Overview's own "Throughput"
+  # chart (the "Relay" concept mock's own sparkline — dropped in the
+  # initial nocturne migration for lack of real data; this is that data).
+  # Distinct from `Cogworker::Stats`' plain running totals (`cogworker:
+  # stats:processed`/`:failed`, incremented forever, no time dimension) and
+  # from `Cogworker::History` (a capped log of individual runs, one entry
+  # per job) — this is neither a total nor a per-job record, just how many
+  # finished in each one-hour bucket, bounded to the last `WINDOW_HOURS` by
+  # each bucket's own TTL rather than any trim/cleanup pass.
+  #
+  # Hour buckets, not minute ones: the mock's own "24h" sparkline is 24
+  # sample points (one per hour), not 1440 — matching that resolution
+  # keeps `.series` a 24-key read (cheap enough to poll every few seconds)
+  # instead of 1440.
+  module Throughput
+    WINDOW_HOURS = 24
+    BUCKET_TTL = (WINDOW_HOURS + 1) * 3600 # a bit of slack past the window itself
+
+    module_function
+
+    def record(outcome, at: Time.now)
+      key = RedisKeys.throughput_bucket(bucket_for(at))
+      Cogworker.config.redis do |c|
+        c.hincrby(key, outcome, 1)
+        c.expire(key, BUCKET_TTL)
+      end
+    end
+
+    # One entry per hour, oldest first, covering exactly the last `hours`
+    # hours ending now — including hours with no activity at all (a real
+    # "0 processed" reads as an honest gap, not a jump the chart papers
+    # over by skipping it).
+    def series(hours: WINDOW_HOURS, now: Time.now)
+      end_bucket = bucket_for(now)
+      buckets = ((end_bucket - hours + 1)..end_bucket).to_a
+      raw = Cogworker.config.redis do |c|
+        c.pipelined { |pipe| buckets.each { |b| pipe.hmget(RedisKeys.throughput_bucket(b), 'processed', 'failed') } }
+      end
+      buckets.zip(raw).map do |bucket, (processed, failed)|
+        { 'time' => Time.at(bucket * 3600).utc, 'processed' => processed.to_i, 'failed' => failed.to_i }
+      end
+    end
+
+    def bucket_for(time)
+      time.to_i / 3600
+    end
+  end
+end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.1.0'
+  VERSION = '0.2.1'
 end