cogworker 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0b88ae2031cced0f33c8de66892dbd952e7aba6ed64ad1fba49719bb335698d
-  data.tar.gz: 6cd9c81f131107a8b04bb170dbedb29898899d248b900bfcf4fcbeebf69463bf
+  metadata.gz: 472527fd7492d45c642f017b2cdc788c0741ebe7226b42b416aee90431b61194
+  data.tar.gz: '078375dda087dfe1064ed79e35b92b4fb2e4dbf017e68fb681b939d96a45fdea'
 SHA512:
-  metadata.gz: 867b3c3cc51e6ee56f2fa8e7433b52598b366873ba9609ac9cfa81904bb1e9f0fdb0c35b6816bee0a7605c6536c0a88749e77f972e8d611c7fd3df726772b5fd
-  data.tar.gz: 1bf378d2c4e52a6034d47c2239a21539ae722e3877c015585ef446cfff6c0456c893c071e088b9c857759ac04849a7a481f804b816dafea85dc5e0fce46fe59a
+  metadata.gz: 4e66fb20f141b58faa5c13eaa1e9df971d890a8bc5a77c845f80fbdcc621beacf2b03c5d9e1dbae68a9b4981a2e322a4147d1e366677e607cb069b846d2d905e
+  data.tar.gz: f3297307cd6201b8421a0acd54217d5a79cdaa1f93fed083ccbd7bd492cb9867340ffc52b787847058a0ce030ed5b17bd1b9d389fbdbc6ef0c64195f83f3422a

data/lib/cogworker/heartbeat.rb

@@ -75,7 +75,8 @@ module Cogworker
 
     def cleanup_presence!
       Cogworker.config.redis do |c|
-        c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
+        c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity),
+              RedisKeys.memory_history(Cogworker.identity))
         c.srem?(RedisKeys::PROCESSES, Cogworker.identity)
         # Clean shutdown: Manager#stop! already requeued this process's
         # in-progress list, so there's nothing left for recovery to find.
@@ -132,6 +133,7 @@ module Cogworker
                'quiet', @manager.quiet?.to_s)
         c.expire(RedisKeys.process(identity), TTL)
         write_registries(c, identity, reliable)
+        secondary('memory history') { MemoryHistory.record(c, identity, info['rss_kb']) }
         touch_running_periodic_locks(c, identity)
         # Not via #secondary: Redis failing here, after the presence key went
         # through, doesn't make this beat a failed one.

data/lib/cogworker/key_guard.rb

@@ -65,6 +65,7 @@ module Cogworker
       readable_members(RedisKeys::PROCESSES).each do |id|
         types[RedisKeys.process(id)] = 'hash'
         types[RedisKeys.workers(id)] = 'hash'
+        types[RedisKeys.memory_history(id)] = 'zset'
       end
       readable_fields(RedisKeys::PERIODIC_SCHEDULE).each do |pjid|
         types[RedisKeys.periodic_running(pjid)] = 'string'

data/lib/cogworker/memory_history.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Cogworker
+  # Per-process RSS for the last 24 hours, in SLOTS fixed SAMPLE_SECONDS
+  # slots, feeding the memory sparkline on the Workers tab's process cards.
+  # Written by Heartbeat on every beat; a slot keeps the *highest* reading
+  # any beat in it took (a peak is what matters for memory, and a short
+  # spike between two slot boundaries would otherwise vanish). One ZSET per
+  # identity, scored by slot start, members "<slot_start>:<rss_kb>" — so a
+  # slot's sample is unique by score, not by value. Old slots are trimmed
+  # on every write, and the key itself expires a little after the window,
+  # so a process that's gone leaves nothing behind for long.
+  module MemoryHistory
+    SAMPLE_SECONDS = 900
+    SLOTS = 96 # 24h of 15-minute slots
+    WINDOW = SLOTS * SAMPLE_SECONDS
+    KEY_TTL = WINDOW + 3600
+
+    # KEYS[1] = the identity's ZSET; ARGV = slot start, rss_kb, window, TTL.
+    RECORD_SCRIPT = <<~LUA
+      local slot = tonumber(ARGV[1])
+      local kb = tonumber(ARGV[2])
+      for _, member in ipairs(redis.call("ZRANGEBYSCORE", KEYS[1], slot, slot)) do
+        local seen = tonumber(string.match(member, ":(%d+)$"))
+        if seen and seen >= kb then kb = nil break end
+      end
+      if kb then
+        redis.call("ZREMRANGEBYSCORE", KEYS[1], slot, slot)
+        redis.call("ZADD", KEYS[1], slot, slot .. ":" .. kb)
+      end
+      redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", "(" .. (slot - tonumber(ARGV[3])))
+      redis.call("EXPIRE", KEYS[1], ARGV[4])
+      return 1
+    LUA
+
+    module_function
+
+    def record(conn, identity, rss_kb, at: Time.now.to_i)
+      return unless rss_kb
+
+      LuaScript.run(conn, RECORD_SCRIPT, keys: [RedisKeys.memory_history(identity)],
+                                         argv: [slot_for(at), rss_kb.to_i, WINDOW, KEY_TTL])
+    end
+
+    # Per identity, exactly SLOTS `[slot_start, rss_kb_or_nil]` pairs,
+    # oldest first, the last one being the current (still filling) slot —
+    # a slot no beat wrote to is `nil`, so the chart always spans the full
+    # 24h whatever data exists. One pipelined read for every identity.
+    def series(identities, now: Time.now.to_i)
+      return {} if identities.empty?
+
+      first = slot_for(now) - ((SLOTS - 1) * SAMPLE_SECONDS)
+      raw = Cogworker.config.redis do |c|
+        c.pipelined do |pipe|
+          identities.each { |id| pipe.zrangebyscore(RedisKeys.memory_history(id), first, '+inf') }
+        end
+      end
+      identities.zip(raw).to_h do |id, members|
+        values = Array(members).to_h { |m| m.split(':', 2).map(&:to_i) }
+        [id, Array.new(SLOTS) { |i| [first + (i * SAMPLE_SECONDS), values[first + (i * SAMPLE_SECONDS)]] }]
+      end
+    end
+
+    def slot_for(time)
+      time.to_i - (time.to_i % SAMPLE_SECONDS)
+    end
+  end
+end

data/lib/cogworker/redis_keys.rb

@@ -47,6 +47,7 @@ module Cogworker
     def job_attempts(jid) = "cogworker:job_attempts:#{jid}"
     def process(identity) = "cogworker:process:#{identity}"
     def workers(identity) = "cogworker:workers:#{identity}"
+    def memory_history(identity) = "cogworker:memory:#{identity}"
     def in_progress(identity) = "#{IN_PROGRESS_PREFIX}#{identity}"
     def signal(identity) = "cogworker:signal:#{identity}"
     def periodic_running(pjid) = "periodic:running:#{pjid}"

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/lib/cogworker/web/routes/overview.rb

@@ -53,6 +53,13 @@ module Cogworker
           'connected_clients' => 'Connections', 'used_memory_human' => 'Memory Usage',
           'used_memory_peak_human' => 'Peak Memory Usage'
         }.freeze
+        # Runtime dependency gem name => card label, for the "Cogworker"
+        # section — read from `Gem.loaded_specs`, so it shows what this
+        # process actually loaded, not the gemspec's version constraint.
+        GEM_INFO_DEPENDENCIES = {
+          'redis' => 'redis gem', 'connection_pool' => 'connection_pool', 'rack' => 'Rack',
+          'zeitwerk' => 'Zeitwerk', 'fugit' => 'Fugit'
+        }.freeze
         # Period switcher options for the "Runs per day" chart —
         # `params['period']` (a plain query string, read fresh on every
         # request) selects one of these by key. Ordered as displayed,
@@ -95,6 +102,9 @@ module Cogworker
                 # part of CONTENT_ID's own poll_div.
                 page_body += Layout.poll_div(REDIS_CONTENT_ID, request.script_name, 'overview/redis',
                                              Overview.redis_section)
+                # Static (versions don't change while the process runs), so
+                # it needs no poll of its own.
+                page_body += Overview.gem_info_section
               end
               extra_head = layout == 'a' ? Overview.chart_head(request.script_name) : ''
               Layout.wrap('Overview', page_body, script_name: request.script_name, extra_head: extra_head)
@@ -261,17 +271,32 @@ module Cogworker
         # it pinned below the charts rather than above them.
         def redis_section
           info = Cogworker::Stats.new.redis_info
-          cards = REDIS_INFO_FIELDS.map do |field, label|
+          info_section('Redis', REDIS_INFO_FIELDS.to_h { |field, label| [label, info[field]] })
+        end
+
+        # The basics about the gem itself and what it's running on — the
+        # last block on the page, below the Redis grid.
+        def gem_info_section
+          spec = Gem.loaded_specs['cogworker']
+          values = { 'Version' => Cogworker::VERSION, 'Ruby' => "#{RUBY_VERSION} (#{RUBY_PLATFORM})" }
+          GEM_INFO_DEPENDENCIES.each { |name, label| values[label] = Gem.loaded_specs[name]&.version&.to_s }
+          values['License'] = spec&.license
+          info_section('Cogworker', values)
+        end
+
+        # A titled grid of label/value cards; a nil value renders as "n/a".
+        def info_section(title, values)
+          cards = values.map do |label, value|
             <<~HTML
               <div class="card elev-sm" style="gap: 4px;">
                 <span class="card-kicker">#{Layout.h(label)}</span>
-                <span style="font-size: 22px; font-family: var(--font-heading);">#{Layout.h(info[field] || 'n/a')}</span>
+                <span style="font-size: 15px; font-family: var(--font-heading);">#{Layout.h(value || 'n/a')}</span>
               </div>
             HTML
           end.join
           <<~HTML
             <section style="background: var(--color-surface); border-radius: var(--radius-md); box-shadow: var(--shadow-sm); padding: 16px 18px; display: flex; flex-direction: column; gap: 12px;">
-              <h4 style="margin: 0; font-family: var(--font-heading); font-weight: var(--font-heading-weight); font-size: 17px;">Redis</h4>
+              <h4 style="margin: 0; font-family: var(--font-heading); font-weight: var(--font-heading-weight); font-size: 17px;">#{Layout.h(title)}</h4>
               <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 12px;">#{cards}</div>
             </section>
           HTML

data/lib/cogworker/web/routes/workers.rb

@@ -13,6 +13,7 @@ module Cogworker
       # honest place to look at what's actually running right now.
       module Workers
         CONTENT_ID = 'workers-content'
+        SPARKLINE_HEIGHT_PX = 32
 
         module_function
 
@@ -22,7 +23,8 @@ module Cogworker
             if hx_request?
               content
             else
-              Layout.wrap('Workers', Layout.poll_div(CONTENT_ID, request.script_name, 'workers', content),
+              Layout.wrap('Workers', Layout.poll_div(CONTENT_ID, request.script_name, 'workers', content) +
+                                     Workers.sparkline_script,
                           script_name: request.script_name)
             end
           end
@@ -90,7 +92,9 @@ module Cogworker
           work_set = Cogworker::WorkSet.new.to_a
           busy_counts = work_set.each_with_object(Hash.new(0)) { |(identity, *), h| h[identity] += 1 }
 
-          cards = process_cards(processes, busy_counts, quiet_path, resume_path, stop_path)
+          memory = Cogworker::MemoryHistory.series(processes.map(&:identity))
+
+          cards = process_cards(processes, busy_counts, memory, quiet_path, resume_path, stop_path)
 
           work_rows = work_set.map do |identity, tid, work|
             [Layout.h(identity), Layout.h(tid), Layout.h(work.job['jid']), Layout.h(work.queue),
@@ -134,19 +138,19 @@ module Cogworker
           HTML
         end
 
-        def process_cards(processes, busy_counts, quiet_path, resume_path, stop_path)
+        def process_cards(processes, busy_counts, memory, quiet_path, resume_path, stop_path)
           if processes.empty?
             return %(<p class="text-muted" style="font-size: 13px; font-style: italic;">No worker processes ) +
                    'are reporting in right now.</p>'
           end
 
           items = processes.map do |p|
-            process_card(p, busy_counts[p.identity], quiet_path, resume_path, stop_path)
+            process_card(p, busy_counts[p.identity], memory[p.identity], quiet_path, resume_path, stop_path)
           end.join
           %(<div style="display: grid; grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); gap: 12px;">#{items}</div>)
         end
 
-        def process_card(process, busy, quiet_path, resume_path, stop_path)
+        def process_card(process, busy, memory_samples, quiet_path, resume_path, stop_path)
           concurrency = process['concurrency'].to_i
           pct = concurrency.positive? ? ((busy.to_f / concurrency) * 100).round : 0
           quiet = [true, 'true'].include?(process['quiet'])
@@ -177,6 +181,7 @@ module Cogworker
                 <span>#{memory_cell(process['rss_kb'])}</span>
                 <span>#{beat_label(process.identity)}</span>
               </div>
+              #{memory_sparkline(process.identity, memory_samples)}
               <div style="height: 4px; border-radius: 2px; background: var(--color-neutral-800); overflow: hidden;">
                 <div style="height: 100%; border-radius: 2px; background: var(--color-accent); width: #{pct}%;"></div>
               </div>
@@ -199,6 +204,152 @@ module Cogworker
           end
         end
 
+        # The last 24h of this process's RSS (MemoryHistory), always the
+        # full 24h wide — one MemoryHistory slot per x position, broken
+        # wherever a slot has no sample (before the process started, or
+        # while Redis was away). An inline SVG rather than a Chart.js chart:
+        # these cards are re-rendered by every htmx poll, and a Chart.js
+        # instance per swap leaks (see CLAUDE.md's Overview charts).
+        # Stretched to the card's width with `preserveAspectRatio="none"`,
+        # so it draws only lines and fills (`non-scaling-stroke` keeps the
+        # line's width) — no point markers, which that stretch would
+        # distort; the hover marker and tooltip are HTML, positioned by
+        # `sparkline_script` from the data attributes.
+        def memory_sparkline(identity, slots)
+          slots ||= Cogworker::MemoryHistory.series([identity])[identity]
+          values = slots.map(&:last)
+          low, high = sparkline_range(values.compact)
+          <<~HTML
+            <div class="cw-spark" data-cw-spark-id="#{Layout.h(identity)}" data-cw-spark="#{Layout.h(JSON.generate(values))}"
+                 data-cw-spark-start="#{slots.first.first}" data-cw-spark-step="#{Cogworker::MemoryHistory::SAMPLE_SECONDS}"
+                 data-cw-spark-min="#{low}" data-cw-spark-max="#{high}"
+                 style="position: relative; height: #{SPARKLINE_HEIGHT_PX}px; cursor: crosshair; background: var(--color-bg); border-radius: 3px;">
+              <span class="cw-spark-label" style="position: absolute; top: 3px; left: 6px; font-size: 10px; line-height: 1; color: var(--color-neutral-500); pointer-events: none;">Memory · 24h</span>
+              <svg viewBox="0 0 100 100" preserveAspectRatio="none" aria-label="Memory, last 24 hours"
+                   style="display: block; width: 100%; height: 100%; overflow: visible;">
+                #{sparkline_segments(values, low, high)}
+              </svg>
+            </div>
+          HTML
+        end
+
+        # One line + fill per run of consecutive slots that have a sample,
+        # each point at its slot's center; a lone sample gets a short
+        # horizontal stroke across its own slot, so it's still visible.
+        def sparkline_segments(values, low, high)
+          width = 100.0 / values.size
+          runs = values.each_with_index.chunk_while { |(a, _), (b, _)| a.nil? == b.nil? }.reject { |run| run[0][0].nil? }
+          runs.map do |run|
+            points = run.map { |kb, i| [(i + 0.5) * width, (1 - ((kb - low) / (high - low))) * 100] }
+            sparkline_segment(points, width)
+          end.join
+        end
+
+        def sparkline_segment(points, width)
+          if points.one?
+            x, y = points.first
+            points = [[x - (width / 2), y], [x + (width / 2), y]]
+          end
+          coords = points.map { |x, y| format('%<x>.2f,%<y>.2f', x: x, y: y) }.join(' ')
+          area = format('%<x0>.2f,100 %<coords>s %<x1>.2f,100', x0: points.first[0], coords: coords, x1: points.last[0])
+          <<~HTML
+            <polygon points="#{area}" style="fill: var(--color-accent); opacity: 0.15;"></polygon>
+            <polyline points="#{coords}" vector-effect="non-scaling-stroke" style="fill: none; stroke: var(--color-accent); stroke-width: 1.5;"></polyline>
+          HTML
+        end
+
+        # A little headroom above and below, so the line never hugs the
+        # card's edges; a flat series gets a range around its one value.
+        def sparkline_range(values)
+          return [0.0, 1.0] if values.empty?
+
+          low = values.min.to_f
+          high = values.max.to_f
+          pad = [(high - low) * 0.1, 1024].max
+          [(low - pad).clamp(0, nil), high + pad]
+        end
+
+        # Hover readout for every `.cw-spark` on the page: the sample
+        # nearest the pointer, as a marker line + dot and a tooltip with
+        # its value and (local) time, hiding the "Memory" label meanwhile.
+        # Delegated from `document`, so it keeps working on cards htmx
+        # swaps in; a poll that replaces the card under the pointer
+        # re-shows the readout on the new one.
+        def sparkline_script
+          <<~HTML
+            <script>
+              (function () {
+                var hover = null;
+                function part(el, cls, css) {
+                  var node = el.querySelector('.' + cls);
+                  if (!node) {
+                    node = document.createElement('div');
+                    node.className = cls;
+                    node.style.cssText = 'position: absolute; pointer-events: none; ' + css;
+                    el.appendChild(node);
+                  }
+                  return node;
+                }
+                function label(el, visible) {
+                  var node = el.querySelector('.cw-spark-label');
+                  if (node) node.style.visibility = visible ? '' : 'hidden';
+                }
+                function hide(el) {
+                  el.querySelectorAll('.cw-spark-line, .cw-spark-dot, .cw-spark-tip').forEach(function (n) { n.remove(); });
+                  label(el, true);
+                }
+                function show(el, clientX) {
+                  var values = JSON.parse(el.getAttribute('data-cw-spark'));
+                  var start = parseInt(el.getAttribute('data-cw-spark-start'), 10);
+                  var step = parseInt(el.getAttribute('data-cw-spark-step'), 10);
+                  var lo = parseFloat(el.getAttribute('data-cw-spark-min'));
+                  var hi = parseFloat(el.getAttribute('data-cw-spark-max'));
+                  var rect = el.getBoundingClientRect();
+                  label(el, false);
+                  var frac = Math.min(Math.max((clientX - rect.left) / rect.width, 0), 1);
+                  var i = Math.min(Math.floor(frac * values.length), values.length - 1);
+                  var x = (i + 0.5) / values.length * rect.width;
+                  var kb = values[i];
+                  part(el, 'cw-spark-line', 'top: 0; bottom: 0; width: 1px; background: var(--color-neutral-500);').style.left = x + 'px';
+                  var dot = part(el, 'cw-spark-dot', 'width: 6px; height: 6px; border-radius: 50%; margin: -3px 0 0 -3px; background: var(--color-accent);');
+                  dot.style.display = kb === null ? 'none' : '';
+                  dot.style.left = x + 'px';
+                  if (kb !== null) dot.style.top = (1 - (kb - lo) / (hi - lo)) * rect.height + 'px';
+                  var tip = part(el, 'cw-spark-tip', 'bottom: calc(100% + 4px); z-index: 10; white-space: nowrap; font-size: 11px; ' +
+                    'padding: 3px 6px; border-radius: 4px; background: var(--color-surface); color: var(--color-text); ' +
+                    'box-shadow: var(--shadow-md); border: 1px solid var(--color-neutral-800);');
+                  var at = new Date((start + i * step) * 1000);
+                  var when = window.cogworkerFormatTime ? window.cogworkerFormatTime(at) : at.toLocaleString();
+                  tip.textContent = (kb === null ? 'no data' : (kb / 1024).toFixed(1) + 'M') + ' · ' + when;
+                  tip.style.left = x + 'px';
+                  tip.style.transform = 'translateX(-' + (frac * 100) + '%)';
+                }
+                function find(id) {
+                  var all = document.querySelectorAll('.cw-spark');
+                  for (var i = 0; i < all.length; i++) if (all[i].getAttribute('data-cw-spark-id') === id) return all[i];
+                  return null;
+                }
+                document.addEventListener('mousemove', function (evt) {
+                  var el = evt.target.closest && evt.target.closest('.cw-spark');
+                  if (hover && (!el || el.getAttribute('data-cw-spark-id') !== hover.id)) {
+                    var old = find(hover.id);
+                    if (old) hide(old);
+                    hover = null;
+                  }
+                  if (!el) return;
+                  hover = { id: el.getAttribute('data-cw-spark-id'), x: evt.clientX };
+                  show(el, evt.clientX);
+                });
+                document.addEventListener('htmx:afterSwap', function () {
+                  if (!hover) return;
+                  var el = find(hover.id);
+                  if (el) show(el, hover.x);
+                });
+              })();
+            </script>
+          HTML
+        end
+
         # `rss_kb` is absent on any process whose heartbeat predates this
         # field (or that never measured cleanly — see
         # `Heartbeat#current_rss_kb`) — render "n/a" rather than "0M".

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cogworker
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - vdonec
@@ -233,6 +233,7 @@ files:
 - lib/cogworker/logging.rb
 - lib/cogworker/lua_script.rb
 - lib/cogworker/manager.rb
+- lib/cogworker/memory_history.rb
 - lib/cogworker/middleware/chain.rb
 - lib/cogworker/owned_key.rb
 - lib/cogworker/periodic/claim.lua