agentmon 0.1.0

data/lib/agentmon/metrics/session_ledger.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+# reading[:session_ledger]: every agent session alive now or ended in the last KEEP_ENDED seconds,
+# as Session records (lib/agentmon/model.rb) with current values and lifetime totals. Stateful: it
+# sees every sample the Engine takes.
+#
+# How the totals are counted, and what they can't see:
+#
+# - cpu_seconds adds, for each member process, the growth of its own + reaped-children CPU time
+#   between samples (a member seen for the first time counts all of it, so a session that started
+#   before agentmon still shows its whole life). Child time matters: the kernel adds a process's
+#   CPU time to its parent's when the parent reaps it, so short-lived tools (git, rg) that start
+#   and exit between two samples are still counted, through their parent.
+#   That also means a member that exits is about to be counted again inside its parent's child
+#   time, so its last seen total goes into the session's `pending` pool and is subtracted from the
+#   session's next growth (for PENDING_SAMPLES samples; then it expires, so a parent outside the
+#   tree or a zombie nobody reaps can't swallow real growth). When a CLI session ends under a parent in another session (Claude Code
+#   under the desktop app), its root's total goes into that session's pool the same way.
+#   A member that leaves the tree alive (an orphan reparented to launchd) keeps what it was
+#   counted and stops counting.
+# - bytes_read / bytes_written add each member's growth between samples (first seen: all of it).
+#   Disk I/O isn't passed to parents, so a process that starts and exits between two samples
+#   isn't seen: these are lower bounds.
+# - peak_footprint is the largest footprint sum (readable members) seen in any sample.
+module Agentmon
+  module Metrics
+    class SessionLedger
+      KEEP_ENDED = 15 * 60
+      # Samples an exited member's CPU time waits to reappear as its parent's child time.
+      PENDING_SAMPLES = 5
+
+      # Mutable per-session accumulator (the metric's state); `to_session` makes the record.
+      Account = Struct.new(:info, :first_seen_at, :last_seen_at, :ended_at, :pending, :cpu_seconds, :bytes_read,
+                           :bytes_written, :peak_footprint, :now, :parent_session) do
+        def alive? = ended_at.nil?
+      end
+
+      def initialize(state) = @state = state
+
+      def call(reading)
+        accounts = (@state[:accounts] ||= {})
+        seen = @state[:seen] || {} # identity => [session id, cpu total, disk read, disk written]
+        map = reading[:sessions]
+        # Attribution failed this sample: keep every account as it was rather than end them all.
+        return sessions(accounts) unless map
+
+        rates = reading[:process_rates] || {}
+        processes = reading.sample[:processes] || []
+        live = processes.select(&:readable).to_h { |p| [p.identity, true] }
+        at = reading.at
+
+        end_missing(accounts, map, at)
+        hand_over_exits(accounts, seen, live)
+        members = processes.group_by { |p| map.by_pid[p.pid] }
+        @state[:seen] = next_seen = {}
+        map.sessions.each do |info|
+          account = accounts[info.id] ||= Account.new(info:, first_seen_at: at, pending: [], cpu_seconds: 0.0,
+                                                      bytes_read: 0, bytes_written: 0, peak_footprint: 0)
+          update(account, info, members.fetch(info.id, []), rates, seen, next_seen, at, map)
+        end
+        accounts.delete_if { |_, a| a.ended_at && at - a.ended_at > KEEP_ENDED }
+        sessions(accounts)
+      end
+
+      private
+
+      def sessions(accounts)
+        accounts.values.sort_by { |a| [a.info.started_at || a.first_seen_at, a.info.id] }.map { |a| to_session(a) }
+      end
+
+      # Pending CPU seconds wait PENDING_SAMPLES samples for the parent to report them as child time
+      # (it reaps within milliseconds); if it never does (reaped outside the session, a zombie
+      # nobody reaps), they expire so the session's real growth isn't swallowed.
+      def expect_handover(account, seconds) = account.pending << [seconds, PENDING_SAMPLES]
+
+      # Takes up to `grown` seconds out of the pending pool, oldest first; returns what it took.
+      def absorb(account, grown)
+        taken = 0.0
+        account.pending.each do |entry|
+          take = [entry[0], grown - taken].min
+          entry[0] -= take
+          taken += take
+        end
+        account.pending.each { |entry| entry[1] -= 1 }
+        account.pending.reject! { |amount, left| amount <= 0 || left <= 0 }
+        taken
+      end
+
+      def end_missing(accounts, map, _at)
+        alive = map.sessions.to_h { |s| [s.id, true] }
+        accounts.each_value do |account|
+          next if alive[account.info.id] || account.ended_at
+
+          account.ended_at = account.last_seen_at
+          account.now = nil
+        end
+      end
+
+      # Members that exited since the last sample will show up again as their parent's child time.
+      def hand_over_exits(accounts, seen, live)
+        seen.each do |identity, (session_id, total)|
+          next if live[identity]
+
+          account = accounts[session_id] or next
+          if account.alive?
+            expect_handover(account, total)
+          elsif identity[0] == account.info.root_pid && account.info.kind == :cli
+            heir = accounts[account.parent_session]
+            expect_handover(heir, total) if heir&.alive?
+          end
+        end
+      end
+
+      def update(account, info, procs, rates, seen, next_seen, at, map)
+        grown = 0.0
+        readable = procs.select(&:readable)
+        readable.each do |p|
+          total = p.cpu_time + p.child_cpu_time
+          was = seen[p.identity]
+          was = nil unless was && was[0] == info.id
+          grown += was ? [total - was[1], 0.0].max : total
+          account.bytes_read += was ? [p.disk_read - was[2], 0].max : p.disk_read
+          account.bytes_written += was ? [p.disk_written - was[3], 0].max : p.disk_written
+          next_seen[p.identity] = [info.id, total, p.disk_read, p.disk_written]
+        end
+        account.cpu_seconds += grown - absorb(account, grown)
+
+        footprint = readable.sum(&:footprint)
+        account.peak_footprint = [account.peak_footprint, footprint].max
+        account.info = info
+        account.last_seen_at = at
+        account.ended_at = nil
+        account.parent_session = map.by_pid[procs.find { |p| p.pid == info.root_pid }&.ppid]
+        account.now = {
+          processes: procs.size,
+          cpu: procs.sum { |p| rates[p.pid]&.cpu || 0.0 },
+          footprint:,
+          resident: procs.sum { |p| p.resident || 0 },
+          read_rate: procs.sum { |p| rates[p.pid]&.read_rate || 0.0 },
+          write_rate: procs.sum { |p| rates[p.pid]&.write_rate || 0.0 }
+        }
+      end
+
+      def to_session(account)
+        info = account.info
+        now = account.now || { processes: 0, cpu: 0.0, footprint: 0, resident: 0, read_rate: 0.0, write_rate: 0.0 }
+        Session.new(
+          id: info.id, kind: info.kind, name: info.name, root_pid: info.root_pid, label: info.label, cwd: info.cwd,
+          started_at: info.started_at || account.first_seen_at, first_seen_at: account.first_seen_at,
+          last_seen_at: account.last_seen_at, ended_at: account.ended_at,
+          peak_footprint: account.peak_footprint, cpu_seconds: account.cpu_seconds,
+          bytes_read: account.bytes_read, bytes_written: account.bytes_written,
+          title: info.title, status: info.status, threads: info.threads, **now
+        )
+      end
+    end
+  end
+
+  metric(:session_ledger) { |reading, state| Metrics::SessionLedger.new(state).call(reading) }
+end

data/lib/agentmon/metrics/session_memory.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# reading[:session_memory]: how much memory each alive agent session holds, as SessionMemory
+# records (lib/agentmon/model.rb), largest footprint first. For example:
+#
+#   claude 4242 · repo   footprint 2.1G  resident 2.4G  wired 1.0M  26.3%  +1.5M/s  10 pageins/s
+#   Claude 59334         footprint 900M  resident 1.1G  wired   0B  11.0%  -12K/s    0 pageins/s
+#
+# - footprint, resident, wired, pageins, pagein_rate, fault_rate: sums over the session's live
+#   members in reading[:process_rows], each pid once. A member whose value is unknown (nil: an
+#   unreadable process, or a rate before two samples) is skipped; a sum with no known member is
+#   nil, never 0. Members are rows whose session_id is the session's, so a process tree counts
+#   each process once (sessions.rb puts every pid in at most one session).
+# - processes: the live members seen in process_rows.
+# - peak_footprint: the ledger's (Session#peak_footprint), the largest footprint sum over its life.
+# - share: footprint as a percent of used memory (reading[:memory].used); nil while memory is
+#   unknown.
+# - growth_rate: footprint change in bytes per second over the last WINDOW seconds of samples, on
+#   the monotonic clock; 0.0 until a session has two samples (as pressure_drivers does), nil while
+#   its footprint is unknown. Only samples with the same readable members (pids with a known
+#   footprint) are compared, so a member joining, leaving or turning unreadable restarts the
+#   window rather than showing as growth.
+#
+# Compressed and swapped bytes per process need task_for_pid (root), so they aren't here; the
+# machine's are in reading[:memory]. Empty when the ledger is nil; when reading[:process_rows] is
+# nil, one record per alive session with nil sums, since no member is known. Stateful: a short
+# footprint history per session id, dropped when the session is no longer alive.
+module Agentmon
+  module Metrics
+    class SessionFootprint
+      WINDOW = 60.0
+
+      def initialize(state) = @state = state
+
+      def call(reading)
+        history = (@state[:history] ||= {}) # session id => [[mono, footprint], ...], oldest first
+        alive = (reading[:session_ledger] || []).select(&:alive?)
+        history.select! { |id, _| alive.any? { |s| s.id == id } }
+        members = (reading[:process_rows] || []).uniq(&:pid).group_by(&:session_id)
+        used = reading[:memory]&.used
+        mono = reading.sample.mono
+
+        alive.map do |session|
+          rows = members[session.id] || []
+          footprint = sum(rows, :footprint)
+          SessionMemory.new(
+            session_id: session.id, label: session.label, processes: rows.size, footprint:,
+            resident: sum(rows, :resident), wired: sum(rows, :wired), peak_footprint: session.peak_footprint,
+            share: footprint && used&.positive? ? footprint * 100.0 / used : nil,
+            growth_rate: growth(history[session.id] ||= [], mono, footprint, readable(rows)),
+            pageins: sum(rows, :pageins), pagein_rate: sum(rows, :pagein_rate), fault_rate: sum(rows, :fault_rate)
+          )
+        end.sort_by { |m| [-(m.footprint || 0), m.session_id] }
+      end
+
+      private
+
+      # The sum of the known values; nil when none is known.
+      def sum(rows, field)
+        known = rows.filter_map { |r| r.public_send(field) }
+        known.empty? ? nil : known.sum
+      end
+
+      # The pids whose footprint is known, sorted.
+      def readable(rows) = rows.reject { |r| r.footprint.nil? }.map(&:pid).sort
+
+      # Adds this sample to a session's points, forgets those older than WINDOW or with other
+      # readable members, and returns the rate from the oldest left to this one; nil (and nothing
+      # recorded) while the footprint is unknown.
+      def growth(points, mono, footprint, pids)
+        return nil if footprint.nil?
+
+        points.pop if points.last && points.last[0] >= mono # the same sample seen again
+        points.clear if points.last && points.last[2] != pids # members changed: restart the window
+        points << [mono, footprint, pids]
+        points.shift while mono - points.first[0] > WINDOW
+        since, was, = points.first
+        mono > since ? (footprint - was) / (mono - since) : 0.0
+      end
+    end
+  end
+
+  metric(:session_memory) { |reading, state| Metrics::SessionFootprint.new(state).call(reading) }
+end

data/lib/agentmon/metrics/session_names.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "json"
+
+# reading[:session_names]: { root pid => SessionName } for the agent CLI processes whose human
+# name (or status) is known. Metrics::Sessions puts the name at the front of the session label.
+#
+# Where names come from (no subprocess, ever):
+#
+# - Claude Code writes ~/.claude/sessions/<pid>.json for each CLI session ("name", "status"
+#   busy/idle, "startedAt" ms, ...). Re-parsed only when its mtime changes.
+# - One codex process (the ChatGPT app's `codex app-server`, or the codex CLI) hosts threads and
+#   keeps each one's rollout file open: $CODEX_HOME/sessions/YYYY/MM/DD/rollout-<time>-<uuid>.jsonl.
+#   $CODEX_HOME/session_index.jsonl maps thread id => thread_name (later lines win); it is
+#   append-only, so it is read incrementally from the last byte offset (from 0 if it shrank).
+#   The open files come from Darwin.open_paths, once per live codex pid. The title is the newest
+#   (rollout mtime) named open thread, plus " +N" for its N other open threads.
+#
+# Cost: nothing per sample. It refreshes at most every REFRESH seconds (monotonic); in between
+# it returns its last value. Unreadable, missing or garbled files mean no name, never an error.
+module Agentmon
+  module Metrics
+    module SessionNames
+      REFRESH = 10.0
+      CLI = /\A(claude|codex)\z/
+      ROLLOUT = /rollout-.*-(\h{8}-\h{4}-\h{4}-\h{4}-\h{12})\.jsonl\z/
+      # A Claude session file written before this process started belongs to an earlier process
+      # that had the same pid (seconds of slack for clock rounding).
+      STALE_SLACK = 60
+
+      class << self
+        # Injectable for tests (test_helper points them at an empty dir and a nil reader).
+        attr_writer :claude_dir, :codex_home, :open_paths
+
+        def claude_dir = @claude_dir || File.join(Dir.home, ".claude", "sessions")
+        def codex_home = @codex_home || ENV["CODEX_HOME"] || File.join(Dir.home, ".codex")
+        def open_paths = @open_paths || Darwin.method(:open_paths)
+
+        # The metric: the last value until REFRESH seconds have passed, then a refresh.
+        def call(reading, state)
+          mono = reading.sample.mono
+          return state[:value] if state[:at] && mono - state[:at] < REFRESH
+
+          state[:at] = mono
+          state[:value] = refresh(reading.sample[:processes] || [], state)
+        end
+
+        # { pid => SessionName } for the live claude/codex roots in `processes`.
+        def refresh(processes, state)
+          by_pid = processes.to_h { |p| [p.pid, p] }
+          cli = ->(p) { p && p.name.to_s.match?(CLI) }
+          roots = processes.select { |p| cli[p] && !cli[by_pid[p.ppid]] } # under another CLI: not a root
+          names = {}
+          claude(roots.select { |p| p.name == "claude" }, state, names)
+          codex = roots.select { |p| p.name == "codex" }
+          codex(codex, state, names) unless codex.empty?
+          names
+        end
+
+        private
+
+        def claude(processes, state, names)
+          cache = (state[:claude] ||= {}) # path => [mtime, SessionName or nil, startedAt seconds]
+          live = {}
+          processes.each do |p|
+            path = File.join(claude_dir, "#{p.pid}.json")
+            live[path] = true
+            entry = claude_file(path, cache)
+            next unless entry
+
+            _, name, started = entry
+            next if started && p.started_at && started < p.started_at - STALE_SLACK
+
+            names[p.pid] = name if name
+          end
+          cache.select! { |path, _| live[path] }
+        end
+
+        def claude_file(path, cache)
+          mtime = File.mtime(path)
+          return cache[path] if cache[path] && cache[path][0] == mtime
+
+          cache[path] = [mtime, *parse_claude(File.read(path))]
+        rescue SystemCallError, IOError
+          cache.delete(path)
+          nil
+        end
+
+        # [SessionName or nil, startedAt in epoch seconds or nil]
+        def parse_claude(text)
+          json = JSON.parse(text)
+          return [nil, nil] unless json.is_a?(Hash)
+
+          title = json["name"].is_a?(String) && !json["name"].strip.empty? ? json["name"].strip : nil
+          status = json["status"].is_a?(String) ? json["status"] : nil
+          started = json["startedAt"].is_a?(Numeric) ? json["startedAt"] / 1000.0 : nil
+          [title || status ? SessionName.new(title:, status:, threads: []) : nil, started]
+        rescue JSON::ParserError, EncodingError
+          [nil, nil]
+        end
+
+        def codex(processes, state, names)
+          index = read_index(state)
+          reader = open_paths
+          processes.each do |p|
+            paths = begin
+              reader.call(p.pid)
+            rescue StandardError
+              nil
+            end
+            name = codex_name(paths || [], index)
+            names[p.pid] = name if name
+          end
+        end
+
+        # The open rollout files' threads, newest first, named from the index.
+        def codex_name(paths, index)
+          threads = paths.filter_map { |path| (m = ROLLOUT.match(path)) && [m[1], path] }.uniq(&:first)
+          dated = threads.filter_map do |id, path|
+            [id, File.mtime(path)]
+          rescue SystemCallError
+            nil
+          end
+          return nil if dated.empty?
+
+          named = dated.sort_by { |_, mtime| -mtime.to_f }.filter_map { |id, _| index[id] }
+          return nil if named.empty?
+
+          more = dated.size - 1
+          SessionName.new(title: more.positive? ? "#{named.first} +#{more}" : named.first, status: nil, threads: named)
+        end
+
+        # The thread id => name map, after reading what was appended since the last refresh.
+        def read_index(state)
+          index = (state[:index] ||= { offset: 0, names: {} })
+          path = File.join(codex_home, "session_index.jsonl")
+          size = File.size(path)
+          index.merge!(offset: 0, names: {}) if size < index[:offset]
+          return index[:names] if size == index[:offset]
+
+          chunk = File.open(path, "rb") do |f|
+            f.seek(index[:offset])
+            f.read(size - index[:offset])
+          end.to_s
+          complete = chunk.rindex("\n")
+          return index[:names] unless complete # a line still being written: next time
+
+          chunk.byteslice(0, complete + 1).each_line { |line| index_line(line, index[:names]) }
+          index[:offset] += complete + 1
+          index[:names]
+        rescue SystemCallError, IOError
+          index[:names]
+        end
+
+        def index_line(line, names)
+          json = JSON.parse(line.force_encoding(Encoding::UTF_8))
+          return unless json.is_a?(Hash) && json["id"].is_a?(String)
+
+          name = json["thread_name"]
+          names[json["id"]] = name.strip if name.is_a?(String) && !name.strip.empty?
+        rescue JSON::ParserError, EncodingError
+          nil
+        end
+      end
+    end
+  end
+
+  metric(:session_names) { |reading, state| Metrics::SessionNames.call(reading, state) }
+end

data/lib/agentmon/metrics/sessions.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# reading[:sessions]: a SessionMap of the agent sessions alive in this sample and which session
+# each pid belongs to.
+#
+# A process belongs to its outermost `claude`/`codex` CLI ancestor (itself included), so shells,
+# tools, MCP servers and sub-agents count toward the session that started them. With no CLI
+# ancestor, it belongs to the topmost agent desktop app ancestor (Claude, ChatGPT/Codex), so all
+# of an app's helpers are one session. Claude Code sessions that the desktop app launches are CLI
+# sessions of their own, not part of the app's. Anything else is in no session.
+module Agentmon
+  module Metrics
+    module Sessions
+      CLI = /\A(claude|codex)\z/
+      APP = /claude|codex|chatgpt/i
+
+      module_function
+
+      # `names`: reading[:session_names], { root pid => SessionName } (nil or {} when unknown).
+      def call(processes, cwds, names = {})
+        processes ||= []
+        cwds ||= {}
+        names ||= {}
+        by_pid = processes.to_h { |p| [p.pid, p] }
+        chains = {}
+        sessions = {}
+        map = {}
+        processes.each do |p|
+          cli, app = chain(p, by_pid, chains)
+          root = cli || app
+          next unless root
+
+          info = sessions[root.pid] ||= info(root, cli ? :cli : :app, cwds, names[root.pid])
+          map[p.pid] = info.id
+        end
+        SessionMap.new(sessions: sessions.values, by_pid: map)
+      end
+
+      # [outermost CLI ancestor-or-self, topmost app ancestor-or-self], memoized per pid. Walks up
+      # iteratively and stops at a cycle (pid 0 is its own parent).
+      def chain(process, by_pid, memo)
+        path = []
+        seen = {}
+        node = process
+        while node && !memo.key?(node.pid) && !seen[node.pid]
+          seen[node.pid] = true
+          path << node
+          node = by_pid[node.ppid]
+        end
+        cli, app = node && memo[node.pid]
+        path.reverse_each do |n|
+          cli ||= n if n.name.match?(CLI)
+          app ||= n if n.name.match?(APP)
+          memo[n.pid] = [cli, app]
+        end
+        memo[process.pid]
+      end
+
+      # Label: "claude 4242 · repo" (directory of a CLI root), "Claude 300" (app); with a human
+      # name first: "Agentmon r2ui integration · claude 4242 · repo".
+      def info(root, kind, cwds, name = nil)
+        cwd = cwds[root.pid]
+        id = [root.name, root.pid, root.started_at&.to_i].compact.join("-")
+        label = if kind == :cli && cwd && cwd != "/"
+                  "#{root.name} #{root.pid} · #{File.basename(cwd)}"
+                else
+                  "#{root.name} #{root.pid}"
+                end
+        title = name&.title
+        label = "#{title} · #{label}" if title && !title.empty?
+        SessionInfo.new(id:, kind:, name: root.name, root_pid: root.pid, label:, cwd:, started_at: root.started_at,
+                        title:, status: name&.status, threads: name&.threads || [])
+      end
+    end
+  end
+
+  metric(:sessions) do |reading|
+    Metrics::Sessions.call(reading.sample[:processes], reading.sample[:cwd], reading[:session_names])
+  end
+end

data/lib/agentmon/metrics/system.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Agentmon
+  # reading[:system]: the whole machine's CPU, load, network and disk now, as a SystemView, derived
+  # from the system probe's SystemStat (sample[:system]) and reading[:process_rows]. Nil while there
+  # is no SystemStat (off macOS, or the probe failed).
+  #
+  #   cpu, user, system  percent of the whole machine (0..100): tick deltas over all CPUs since the
+  #                      previous sample / total tick delta. cpu counts user + system + nice
+  #   ncpu               logical CPUs
+  #   load1/5/15         load averages
+  #   net_in_rate,       bytes/s over the monotonic interval (the probes' at_mono, else the
+  #   net_out_rate       samples' mono)
+  #   disk_read_rate,    bytes/s: the sum of read_rate / write_rate over reading[:process_rows]
+  #   disk_write_rate    where known (readable processes; other users' are not counted). Nil with
+  #                      no rows or no known rate
+  #   *_trend            the last 120 values (four minutes at 2 s), oldest first: metric state
+  #
+  # Every delta is nil on the first sample and when a counter went backwards (an interface reset,
+  # a wrapped tick counter): unknown, never negative. Unknown values are not appended to trends.
+  #
+  # Metrics are computed on demand (Reading#[]), so reading[:process_rows] is ready whichever order
+  # the metrics are registered in. Session focus does not narrow this value: it is machine-wide.
+  SystemView = Data.define(
+    :cpu, :user, :system,                  # percent of the machine
+    :ncpu,
+    :load1, :load5, :load15,
+    :net_in_rate, :net_out_rate,           # bytes/s
+    :disk_read_rate, :disk_write_rate,     # bytes/s
+    :cpu_trend, :net_in_trend, :net_out_trend, :disk_trend # last TREND values, oldest first
+  )
+
+  module Metrics
+    # Named apart from the SystemView/SystemStat models and the system probe's module.
+    module MachineLoad
+      TREND = 120
+      TRENDS = %i[cpu net_in net_out disk].freeze
+
+      module_function
+
+      # `now`/`before` are SystemStats (`before` nil on the first sample), `rows` the reading's
+      # process rows, `fallback` the samples' monotonic interval; `state` the metric's state Hash.
+      def call(now, before, rows, fallback, state)
+        return nil unless now
+
+        user, system, cpu = percents(now.cpu_ticks, before&.cpu_ticks)
+        interval = interval(now, before, fallback)
+        net_in = rate(now.net_in, before&.net_in, interval)
+        net_out = rate(now.net_out, before&.net_out, interval)
+        disk_read = disk(rows, :read_rate)
+        disk_write = disk(rows, :write_rate)
+        disk_total = disk_read && disk_write ? disk_read + disk_write : nil
+
+        trends = TRENDS.to_h { |name| [name, state[name] ||= []] }
+        { cpu:, net_in:, net_out:, disk: disk_total }.each { |name, value| push(trends[name], value) }
+
+        load1, load5, load15 = now.load
+        SystemView.new(cpu:, user:, system:, ncpu: now.ncpu, load1:, load5:, load15:,
+                       net_in_rate: net_in, net_out_rate: net_out,
+                       disk_read_rate: disk_read, disk_write_rate: disk_write,
+                       cpu_trend: trends[:cpu].dup.freeze, net_in_trend: trends[:net_in].dup.freeze,
+                       net_out_trend: trends[:net_out].dup.freeze, disk_trend: trends[:disk].dup.freeze)
+      end
+
+      # [user, system, busy] percents between two [user, system, idle, nice] tick readings, or
+      # [nil, nil, nil] when either is unknown, no ticks passed, or a counter went backwards.
+      def percents(now, was)
+        return [nil, nil, nil] unless now && was
+
+        user, system, idle, nice = now.zip(was).map { |a, b| a - b }
+        total = user + system + idle + nice
+        return [nil, nil, nil] if [user, system, idle, nice].any?(&:negative?) || !total.positive?
+
+        pct = ->(ticks) { ticks * 100.0 / total }
+        [pct[user], pct[system], pct[user + system + nice]]
+      end
+
+      # Seconds between the two probe reads, else between the samples.
+      def interval(now, before, fallback)
+        return nil unless before
+
+        if now.at_mono && before.at_mono
+          now.at_mono - before.at_mono
+        else
+          fallback
+        end
+      end
+
+      # Bytes/s between two readings of a cumulative counter; nil when unknown or it went backwards.
+      def rate(now, was, interval)
+        return nil if now.nil? || was.nil? || interval.nil? || !interval.positive? || now < was
+
+        (now - was) / interval.to_f
+      end
+
+      def disk(rows, field)
+        known = rows&.filter_map(&field)
+        known.nil? || known.empty? ? nil : known.sum.to_f
+      end
+
+      def push(trend, value)
+        return if value.nil?
+
+        trend << value
+        trend.shift while trend.size > TREND
+      end
+    end
+  end
+
+  metric(:system) do |reading, state|
+    Metrics::MachineLoad.call(reading.sample[:system], reading.previous&.[](:system), reading[:process_rows],
+                              reading.interval, state)
+  end
+end