@cohortapp/agent-sdk 2.5.1 → 2.6.0

package/lib/telemetry/collect.mjs

@@ -14,11 +14,21 @@
  *     subAgentsRunning: number,         // running claude child processes (best-effort)
  *     session: { id?, taskClass?, model? } | null,
  *     machine: {
+ *       // MOMENTARY — true at `ts`, expires with the snapshot
  *       loadavg: [n,n,n],
  *       memUsedPct: number,             // 1 - free/total
- *       cpuPct?: number,                // best-effort (powermetrics)
- *       tempC?: number,                 // best-effort (powermetrics)
+ *       cpuPct?: number,                // powermetrics if root, else loadavg/cores
+ *       cpuSource?: "powermetrics"|"sudo-powermetrics"|"loadavg",
+ *       tempC?: number,                 // best-effort (powermetrics; needs root)
+ *       tempSource?: "powermetrics"|"sudo-powermetrics",
+ *       tempDetail?: string,            // why there is no tempC (never silent)
  *       diskUsedPct?: number,
+ *       uptimeDays?: number,
+ *       spend24h?: number,              // rolling 24h billable USD off the cost ledger
+ *       // INVENTORY — describes the machine; does not perish with the snapshot
+ *       device?: string,                // "Mac mini M4 Pro"
+ *       ram?: string,                   // "24 GB"
+ *       cores?: number,
  *     },
  *     claudeAuth: "ok"|"relogin_required"|"unknown",
  *     alerts: [{ id, severity, kind, detail }],
@@ -48,10 +58,11 @@
 
 import os from "node:os";
 import { execFile } from "node:child_process";
-import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 
 import { list as listPresence } from "../collective/presence.mjs";
+import { billableUsd } from "../cost/ledger-row.mjs";
 import { liveClaudeStats } from "../resource-governor.mjs";
 
 /** Default temperature probe ceiling — used only as a guard in alerts; here we just report. */
@@ -191,10 +202,40 @@ function humanActivity(intent, taskClass) {
 
 /**
  * Gather machine stats from node:os plus best-effort `powermetrics` (temp/cpu)
- * and `df` (disk). The exec probes are guarded + injected; on macOS powermetrics
- * needs root so a non-root agent simply gets no temp/cpu (those fields drop out).
+ * and `df` (disk). The exec probes are guarded + injected.
  *
- * @returns {Promise<{loadavg:number[], memUsedPct:number, cpuPct?:number, tempC?:number, diskUsedPct?:number}>}
+ * ── THE VOCABULARY IS THE CONTRACT ───────────────────────────────────────────
+ * Every key below is READ BY NAME on the other side of the beat
+ * (`hq/src/components/fleet/machine.ts#buildFleetMachine`). This collector and
+ * that reader once shared ZERO keys in production — it emitted
+ * `loadavg/memUsedPct/cpuPct/diskUsedPct` and the Fleet row read
+ * `tempC/uptimeDays/spend24h/ram/device`, so four measured numbers were
+ * persisted and rendered nowhere while five columns em-dashed for want of a
+ * producer. Adding a key here without adding its reader there (or vice versa)
+ * re-opens exactly that hole; `machine.test.ts` asserts the two sets agree.
+ *
+ * ── MOMENTARY vs INVENTORY ───────────────────────────────────────────────────
+ * MOMENTARY (describes the machine AT `status.ts`, and expires with it):
+ *   loadavg, memUsedPct, cpuPct, tempC, diskUsedPct, uptimeDays, spend24h
+ * INVENTORY (describes the machine, does not perish):
+ *   device, ram, cores
+ * The reader nulls the momentary set once the snapshot goes stale and keeps the
+ * inventory set. That split is why they are documented as one list here.
+ *
+ * ── EVERY BEST-EFFORT FIELD CARRIES ITS PROVENANCE ───────────────────────────
+ * `cpuSource` and `tempSource` say WHERE the number came from, and `tempDetail`
+ * says why there is none. A `cpuPct` derived from loadavg/cores is a different
+ * claim from one measured by powermetrics, and a missing `tempC` because the
+ * probe needs root is a different fact from a machine with no sensor. Both used
+ * to be indistinguishable from success: `collectPowermetrics` returned `{}` on
+ * every failure path and the field simply vanished. Fail-open is fine; silent
+ * is not.
+ *
+ * @returns {Promise<{
+ *   loadavg:number[], memUsedPct:number, cpuPct?:number, cpuSource?:string,
+ *   tempC?:number, tempSource?:string, tempDetail?:string, diskUsedPct?:number,
+ *   uptimeDays?:number, spend24h?:number, device?:string, ram?:string, cores?:number
+ * }>}
  */
 export async function collectMachine(o = {}) {
   const osImpl = o.os || os;
@@ -224,15 +265,34 @@ export async function collectMachine(o = {}) {
     const n = Array.isArray(cpus) ? cpus.length : 0;
     if (n > 0 && Number.isFinite(machine.loadavg[0])) {
       const est = (machine.loadavg[0] / n) * 100;
-      if (Number.isFinite(est)) machine.cpuPct = pct(est);
+      if (Number.isFinite(est)) {
+        machine.cpuPct = pct(est);
+        machine.cpuSource = "loadavg";
+      }
     }
   } catch { /* no cpuPct */ }
 
+  // uptimeDays — os.uptime() is seconds since boot. Momentary, and one of the
+  // four fields the Fleet row read from a producer that never wrote them.
+  try {
+    const secs = Number(osImpl.uptime ? osImpl.uptime() : NaN);
+    if (Number.isFinite(secs) && secs >= 0) machine.uptimeDays = round1(secs / 86400);
+  } catch { /* no uptimeDays */ }
+
   // Best-effort temp + cpu via powermetrics (guarded, injected, never blocks).
+  // ALWAYS records why it has no answer — see collectPowermetrics.
   if (o.powermetrics !== false) {
     const pm = await collectPowermetrics(o);
-    if (Number.isFinite(pm.tempC)) machine.tempC = round1(pm.tempC);
-    if (Number.isFinite(pm.cpuPct)) machine.cpuPct = pct(pm.cpuPct);
+    if (Number.isFinite(pm.tempC)) {
+      machine.tempC = round1(pm.tempC);
+      machine.tempSource = pm.source || "powermetrics";
+    } else if (pm.detail) {
+      machine.tempDetail = pm.detail;
+    }
+    if (Number.isFinite(pm.cpuPct)) {
+      machine.cpuPct = pct(pm.cpuPct);
+      machine.cpuSource = pm.source || "powermetrics";
+    }
   }
 
   // Best-effort diskUsedPct via df.
@@ -241,30 +301,121 @@ export async function collectMachine(o = {}) {
     if (Number.isFinite(disk)) machine.diskUsedPct = pct(disk);
   }
 
+  // INVENTORY — device / ram / cores. Cached on disk (see collectInventory):
+  // the chip and the installed RAM do not change between beats, so the
+  // system_profiler fork happens once per machine, not every 30 seconds.
+  if (o.inventory !== false) {
+    const inv = await collectInventory(o);
+    if (inv.device) machine.device = inv.device;
+    if (inv.ram) machine.ram = inv.ram;
+    if (Number.isFinite(inv.cores)) machine.cores = inv.cores;
+  }
+
+  // spend24h — the agent's own rolling 24h LLM spend, off the cost ledger.
+  if (o.spend !== false) {
+    const spend = collectSpend24h(o);
+    if (Number.isFinite(spend)) machine.spend24h = spend;
+  }
+
   return machine;
 }
 
+/**
+ * The `powermetrics` sampler that carries die temperature, BY CPU ARCHITECTURE.
+ *
+ * ── A BUG THAT MADE THE PROBE UNREACHABLE ON EVERY APPLE-SILICON MAC ─────────
+ * This used to be the constant `"smc,cpu_power"`. `smc` is an INTEL-ONLY
+ * sampler: on arm64 `powermetrics` rejects the argument list outright —
+ * `powermetrics: unrecognized sampler: smc` — and exits non-zero BEFORE it ever
+ * reaches the root check. So on the only hardware the fleet actually runs on,
+ * the probe failed at argument parsing, `execFileSafe` swallowed the non-zero
+ * exit into `""`, and the caller could not tell that apart from a machine with
+ * no thermal sensor. The arm64 spelling is `thermal`.
+ *
+ * `cpu_power` is valid on both and is what yields the idle-residency line the
+ * parser reads for `cpuPct`, so it stays in either list.
+ */
+export function powermetricsSamplers(arch) {
+  return String(arch || "") === "arm64" ? "thermal,cpu_power" : "smc,cpu_power";
+}
+
 /**
  * Parse `powermetrics` for CPU temperature + active CPU%. Guarded: only attempts
  * the exec on darwin (unless an explicit `deps.execFile` + `deps.platform` say
- * otherwise for tests). Returns {} on any failure. NEVER blocks past timeoutMs.
+ * otherwise for tests). NEVER blocks past timeoutMs, NEVER throws.
  *
- * @returns {Promise<{ tempC?:number, cpuPct?:number }>}
+ * ── IT ALWAYS SAYS WHY IT HAS NOTHING ────────────────────────────────────────
+ * This returned a bare `{}` for all four of its failure modes — wrong sampler,
+ * missing binary, not root, no sensor — and the caller dropped the field. The
+ * Fleet view then printed an em-dash that meant "we do not know, and we cannot
+ * tell you why", which is how a two-month-dead probe stayed invisible. Every
+ * exit now carries a `detail` sentence that the snapshot ships and the drawer
+ * prints.
+ *
+ * ── ROOT, AND THE ONE HONEST WAY AROUND IT ───────────────────────────────────
+ * `powermetrics` requires superuser on macOS, full stop; the daemon runs from a
+ * user LaunchAgent. There is NO non-root numeric temperature source on Apple
+ * Silicon without a compiled IOKit helper (`ioreg` exposes the PMU/NVMe sensors
+ * as HID services whose values are only readable through IOHIDEventSystem, not
+ * through the registry), so this does not pretend to have one. Instead it
+ * supports an OPT-IN sudo path: set `TELEMETRY_SUDO_POWERMETRICS=1` once the
+ * operator has allowlisted the binary NOPASSWD in sudoers. `sudo -n` never
+ * prompts — with no allowlist it fails instantly rather than hanging a beat on
+ * a password prompt no one will ever see.
+ *
+ * @returns {Promise<{ tempC?:number, cpuPct?:number, source?:string, detail?:string }>}
  */
 export async function collectPowermetrics(o = {}) {
   const platform = o.platform || (o.os && o.os.platform && o.os.platform()) || process.platform;
   const execFileImpl = o.execFile || execFile;
-  if (platform !== "darwin" && !o.execFile) return {}; // only darwin in the wild; tests inject execFile
+  // only darwin in the wild; tests inject execFile
+  if (platform !== "darwin" && !o.execFile) {
+    return { detail: `no temperature probe on platform ${platform}` };
+  }
   const timeoutMs = Number.isFinite(o.pmTimeoutMs) ? o.pmTimeoutMs : 1500;
   const bin = o.powermetricsBin || "/usr/bin/powermetrics";
-  if (!o.execFile && !existsSync(bin)) return {};
-  const out = await execFileSafe(
-    execFileImpl, bin,
-    ["-n", "1", "-i", "200", "--samplers", "smc,cpu_power", "--show-process-energy"],
-    timeoutMs,
-  );
-  if (!out) return {};
-  return parsePowermetrics(out);
+  if (!o.execFile && !existsSync(bin)) {
+    return { detail: `${bin} not present` };
+  }
+  const arch = o.arch || (o.os && o.os.arch && o.os.arch()) || process.arch;
+  const args = [
+    "-n", "1", "-i", "200",
+    "--samplers", powermetricsSamplers(arch),
+    "--show-process-energy",
+  ];
+
+  // 1. Direct. Works when the daemon happens to run as root.
+  const direct = await execFileSafe(execFileImpl, bin, args, timeoutMs);
+  if (direct) {
+    const parsed = parsePowermetrics(direct);
+    if (Number.isFinite(parsed.tempC) || Number.isFinite(parsed.cpuPct)) {
+      return { ...parsed, source: "powermetrics" };
+    }
+  }
+
+  // 2. Opt-in sudo, only when the operator has allowlisted it. `-n` = never prompt.
+  const sudoEnabled = o.sudoPowermetrics !== undefined
+    ? Boolean(o.sudoPowermetrics)
+    : String(process.env.TELEMETRY_SUDO_POWERMETRICS || "") === "1";
+  if (sudoEnabled) {
+    const sudoBin = o.sudoBin || "/usr/bin/sudo";
+    const out = await execFileSafe(execFileImpl, sudoBin, ["-n", bin, ...args], timeoutMs);
+    if (out) {
+      const parsed = parsePowermetrics(out);
+      if (Number.isFinite(parsed.tempC) || Number.isFinite(parsed.cpuPct)) {
+        return { ...parsed, source: "sudo-powermetrics" };
+      }
+    }
+    return {
+      detail: `${bin} yielded nothing under 'sudo -n' — check the NOPASSWD sudoers allowlist`,
+    };
+  }
+
+  return {
+    detail:
+      `${bin} requires root (agent runs unprivileged); ` +
+      "allowlist it NOPASSWD in sudoers and set TELEMETRY_SUDO_POWERMETRICS=1 to enable",
+  };
 }
 
 /** Pure parser for powermetrics text → { tempC?, cpuPct? }. Tolerant of format drift. */
@@ -307,6 +458,190 @@ export function parseDf(text) {
 function round1(n) { return Math.round(Number(n) * 10) / 10; }
 function round2(n) { return Math.round(Number(n) * 100) / 100; }
 
+// ---------------------------------------------------------------------------
+// Inventory (device / ram / cores) — what the machine IS, not what it is doing
+// ---------------------------------------------------------------------------
+
+/**
+ * Human-readable installed RAM, e.g. `24 GB`. Binary GB (the unit every Apple
+ * spec sheet and Activity Monitor uses), rounded to whole units — 25769803776
+ * bytes is "24 GB", never "25.8 GB".
+ */
+export function ramLabel(totalBytes) {
+  const b = Number(totalBytes);
+  if (!Number.isFinite(b) || b <= 0) return "";
+  const gb = b / 1024 ** 3;
+  return `${gb >= 10 ? Math.round(gb) : round1(gb)} GB`;
+}
+
+/**
+ * Pure parser for `system_profiler SPHardwareDataType` text → the inventory
+ * triple. Tolerant of format drift and of a machine that reports only some of
+ * the three (an absent key simply drops out).
+ *
+ * @returns {{device?:string, ram?:string, cores?:number}}
+ */
+export function parseSystemProfilerHardware(text) {
+  const s = String(text || "");
+  const out = {};
+  const grab = (label) => {
+    const m = s.match(new RegExp(`^\\s*${label}:\\s*(.+)$`, "im"));
+    return m ? m[1].trim() : "";
+  };
+  // "Mac mini" + "Apple M4 Pro" → "Mac mini M4 Pro", matching the design's
+  // "Mac Studio M3 Ultra" idiom. Either half alone is still an honest answer.
+  const model = grab("Model Name");
+  const chip = grab("Chip") || grab("Processor Name");
+  const chipShort = chip.replace(/^Apple\s+/i, "");
+  const device = [model, model && chipShort && chipShort !== model ? chipShort : chip && !model ? chip : ""]
+    .filter(Boolean)
+    .join(" ")
+    .trim();
+  if (device) out.device = device;
+
+  const mem = grab("Memory");
+  if (mem) out.ram = mem;
+
+  const cores = grab("Total Number of Cores");
+  const n = Number(String(cores).match(/^\d+/)?.[0]);
+  if (Number.isFinite(n) && n > 0) out.cores = n;
+
+  return out;
+}
+
+/**
+ * Resolve device / ram / cores, CACHED ON DISK.
+ *
+ * WHY CACHED. `system_profiler` costs ~200ms and forks a privileged-ish helper;
+ * the chip and the installed RAM do not change between two beats 30 seconds
+ * apart. Probing once per machine and reading a JSON file thereafter keeps the
+ * beat's cost where it belongs. The cache is KEYED on `os.cpus()[0].model` +
+ * total bytes, so swapping the machine (or an agent root moved to different
+ * hardware) invalidates it rather than reporting the old box forever.
+ *
+ * Fail-open in both directions: no agentRoot → probe every time; probe fails →
+ * fall back to what node:os alone can prove (`Apple M4 Pro`, `24 GB`, `12`),
+ * which is never nothing.
+ *
+ * @returns {Promise<{device?:string, ram?:string, cores?:number}>}
+ */
+export async function collectInventory(o = {}) {
+  const osImpl = o.os || os;
+  let chip = "";
+  let totalBytes = 0;
+  let cores = 0;
+  try {
+    const cpus = osImpl.cpus();
+    cores = Array.isArray(cpus) ? cpus.length : 0;
+    chip = (Array.isArray(cpus) && cpus[0] && cpus[0].model) || "";
+  } catch { /* fall through */ }
+  try { totalBytes = Number(osImpl.totalmem()) || 0; } catch { /* fall through */ }
+
+  const key = `${chip}|${totalBytes}`;
+  const cachePath = o.agentRoot
+    ? join(resolve(o.agentRoot), "state", "telemetry", "inventory.json")
+    : null;
+
+  if (cachePath && o.inventoryCache !== false) {
+    const cached = safeReadJson(cachePath);
+    if (cached && cached.key === key && cached.inv && typeof cached.inv === "object") {
+      return cached.inv;
+    }
+  }
+
+  // What node:os alone can prove — the floor, never overwritten by a worse probe.
+  const inv = {};
+  if (chip) inv.device = chip;
+  const ram = ramLabel(totalBytes);
+  if (ram) inv.ram = ram;
+  if (cores > 0) inv.cores = cores;
+
+  const platform = o.platform || (osImpl.platform && osImpl.platform()) || process.platform;
+  if (platform === "darwin" || o.execFile) {
+    const execFileImpl = o.execFile || execFile;
+    const bin = o.systemProfilerBin || "/usr/sbin/system_profiler";
+    const timeoutMs = Number.isFinite(o.spTimeoutMs) ? o.spTimeoutMs : 3000;
+    if (o.execFile || existsSync(bin)) {
+      const out = await execFileSafe(execFileImpl, bin, ["SPHardwareDataType"], timeoutMs);
+      if (out) Object.assign(inv, parseSystemProfilerHardware(out));
+    }
+  }
+
+  if (cachePath && Object.keys(inv).length > 0) {
+    try {
+      mkdirSync(join(resolve(o.agentRoot), "state", "telemetry"), { recursive: true });
+      writeFileSync(cachePath, JSON.stringify({ key, inv }, null, 2));
+    } catch { /* cache is an optimisation; never fatal */ }
+  }
+  return inv;
+}
+
+// ---------------------------------------------------------------------------
+// spend24h — the agent's own rolling LLM spend
+// ---------------------------------------------------------------------------
+
+/**
+ * Sum the agent's BILLABLE spend over the trailing 24 hours from the cost
+ * ledger (`state/cost-tracking/<UTC date>.jsonl`).
+ *
+ * Reads today's and yesterday's files only — a 24h window can never span three
+ * UTC dates — and filters by each row's own `ts`, so the number is a true
+ * rolling window rather than "today so far".
+ *
+ * PRICING IS NOT RE-IMPLEMENTED HERE. `lib/cost/ledger-row.mjs#billableUsd` is
+ * the one definition of what a row cost and whether we actually know; it is
+ * imported rather than approximated, because the two ledger row shapes (v1
+ * tracker rows, v2 model-router rows) and the cache-tier pricing hole are
+ * exactly the things a second summing loop would get wrong again. A row whose
+ * cost is UNKNOWN (`usd === null`) contributes nothing and is not counted as
+ * zero — an unmeasured session is not a free one.
+ *
+ * Returns `undefined` (field drops out) when there is no ledger to read, which
+ * is honestly different from `0` ("the ledger says nothing was spent").
+ *
+ * @returns {number|undefined} USD, 2dp
+ */
+export function collectSpend24h(o = {}) {
+  const agentRoot = o.agentRoot;
+  if (!agentRoot) return undefined;
+  const dir = o.costDir || join(resolve(agentRoot), "state", "cost-tracking");
+  let existsDir = false;
+  try { existsDir = existsSync(dir); } catch { existsDir = false; }
+  if (!existsDir) return undefined;
+
+  const nowMs = nowMsFrom(o.now);
+  const cutoff = nowMs - 24 * 60 * 60 * 1000;
+  const day = (ms) => new Date(ms).toISOString().slice(0, 10);
+  const files = [...new Set([day(cutoff), day(nowMs)])];
+
+  let total = 0;
+  let sawAny = false;
+  for (const name of files) {
+    let body = "";
+    try {
+      const p = join(dir, `${name}.jsonl`);
+      if (!existsSync(p)) continue;
+      body = readFileSync(p, "utf8");
+    } catch { continue; }
+    sawAny = true;
+    for (const line of body.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      let row;
+      try { row = JSON.parse(trimmed); } catch { continue; }
+      const ts = Date.parse(row && row.ts);
+      if (!Number.isFinite(ts) || ts < cutoff || ts > nowMs) continue;
+      let billed;
+      try { billed = billableUsd(row); } catch { continue; }
+      // `usd === null` means UNKNOWN, not free. Skipping it under-reports on
+      // purpose rather than inventing a zero — the same call budget-guard makes.
+      if (billed && Number.isFinite(billed.usd)) total += billed.usd;
+    }
+  }
+  if (!sawAny) return undefined;
+  return Math.round(total * 100) / 100;
+}
+
 // ---------------------------------------------------------------------------
 // claudeAuth detection
 // ---------------------------------------------------------------------------
@@ -499,9 +834,14 @@ export const _internals = {
   humanActivity,
   collectMachine,
   collectPowermetrics,
+  powermetricsSamplers,
   parsePowermetrics,
   collectDisk,
   parseDf,
+  collectInventory,
+  parseSystemProfilerHardware,
+  ramLabel,
+  collectSpend24h,
   detectClaudeAuth,
   scanForRelogin,
   countSubAgents,