@cohortapp/agent-sdk 2.3.0

package/lib/telemetry/collect.mjs

@@ -0,0 +1,512 @@
+/**
+ * lib/telemetry/collect.mjs — build the SHARED STATUS SNAPSHOT for an agent.
+ *
+ * THE GAP this closes: the org mesh presence-beat already proves an agent is
+ * alive, but it carries no picture of WHAT the agent is doing or HOW healthy its
+ * machine is. `collectStatus` assembles the one snapshot the whole fleet agrees
+ * on — emitted by the client on every presence beat, stored by the server, and
+ * read back by `lib/org/activity.mjs` so any agent (or a human) can ask "who is
+ * doing what right now". The snapshot shape is the frozen contract:
+ *
+ *   status = {
+ *     state: "active"|"idle"|"busy"|"error"|"offline",
+ *     activity: string,                 // human-readable ("handling inbox" / "session: <taskClass>" / "idle")
+ *     subAgentsRunning: number,         // running claude child processes (best-effort)
+ *     session: { id?, taskClass?, model? } | null,
+ *     machine: {
+ *       loadavg: [n,n,n],
+ *       memUsedPct: number,             // 1 - free/total
+ *       cpuPct?: number,                // best-effort (powermetrics)
+ *       tempC?: number,                 // best-effort (powermetrics)
+ *       diskUsedPct?: number,
+ *     },
+ *     claudeAuth: "ok"|"relogin_required"|"unknown",
+ *     alerts: [{ id, severity, kind, detail }],
+ *     ts: ISO8601,
+ *   }
+ *
+ * It rides `presence.beat` params as `params.status` — NO protocol change (the
+ * server handler validates beat params and stores the snapshot).
+ *
+ * DESIGN: pure-ish + fully injectable. The real I/O surfaces (node:os, the
+ * session/presence state on disk, `powermetrics`, `df`, the log files) are all
+ * reached through an injectable `deps` bag so tests are hermetic and CI never
+ * forks `powermetrics`. Every gather step is independently fail-open: a broken
+ * probe degrades that one field (best-effort fields drop out; required fields get
+ * a benign default), it NEVER throws and NEVER blocks the beat.
+ *
+ * G16 (zero-LLM-in-the-coordination-loop): pure os reads + file reads + a guarded
+ * `powermetrics`/`df` exec. ZERO LLM calls.
+ *
+ * Constraints (CLAUDE.md): ESM, execFile (not exec), Node built-ins only,
+ * everything injectable.
+ *
+ * @module lib/telemetry/collect
+ */
+
+"use strict";
+
+import os from "node:os";
+import { execFile } from "node:child_process";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+import { list as listPresence } from "../collective/presence.mjs";
+import { liveClaudeStats } from "../resource-governor.mjs";
+
+/** Default temperature probe ceiling — used only as a guard in alerts; here we just report. */
+const VALID_STATES = new Set(["active", "idle", "busy", "error", "offline"]);
+
+// ---------------------------------------------------------------------------
+// Small helpers
+// ---------------------------------------------------------------------------
+
+function nowMsFrom(now) {
+  if (typeof now === "function") { const v = Number(now()); return Number.isFinite(v) ? v : Date.now(); }
+  if (Number.isFinite(now)) return Number(now);
+  return Date.now();
+}
+
+function pct(n) {
+  const v = Number(n);
+  if (!Number.isFinite(v)) return 0;
+  return Math.round(Math.max(0, Math.min(100, v)) * 10) / 10;
+}
+
+function safeReadJson(path) {
+  try { return JSON.parse(readFileSync(path, "utf8")); } catch { return null; }
+}
+
+/**
+ * Run a command via execFile with a hard timeout, resolving to its stdout string
+ * or "" on ANY failure (timeout, missing binary, non-zero exit). Never rejects —
+ * a slow/absent powermetrics must never wedge the beat.
+ */
+function execFileSafe(execFileImpl, cmd, args, timeoutMs) {
+  return new Promise((res) => {
+    let done = false;
+    const finish = (v) => { if (!done) { done = true; res(v || ""); } };
+    try {
+      const child = execFileImpl(cmd, args, { timeout: timeoutMs, maxBuffer: 4 * 1024 * 1024 }, (err, stdout) => {
+        if (err) return finish("");
+        finish(typeof stdout === "string" ? stdout : "");
+      });
+      // execFile's own timeout kills the child; this belt-and-braces guard
+      // resolves us even if the callback never fires.
+      const t = setTimeout(() => finish(""), timeoutMs + 250);
+      if (t && typeof t.unref === "function") t.unref();
+      if (child && typeof child.on === "function") child.on("error", () => finish(""));
+    } catch { finish(""); }
+  });
+}
+
+// ---------------------------------------------------------------------------
+// State + activity (what is the agent doing right now)
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the current session/activity from local daemon + presence state. We
+ * look at, in order:
+ *   1. an explicit `deps.sessions` array (injected; tests)
+ *   2. the live presence registry (state/collective/presence) — daemon + sessions
+ * The "primary" session is the most-recently-beating non-daemon session, else the
+ * daemon record. We derive:
+ *   - session: { id?, taskClass?, model? } | null
+ *   - activity: a human-readable line
+ *   - state: idle | busy | active
+ * Best-effort; never throws.
+ *
+ * @returns {{ session:object|null, activity:string, state:string }}
+ */
+export function resolveActivity(o = {}) {
+  const agentRoot = o.agentRoot;
+  const nowMs = nowMsFrom(o.now);
+  let records = [];
+  try {
+    if (Array.isArray(o.sessions)) records = o.sessions;
+    else if (agentRoot) records = listPresence(agentRoot, { nowMs, reap: false });
+  } catch { records = []; }
+  records = Array.isArray(records) ? records.filter((r) => r && typeof r === "object") : [];
+
+  // Split daemon vs work sessions.
+  const work = records.filter((r) => (r.kind || "interactive") !== "daemon");
+  const daemons = records.filter((r) => (r.kind || "interactive") === "daemon");
+
+  // Primary = newest-beating work session, else the daemon.
+  const byBeat = (a, b) => Date.parse(b.lastBeat || b.startedAt || 0) - Date.parse(a.lastBeat || a.startedAt || 0);
+  const primary = work.slice().sort(byBeat)[0] || daemons.slice().sort(byBeat)[0] || null;
+
+  // Map a presence record's intent → a taskClass label. The daemon writes
+  // intent like "session: <taskClass>" / "<cadence>"; we surface the raw intent
+  // as the activity line and best-effort split a taskClass out of it.
+  let session = null;
+  let activity = "idle";
+  let state = "idle";
+
+  if (work.length > 0 && primary) {
+    state = "busy";
+    const taskClass = deriveTaskClass(primary.intent);
+    session = {
+      id: primary.sessionId || undefined,
+      taskClass: taskClass || undefined,
+      model: primary.model || undefined,
+    };
+    activity = humanActivity(primary.intent, taskClass);
+  } else if (daemons.length > 0) {
+    // Daemon up but no work session running → the agent is alive and idle.
+    state = "idle";
+    activity = "idle";
+    session = null;
+  } else {
+    state = "idle";
+    activity = "idle";
+    session = null;
+  }
+
+  return { session, activity, state };
+}
+
+/** Pull a taskClass out of an intent string ("session: inbox" → "inbox"). */
+function deriveTaskClass(intent) {
+  const s = String(intent || "").trim();
+  if (!s) return "";
+  const m = s.match(/(?:session|task|cadence)\s*[:=]\s*([a-z0-9._-]+)/i);
+  if (m) return m[1];
+  // A bare single-token intent (e.g. "inbox", "morning-brief") is itself the class.
+  if (/^[a-z0-9._-]+$/i.test(s)) return s;
+  return "";
+}
+
+/** Build a human-readable activity line from an intent + derived taskClass. */
+function humanActivity(intent, taskClass) {
+  const s = String(intent || "").trim();
+  if (s) return s.length > 120 ? `${s.slice(0, 117)}...` : s;
+  if (taskClass) return `session: ${taskClass}`;
+  return "working";
+}
+
+// ---------------------------------------------------------------------------
+// Machine stats
+// ---------------------------------------------------------------------------
+
+/**
+ * Gather machine stats from node:os plus best-effort `powermetrics` (temp/cpu)
+ * and `df` (disk). The exec probes are guarded + injected; on macOS powermetrics
+ * needs root so a non-root agent simply gets no temp/cpu (those fields drop out).
+ *
+ * @returns {Promise<{loadavg:number[], memUsedPct:number, cpuPct?:number, tempC?:number, diskUsedPct?:number}>}
+ */
+export async function collectMachine(o = {}) {
+  const osImpl = o.os || os;
+  const machine = { loadavg: [0, 0, 0], memUsedPct: 0 };
+
+  // loadavg
+  try {
+    const la = osImpl.loadavg();
+    if (Array.isArray(la) && la.length >= 3) {
+      machine.loadavg = [round2(la[0]), round2(la[1]), round2(la[2])];
+    }
+  } catch { /* keep [0,0,0] */ }
+
+  // memUsedPct = 1 - free/total
+  try {
+    const total = Number(osImpl.totalmem());
+    const free = Number(osImpl.freemem());
+    if (Number.isFinite(total) && total > 0 && Number.isFinite(free)) {
+      machine.memUsedPct = pct((1 - free / total) * 100);
+    }
+  } catch { /* keep 0 */ }
+
+  // cpuPct — coarse instantaneous estimate from loadavg/cpus when powermetrics
+  // is unavailable. powermetrics (if it yields) overrides below.
+  try {
+    const cpus = osImpl.cpus();
+    const n = Array.isArray(cpus) ? cpus.length : 0;
+    if (n > 0 && Number.isFinite(machine.loadavg[0])) {
+      const est = (machine.loadavg[0] / n) * 100;
+      if (Number.isFinite(est)) machine.cpuPct = pct(est);
+    }
+  } catch { /* no cpuPct */ }
+
+  // Best-effort temp + cpu via powermetrics (guarded, injected, never blocks).
+  if (o.powermetrics !== false) {
+    const pm = await collectPowermetrics(o);
+    if (Number.isFinite(pm.tempC)) machine.tempC = round1(pm.tempC);
+    if (Number.isFinite(pm.cpuPct)) machine.cpuPct = pct(pm.cpuPct);
+  }
+
+  // Best-effort diskUsedPct via df.
+  if (o.disk !== false) {
+    const disk = await collectDisk(o);
+    if (Number.isFinite(disk)) machine.diskUsedPct = pct(disk);
+  }
+
+  return machine;
+}
+
+/**
+ * Parse `powermetrics` for CPU temperature + active CPU%. Guarded: only attempts
+ * the exec on darwin (unless an explicit `deps.execFile` + `deps.platform` say
+ * otherwise for tests). Returns {} on any failure. NEVER blocks past timeoutMs.
+ *
+ * @returns {Promise<{ tempC?:number, cpuPct?:number }>}
+ */
+export async function collectPowermetrics(o = {}) {
+  const platform = o.platform || (o.os && o.os.platform && o.os.platform()) || process.platform;
+  const execFileImpl = o.execFile || execFile;
+  if (platform !== "darwin" && !o.execFile) return {}; // only darwin in the wild; tests inject execFile
+  const timeoutMs = Number.isFinite(o.pmTimeoutMs) ? o.pmTimeoutMs : 1500;
+  const bin = o.powermetricsBin || "/usr/bin/powermetrics";
+  if (!o.execFile && !existsSync(bin)) return {};
+  const out = await execFileSafe(
+    execFileImpl, bin,
+    ["-n", "1", "-i", "200", "--samplers", "smc,cpu_power", "--show-process-energy"],
+    timeoutMs,
+  );
+  if (!out) return {};
+  return parsePowermetrics(out);
+}
+
+/** Pure parser for powermetrics text → { tempC?, cpuPct? }. Tolerant of format drift. */
+export function parsePowermetrics(text) {
+  const out = {};
+  const s = String(text || "");
+  // Temperature: "CPU die temperature: 54.21 C" or "GPU die temperature: ..."
+  const temp = s.match(/CPU die temperature:\s*([\d.]+)\s*C/i) || s.match(/die temperature:\s*([\d.]+)\s*C/i);
+  if (temp) { const v = Number(temp[1]); if (Number.isFinite(v)) out.tempC = v; }
+  // Active CPU%: "CPU Average frequency as fraction of nominal: ..." isn't util;
+  // prefer an explicit "... active residency" total or a "CPU usage" line.
+  const idle = s.match(/CPU\s+(?:Average\s+)?idle\s+residency:\s*([\d.]+)\s*%/i);
+  if (idle) { const v = Number(idle[1]); if (Number.isFinite(v)) out.cpuPct = Math.max(0, 100 - v); }
+  return out;
+}
+
+/** Best-effort diskUsedPct for the agent root's filesystem via `df -k .`. */
+export async function collectDisk(o = {}) {
+  const execFileImpl = o.execFile || execFile;
+  const root = o.agentRoot ? resolve(o.agentRoot) : (process.env.AGENT_ROOT || process.cwd());
+  const timeoutMs = Number.isFinite(o.dfTimeoutMs) ? o.dfTimeoutMs : 1500;
+  const out = await execFileSafe(execFileImpl, "/bin/df", ["-k", root], timeoutMs);
+  if (!out) return undefined;
+  return parseDf(out);
+}
+
+/** Pure parser for `df -k` text → used percent number, or undefined. */
+export function parseDf(text) {
+  const lines = String(text || "").trim().split("\n");
+  if (lines.length < 2) return undefined;
+  // The data row may wrap; scan for a token ending in "%".
+  for (let i = lines.length - 1; i >= 1; i--) {
+    const cols = lines[i].trim().split(/\s+/);
+    const capCol = cols.find((c) => /^\d+%$/.test(c));
+    if (capCol) { const v = Number(capCol.replace("%", "")); if (Number.isFinite(v)) return v; }
+  }
+  return undefined;
+}
+
+function round1(n) { return Math.round(Number(n) * 10) / 10; }
+function round2(n) { return Math.round(Number(n) * 100) / 100; }
+
+// ---------------------------------------------------------------------------
+// claudeAuth detection
+// ---------------------------------------------------------------------------
+
+/** Auth-failure fingerprints we treat as "the agent needs to re-login". */
+const RELOGIN_FINGERPRINTS = [
+  "relogin_required",
+  "please run /login",
+  "please run claude login",
+  "run `claude login`",
+  "invalid api key",
+  "oauth token has expired",
+  "authentication_error",
+  "credentials have expired",
+  "not logged in",
+];
+
+/**
+ * Detect whether Claude auth is healthy by scanning recent log tails for a
+ * re-login/auth fingerprint. Looks at the most recent file in each of a small set
+ * of log dirs (sessions, diagnostics) within a freshness window.
+ *
+ * Returns:
+ *   "relogin_required"  a recent auth/login failure is present
+ *   "ok"                logs exist and show no recent auth failure
+ *   "unknown"           no logs to judge from
+ *
+ * `deps.logText` (string) short-circuits the file scan for tests.
+ *
+ * @returns {"ok"|"relogin_required"|"unknown"}
+ */
+export function detectClaudeAuth(o = {}) {
+  const nowMs = nowMsFrom(o.now);
+  const windowMs = Number.isFinite(o.authWindowMs) ? o.authWindowMs : 6 * 60 * 60 * 1000; // 6h
+
+  // Test / explicit override.
+  if (typeof o.logText === "string") {
+    return scanForRelogin(o.logText) ? "relogin_required" : (o.logText ? "ok" : "unknown");
+  }
+
+  const agentRoot = o.agentRoot;
+  if (!agentRoot) return "unknown";
+
+  const dirs = [
+    join(agentRoot, "logs", "sessions"),
+    join(agentRoot, "logs", "diagnostics"),
+    join(agentRoot, "logs"),
+  ];
+  let sawAny = false;
+  let sawRelogin = false;
+  for (const dir of dirs) {
+    const file = newestFileWithin(dir, nowMs, windowMs);
+    if (!file) continue;
+    sawAny = true;
+    const text = tailFile(file, 64 * 1024);
+    if (scanForRelogin(text)) { sawRelogin = true; break; }
+  }
+  if (sawRelogin) return "relogin_required";
+  return sawAny ? "ok" : "unknown";
+}
+
+function scanForRelogin(text) {
+  const lower = String(text || "").toLowerCase();
+  return RELOGIN_FINGERPRINTS.some((f) => lower.includes(f));
+}
+
+/** Newest *.jsonl/*.log file in dir modified within windowMs of now, else null. */
+function newestFileWithin(dir, nowMs, windowMs) {
+  try {
+    if (!existsSync(dir)) return null;
+    let best = null;
+    for (const name of readdirSync(dir)) {
+      if (!/\.(jsonl|log|txt)$/i.test(name)) continue;
+      const p = join(dir, name);
+      let st;
+      try { st = statSync(p); } catch { continue; }
+      if (!st.isFile()) continue;
+      const age = nowMs - st.mtimeMs;
+      if (!Number.isFinite(age) || age > windowMs) continue;
+      if (!best || st.mtimeMs > best.mtimeMs) best = { p, mtimeMs: st.mtimeMs };
+    }
+    return best ? best.p : null;
+  } catch { return null; }
+}
+
+/** Read the last `bytes` of a file as utf8 (best-effort). */
+function tailFile(path, bytes) {
+  try {
+    const buf = readFileSync(path);
+    if (buf.length <= bytes) return buf.toString("utf8");
+    return buf.slice(buf.length - bytes).toString("utf8");
+  } catch { return ""; }
+}
+
+// ---------------------------------------------------------------------------
+// subAgentsRunning
+// ---------------------------------------------------------------------------
+
+/**
+ * Count running claude child processes (best-effort). Prefers an injected count;
+ * else reuses the resource-governor's `liveClaudeStats()` (one `ps` parse, 2s
+ * cached) so the governor and telemetry agree on "live claude". Fail-open → 0.
+ */
+export function countSubAgents(o = {}) {
+  if (Number.isFinite(o.subAgentsRunning)) return Math.max(0, Math.floor(o.subAgentsRunning));
+  if (typeof o.liveClaudeStats === "function") {
+    try { const v = o.liveClaudeStats(); return Math.max(0, Math.floor(Number(v && v.count) || 0)); }
+    catch { return 0; }
+  }
+  try { const v = liveClaudeStats(); return Math.max(0, Math.floor(Number(v && v.count) || 0)); }
+  catch { return 0; }
+}
+
+// ---------------------------------------------------------------------------
+// collectStatus — the public assembler
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the SHARED STATUS SNAPSHOT. Every step is independently fail-open; the
+ * function never throws and always returns a well-formed snapshot.
+ *
+ * @param {object} o
+ * @param {string} [o.agentRoot]          agent repo root (for state/log reads)
+ * @param {number|Function} [o.now]       clock (ms or () => ms); default Date.now
+ * @param {object} [o.deps]               injectable probes (see below) — also accepted at top level
+ * @param {object[]} [o.sessions]         presence records override (tests)
+ * @param {number} [o.subAgentsRunning]   override the child-process count (tests)
+ * @param {Function} [o.liveClaudeStats]  override the ps-based count (tests)
+ * @param {object}  [o.os]                node:os override (tests)
+ * @param {Function}[o.execFile]          execFile override (tests; gates powermetrics/df)
+ * @param {string}  [o.logText]           log text override for auth detection (tests)
+ * @param {object}  [o.thresholds]        alert thresholds passed to deriveAlerts
+ * @param {boolean} [o.withAlerts=true]   derive + attach alerts inline
+ * @returns {Promise<object>} the status snapshot
+ */
+export async function collectStatus(o = {}) {
+  // Allow callers to pass probes under `deps` or at top level (top level wins).
+  const deps = (o && typeof o.deps === "object" && o.deps) || {};
+  const opt = { ...deps, ...o };
+  const nowMs = nowMsFrom(opt.now);
+
+  // 1. state + activity + session
+  let act = { session: null, activity: "idle", state: "idle" };
+  try { act = resolveActivity(opt); } catch { /* defaults */ }
+
+  // 2. subAgentsRunning
+  let subAgentsRunning = 0;
+  try { subAgentsRunning = countSubAgents(opt); } catch { subAgentsRunning = 0; }
+
+  // 3. machine
+  let machine = { loadavg: [0, 0, 0], memUsedPct: 0 };
+  try { machine = await collectMachine(opt); } catch { machine = { loadavg: [0, 0, 0], memUsedPct: 0 }; }
+
+  // 4. claudeAuth
+  let claudeAuth = "unknown";
+  try { claudeAuth = detectClaudeAuth(opt); } catch { claudeAuth = "unknown"; }
+
+  // Promote state to "busy" if there are sub-agents but resolveActivity said idle
+  // (a daemon spawning work but its own presence record lags). Honour an explicit
+  // override.
+  let state = VALID_STATES.has(opt.state) ? opt.state : act.state;
+  if (claudeAuth === "relogin_required") state = "error";
+  else if (state === "idle" && subAgentsRunning > 0) state = "busy";
+
+  const status = {
+    state,
+    activity: act.activity || "idle",
+    subAgentsRunning,
+    session: act.session || null,
+    machine,
+    claudeAuth,
+    alerts: [],
+    ts: new Date(nowMs).toISOString(),
+  };
+
+  // 5. alerts (inline, unless the caller wants to derive them itself)
+  if (opt.withAlerts !== false) {
+    try {
+      const mod = await import("./alerts.mjs");
+      status.alerts = mod.deriveAlerts(status, opt.thresholds) || [];
+    } catch { status.alerts = []; }
+  }
+
+  return status;
+}
+
+export const _internals = {
+  resolveActivity,
+  deriveTaskClass,
+  humanActivity,
+  collectMachine,
+  collectPowermetrics,
+  parsePowermetrics,
+  collectDisk,
+  parseDf,
+  detectClaudeAuth,
+  scanForRelogin,
+  countSubAgents,
+  execFileSafe,
+  RELOGIN_FINGERPRINTS,
+};
+
+export default { collectStatus, _internals };