@cohortapp/agent-sdk 2.7.0 → 2.8.1

package/lib/telemetry/collect.mjs

@@ -19,9 +19,11 @@
  *       memUsedPct: number,             // 1 - free/total
  *       cpuPct?: number,                // powermetrics if root, else loadavg/cores
  *       cpuSource?: "powermetrics"|"sudo-powermetrics"|"loadavg",
- *       tempC?: number,                 // best-effort (powermetrics; needs root)
- *       tempSource?: "powermetrics"|"sudo-powermetrics",
- *       tempDetail?: string,            // why there is no tempC (never silent)
+ *       tempC?: number,                 // real die °C — unprivileged, via IOHIDEventSystem
+ *       tempSource?: "iohid"|"powermetrics"|"sudo-powermetrics",
+ *       thermalState?: "nominal"|"fair"|"serious"|"critical",  // coarse; ALWAYS available on darwin
+ *       thermalSource?: "processinfo"|"sudo-powermetrics"|"pmset",
+ *       tempDetail?: string,            // NOTE ONLY — never the value (see collectThermal)
  *       diskUsedPct?: number,
  *       uptimeDays?: number,
  *       spend24h?: number,              // rolling 24h billable USD off the cost ledger
@@ -60,6 +62,7 @@ import os from "node:os";
 import { execFile } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 
 import { list as listPresence } from "../collective/presence.mjs";
 import { billableUsd } from "../cost/ledger-row.mjs";
@@ -233,10 +236,52 @@ function humanActivity(intent, taskClass) {
  *
  * @returns {Promise<{
  *   loadavg:number[], memUsedPct:number, cpuPct?:number, cpuSource?:string,
- *   tempC?:number, tempSource?:string, tempDetail?:string, diskUsedPct?:number,
+ *   tempC?:number, tempSource?:string, thermalState?:string, thermalSource?:string,
+ *   tempDetail?:string, diskUsedPct?:number,
  *   uptimeDays?:number, spend24h?:number, device?:string, ram?:string, cores?:number
  * }>}
  */
+/**
+ * Percentage of memory genuinely unavailable.
+ *
+ * On macOS this is `100 - (memory_pressure's System-wide memory free
+ * percentage)`. That is the same source `scripts/watchdog/memory-watchdog.sh`
+ * thresholds on, so the number in the Fleet view and the number the watchdog
+ * acts on can no longer disagree — they used to disagree by ~66 points, which
+ * is how a healthy machine came to show a standing CRITICAL.
+ *
+ * Anywhere else (or if memory_pressure yields nothing) falls back to
+ * `1 - free/total`, which on Linux does mean what it says.
+ *
+ * @param {object} o    collector options; `o.execFile` is the test seam
+ * @param {object} osImpl
+ * @returns {Promise<number>}
+ */
+async function collectMemUsedPct(o, osImpl) {
+  const fallback = () => {
+    try {
+      const total = Number(osImpl.totalmem());
+      const free = Number(osImpl.freemem());
+      if (Number.isFinite(total) && total > 0 && Number.isFinite(free)) {
+        return pct((1 - free / total) * 100);
+      }
+    } catch { /* fall through */ }
+    return 0;
+  };
+
+  const platform = o.platform || process.platform;
+  if (platform !== "darwin" && !o.execFile) return fallback();
+
+  const execFileImpl = o.execFile || execFile;
+  const out = await execFileSafe(execFileImpl, "/usr/bin/memory_pressure", [], 4000);
+  const m = /System-wide memory free percentage:\s*([\d.]+)%/.exec(out || "");
+  if (!m) return fallback();
+
+  const freePct = Number(m[1]);
+  if (!Number.isFinite(freePct) || freePct < 0 || freePct > 100) return fallback();
+  return pct(100 - freePct);
+}
+
 export async function collectMachine(o = {}) {
   const osImpl = o.os || os;
   const machine = { loadavg: [0, 0, 0], memUsedPct: 0 };
@@ -249,14 +294,21 @@ export async function collectMachine(o = {}) {
     }
   } catch { /* keep [0,0,0] */ }
 
-  // memUsedPct = 1 - free/total
-  try {
-    const total = Number(osImpl.totalmem());
-    const free = Number(osImpl.freemem());
-    if (Number.isFinite(total) && total > 0 && Number.isFinite(free)) {
-      machine.memUsedPct = pct((1 - free / total) * 100);
-    }
-  } catch { /* keep 0 */ }
+  // memUsedPct — how much memory is genuinely UNAVAILABLE, not how much is
+  // merely occupied.
+  //
+  // This used to be `1 - os.freemem()/os.totalmem()`, which is not a pressure
+  // measure on macOS. os.freemem() counts only entirely-free pages, and macOS
+  // deliberately keeps almost all RAM occupied with file cache, purgeable and
+  // compressed pages that are reclaimable on demand. Measured on a healthy
+  // idle Mac mini M4 Pro at the same instant: os.freemem() → 99.2% "used",
+  // `memory_pressure` → 68% free. The old number sat permanently near 100 and
+  // fired CRITICAL alerts on machines with nothing wrong with them, while the
+  // memory watchdog — which reads memory_pressure — correctly stayed quiet.
+  //
+  // Prefer macOS's own answer; fall back to the free/total ratio only where
+  // memory_pressure does not exist (Linux, where the figure means what it says).
+  machine.memUsedPct = await collectMemUsedPct(o, osImpl);
 
   // cpuPct — coarse instantaneous estimate from loadavg/cpus when powermetrics
   // is unavailable. powermetrics (if it yields) overrides below.
@@ -279,15 +331,37 @@ export async function collectMachine(o = {}) {
     if (Number.isFinite(secs) && secs >= 0) machine.uptimeDays = round1(secs / 86400);
   } catch { /* no uptimeDays */ }
 
-  // Best-effort temp + cpu via powermetrics (guarded, injected, never blocks).
-  // ALWAYS records why it has no answer — see collectPowermetrics.
+  // ── THERMAL ────────────────────────────────────────────────────────────────
+  // Real °C where the hardware gives it up unprivileged, a coarse state word
+  // everywhere else, and `tempDetail` as an EXPLANATORY NOTE that rides
+  // ALONGSIDE the reading rather than standing in for it. See collectThermal
+  // for why this no longer runs through powermetrics.
+  // `powermetrics: false` has always been the caller's "fork no probes" switch
+  // (it is how every hermetic test disables the exec seam), so it still governs
+  // thermal unless a caller opts in or out of thermal explicitly.
+  const thermalEnabled = o.thermal !== undefined ? o.thermal !== false : o.powermetrics !== false;
+  if (thermalEnabled) {
+    const th = await collectThermal(o);
+    if (Number.isFinite(th.tempC)) {
+      machine.tempC = round1(th.tempC);
+      if (th.tempSource) machine.tempSource = th.tempSource;
+    }
+    if (th.thermalState) {
+      machine.thermalState = th.thermalState;
+      if (th.thermalSource) machine.thermalSource = th.thermalSource;
+    }
+    if (th.detail) machine.tempDetail = th.detail;
+  }
+
+  // powermetrics is now asked for CPU utilisation only — plus `tempC` on Intel,
+  // where the `smc` sampler genuinely carries a die temperature and the IOHID
+  // path above does not apply. On Apple Silicon it has no temperature to give
+  // at ANY privilege level, so it never gets to answer the thermal question.
   if (o.powermetrics !== false) {
     const pm = await collectPowermetrics(o);
-    if (Number.isFinite(pm.tempC)) {
+    if (machine.tempC === undefined && Number.isFinite(pm.tempC)) {
       machine.tempC = round1(pm.tempC);
       machine.tempSource = pm.source || "powermetrics";
-    } else if (pm.detail) {
-      machine.tempDetail = pm.detail;
     }
     if (Number.isFinite(pm.cpuPct)) {
       machine.cpuPct = pct(pm.cpuPct);
@@ -320,6 +394,212 @@ export async function collectMachine(o = {}) {
   return machine;
 }
 
+// ---------------------------------------------------------------------------
+// Thermal — real °C without root, and a coarse state word that never fails
+// ---------------------------------------------------------------------------
+
+/** The four thermal pressure levels macOS reports, coldest → hottest. */
+export const THERMAL_STATES = Object.freeze(["nominal", "fair", "serious", "critical"]);
+
+/** Normalise any casing/spelling of a pressure level to our enum, or undefined. */
+function normalizeThermalState(raw) {
+  const v = String(raw || "").trim().toLowerCase();
+  return THERMAL_STATES.includes(v) ? v : undefined;
+}
+
+/**
+ * Pure parser for `thermal-probe.swift` stdout → { thermalState?, tempC?, sensors? }.
+ * Format is `key=value` lines; unknown keys are ignored so the probe can grow.
+ */
+export function parseThermalProbe(text) {
+  const out = {};
+  for (const line of String(text || "").split("\n")) {
+    const m = line.match(/^\s*(\w+)\s*=\s*(.+?)\s*$/);
+    if (!m) continue;
+    const [, k, v] = m;
+    if (k === "thermalState") {
+      const st = normalizeThermalState(v);
+      if (st) out.thermalState = st;
+    } else if (k === "tempC") {
+      const n = Number(v);
+      // Band-check AGAIN on this side. The probe filters, but a collector that
+      // trusts a subprocess's arithmetic is how "-9201.14 °C" reaches a dashboard.
+      if (Number.isFinite(n) && n > 0 && n < 150) out.tempC = n;
+    } else if (k === "tempSensors") {
+      const n = Number(v);
+      if (Number.isFinite(n)) out.sensors = n;
+    }
+  }
+  return out;
+}
+
+/**
+ * Pure parser for the `**** Thermal pressure ****` block that `powermetrics
+ * --samplers thermal` prints — the ONE thermal fact that binary has on Apple
+ * Silicon. Root-only, so this is a fallback, not the primary path.
+ */
+export function parseThermalPressure(text) {
+  const m = String(text || "").match(/Current pressure level:\s*(\w+)/i);
+  return m ? normalizeThermalState(m[1]) : undefined;
+}
+
+/**
+ * Pure parser for `pmset -g therm`. Unprivileged and always present, but it
+ * reports a HISTORY of recorded warnings, not a live reading: on a healthy
+ * machine every line is "No … has been recorded". That absence is a weak signal
+ * and this returns `undefined` for it rather than inventing "nominal" — see
+ * collectThermal, which is where the inference is made explicit and labelled.
+ */
+export function parsePmsetTherm(text) {
+  const s = String(text || "");
+  const m = s.match(/CPU_Scheduler_Limit\s*=\s*(\d+)/i) || s.match(/CPU_Speed_Limit\s*=\s*(\d+)/i);
+  if (m) {
+    const limit = Number(m[1]);
+    // 100 = no throttling. Anything below it means the SMC is actively pulling
+    // the CPU back, which is a REAL thermal event, not a recorded memory of one.
+    if (Number.isFinite(limit)) {
+      if (limit >= 100) return "nominal";
+      if (limit >= 75) return "fair";
+      if (limit >= 50) return "serious";
+      return "critical";
+    }
+  }
+  if (/No thermal warning level has been recorded/i.test(s)) return "no-events";
+  return undefined;
+}
+
+/**
+ * Collect the machine's thermal signal — REAL °C when the hardware will give it
+ * up unprivileged, a coarse state word when it will not.
+ *
+ * ── THE ADVICE THIS REPLACES WAS FUTILE ─────────────────────────────────────
+ * This used to emit, on every Apple-Silicon seat: "powermetrics requires root
+ * (agent runs unprivileged); allowlist it NOPASSWD in sudoers and set
+ * TELEMETRY_SUDO_POWERMETRICS=1 to enable". Both halves of that were wrong.
+ *
+ *   - `powermetrics` produces NO die temperature on Apple Silicon at any
+ *     privilege level. Verified on Mac16,11 / macOS 24D70: `sudo powermetrics
+ *     -A` (every sampler, 242 lines) contains no `temperature` line at all. An
+ *     operator who did exactly what the message asked would STILL see a dash.
+ *   - °C does not need root. It needs IOHIDEventSystem, which any uid can read.
+ *
+ * ── THE LADDER ──────────────────────────────────────────────────────────────
+ *   1. `swift thermal-probe.swift` — unprivileged. Yields BOTH a real `tempC`
+ *      (private IOHID sensors) and `thermalState` (public ProcessInfo). This is
+ *      the answer on any Mac with the Xcode command-line tools present.
+ *   2. `sudo -n powermetrics --samplers thermal` — ONLY if the operator opted
+ *      in. No °C, but a true `Current pressure level:`. State-only.
+ *   3. `pmset -g therm` — unprivileged, always present, but only reports a
+ *      live level while the SMC is actually throttling.
+ *
+ * Every rung records WHY it has no better answer in `detail`. Crucially
+ * `detail` is now a NOTE that ships ALONGSIDE a reading, never instead of one:
+ * the old code made them mutually exclusive, so hq had nothing to print in the
+ * value slot and rendered the 118-character sudoers sentence there instead.
+ *
+ * @returns {Promise<{ tempC?:number, tempSource?:string, thermalState?:string,
+ *                     thermalSource?:string, detail?:string }>}
+ */
+export async function collectThermal(o = {}) {
+  const platform = o.platform || (o.os && o.os.platform && o.os.platform()) || process.platform;
+  const execFileImpl = o.execFile || execFile;
+  if (platform !== "darwin" && !o.execFile) {
+    return { detail: `no thermal probe on platform ${platform}` };
+  }
+  const timeoutMs = Number.isFinite(o.thermalTimeoutMs) ? o.thermalTimeoutMs : 4000;
+
+  // ── 1. The unprivileged probe: real °C + public thermal state ─────────────
+  const swiftBin = o.swiftBin || "/usr/bin/swift";
+  const probePath = o.thermalProbePath || defaultThermalProbePath();
+  const haveSwift = Boolean(o.execFile) || (existsSync(swiftBin) && existsSync(probePath));
+  if (haveSwift) {
+    const out = await execFileSafe(execFileImpl, swiftBin, [probePath], timeoutMs);
+    const parsed = parseThermalProbe(out);
+    if (Number.isFinite(parsed.tempC)) {
+      return {
+        tempC: parsed.tempC,
+        tempSource: "iohid",
+        thermalState: parsed.thermalState,
+        thermalSource: parsed.thermalState ? "processinfo" : undefined,
+        detail: parsed.sensors
+          ? `die temperature: hottest of ${parsed.sensors} on-SoC sensors (IOHIDEventSystem, unprivileged)`
+          : "die temperature via IOHIDEventSystem (unprivileged)",
+      };
+    }
+    if (parsed.thermalState) {
+      return {
+        thermalState: parsed.thermalState,
+        thermalSource: "processinfo",
+        detail:
+          "no numeric °C: the IOHID temperature sensors returned nothing on this hardware; " +
+          "showing the OS thermal pressure level instead",
+      };
+    }
+  }
+
+  // ── 2. Opt-in sudo powermetrics — pressure level only, never °C ───────────
+  const sudoEnabled = o.sudoPowermetrics !== undefined
+    ? Boolean(o.sudoPowermetrics)
+    : String(process.env.TELEMETRY_SUDO_POWERMETRICS || "") === "1";
+  if (sudoEnabled) {
+    const sudoBin = o.sudoBin || "/usr/bin/sudo";
+    const pmBin = o.powermetricsBin || "/usr/bin/powermetrics";
+    const arch = o.arch || (o.os && o.os.arch && o.os.arch()) || process.arch;
+    const out = await execFileSafe(
+      execFileImpl, sudoBin,
+      ["-n", pmBin, "-n", "1", "-i", "200", "--samplers", powermetricsSamplers(arch)],
+      timeoutMs
+    );
+    const level = parseThermalPressure(out);
+    if (level) {
+      return {
+        thermalState: level,
+        thermalSource: "sudo-powermetrics",
+        detail:
+          "no numeric °C: powermetrics reports thermal PRESSURE only — Apple Silicon " +
+          "exposes no die temperature to it even as root",
+      };
+    }
+  }
+
+  // ── 3. pmset — unprivileged, but only speaks up while actually throttling ──
+  const pmsetBin = o.pmsetBin || "/usr/bin/pmset";
+  const out = await execFileSafe(execFileImpl, pmsetBin, ["-g", "therm"], timeoutMs);
+  const therm = parsePmsetTherm(out);
+  if (therm && therm !== "no-events") {
+    return {
+      thermalState: therm,
+      thermalSource: "pmset",
+      detail: "derived from the SMC CPU speed limit reported by `pmset -g therm`",
+    };
+  }
+  if (therm === "no-events") {
+    // NOT reported as `nominal`. "macOS has never recorded a thermal warning"
+    // is a statement about history, and dressing it up as a live reading is
+    // exactly the kind of invented number this rewrite exists to remove.
+    return {
+      detail:
+        "no thermal reading: install the Xcode command-line tools (`xcode-select --install`) " +
+        "to enable the unprivileged °C probe. `pmset` reports no thermal events on record.",
+    };
+  }
+
+  return {
+    detail:
+      "no thermal reading available: the unprivileged °C probe needs `/usr/bin/swift` " +
+      "(Xcode command-line tools) and no fallback thermal source responded",
+  };
+}
+
+/** Absolute path to the shipped swift probe, resolved next to this module. */
+function defaultThermalProbePath() {
+  try {
+    return fileURLToPath(new URL("./thermal-probe.swift", import.meta.url));
+  } catch {
+    return "";
+  }
+}
+
 /**
  * The `powermetrics` sampler that carries die temperature, BY CPU ARCHITECTURE.
  *
@@ -352,16 +632,20 @@ export function powermetricsSamplers(arch) {
  * exit now carries a `detail` sentence that the snapshot ships and the drawer
  * prints.
  *
- * ── ROOT, AND THE ONE HONEST WAY AROUND IT ───────────────────────────────────
- * `powermetrics` requires superuser on macOS, full stop; the daemon runs from a
- * user LaunchAgent. There is NO non-root numeric temperature source on Apple
- * Silicon without a compiled IOKit helper (`ioreg` exposes the PMU/NVMe sensors
- * as HID services whose values are only readable through IOHIDEventSystem, not
- * through the registry), so this does not pretend to have one. Instead it
- * supports an OPT-IN sudo path: set `TELEMETRY_SUDO_POWERMETRICS=1` once the
- * operator has allowlisted the binary NOPASSWD in sudoers. `sudo -n` never
- * prompts — with no allowlist it fails instantly rather than hanging a beat on
- * a password prompt no one will ever see.
+ * ── THIS IS NO LONGER THE TEMPERATURE PATH ───────────────────────────────────
+ * It used to be, and the comment here used to assert that no unprivileged
+ * numeric temperature source existed on Apple Silicon. That was wrong twice
+ * over: powermetrics has no die temperature to give on Apple Silicon EVEN AS
+ * ROOT (`sudo powermetrics -A` contains no `temperature` line), and °C is
+ * readable by any uid through IOHIDEventSystem. `collectThermal` owns that now;
+ * this function is kept for CPU utilisation, for the Intel `smc` sampler that
+ * genuinely does carry a temperature, and for the thermal-pressure line.
+ *
+ * `powermetrics` still requires superuser, and the daemon still runs from a user
+ * LaunchAgent, so the OPT-IN sudo path remains: set
+ * `TELEMETRY_SUDO_POWERMETRICS=1` once the operator has allowlisted the binary
+ * NOPASSWD in sudoers. `sudo -n` never prompts — with no allowlist it fails
+ * instantly rather than hanging a beat on a password prompt no one will see.
  *
  * @returns {Promise<{ tempC?:number, cpuPct?:number, source?:string, detail?:string }>}
  */
@@ -407,14 +691,12 @@ export async function collectPowermetrics(o = {}) {
       }
     }
     return {
-      detail: `${bin} yielded nothing under 'sudo -n' — check the NOPASSWD sudoers allowlist`,
+      detail: `${bin} ran under 'sudo -n' but reported no CPU utilisation this platform can parse`,
     };
   }
 
   return {
-    detail:
-      `${bin} requires root (agent runs unprivileged); ` +
-      "allowlist it NOPASSWD in sudoers and set TELEMETRY_SUDO_POWERMETRICS=1 to enable",
+    detail: `${bin} requires root (agent runs unprivileged); CPU% falls back to the loadavg estimate`,
   };
 }
 
@@ -425,10 +707,33 @@ export function parsePowermetrics(text) {
   // Temperature: "CPU die temperature: 54.21 C" or "GPU die temperature: ..."
   const temp = s.match(/CPU die temperature:\s*([\d.]+)\s*C/i) || s.match(/die temperature:\s*([\d.]+)\s*C/i);
   if (temp) { const v = Number(temp[1]); if (Number.isFinite(v)) out.tempC = v; }
-  // Active CPU%: "CPU Average frequency as fraction of nominal: ..." isn't util;
-  // prefer an explicit "... active residency" total or a "CPU usage" line.
-  const idle = s.match(/CPU\s+(?:Average\s+)?idle\s+residency:\s*([\d.]+)\s*%/i);
-  if (idle) { const v = Number(idle[1]); if (Number.isFinite(v)) out.cpuPct = Math.max(0, 100 - v); }
+  // Thermal pressure — on Apple Silicon this is the ONLY thermal fact in the
+  // whole of powermetrics' output, at any privilege level.
+  const pressure = parseThermalPressure(s);
+  if (pressure) out.pressure = pressure;
+  // Active CPU% = 100 - idle residency.
+  //
+  // ── THE arm64 SHAPE, WHICH NEVER MATCHED ──────────────────────────────────
+  // This only ever looked for a bare `CPU idle residency:` line. Apple Silicon
+  // does not print one: it reports PER-CLUSTER (`E-Cluster idle residency:
+  // 15.69%`, `P-Cluster …`) and per-core (`CPU 0 idle residency: 46.81%`)
+  // lines and no total. `grep -cE "^CPU\s+(Average\s+)?idle residency"` on real
+  // M4 Pro output returns 0, so with sudo enabled this fell through to a detail
+  // string blaming the operator's sudoers file when sudo had worked perfectly.
+  // Fall back to averaging the per-CORE lines, which is the true whole-CPU idle.
+  const idle = s.match(/^CPU\s+(?:Average\s+)?idle\s+residency:\s*([\d.]+)\s*%/im);
+  if (idle) {
+    const v = Number(idle[1]);
+    if (Number.isFinite(v)) out.cpuPct = Math.max(0, 100 - v);
+  } else {
+    const perCore = [...s.matchAll(/^CPU\s+\d+\s+idle\s+residency:\s*([\d.]+)\s*%/gim)]
+      .map((m) => Number(m[1]))
+      .filter((n) => Number.isFinite(n));
+    if (perCore.length > 0) {
+      const avgIdle = perCore.reduce((a, b) => a + b, 0) / perCore.length;
+      out.cpuPct = Math.max(0, 100 - avgIdle);
+    }
+  }
   return out;
 }
 

package/lib/telemetry/collect.test.mjs

@@ -25,6 +25,9 @@ import {
   parseDf,
   collectMachine,
   collectPowermetrics,
+  collectThermal,
+  parseThermalProbe,
+  parseThermalPressure,
   powermetricsSamplers,
   parseSystemProfilerHardware,
   ramLabel,
@@ -170,7 +173,11 @@ test("detectClaudeAuth: scans recent log files under agentRoot", () => {
 });
 
 test("collectMachine: derives memUsedPct + loadavg without powermetrics/df", async () => {
-  const m = await collectMachine({ os: fakeOs({ totalmem: 10e9, freemem: 2e9, loadavg: [4, 2, 1], cpus: 4 }), powermetrics: false, disk: false });
+  // memUsedPct now prefers macOS's memory_pressure, so "no external tools" has
+  // to include that one too — otherwise this test reads the REAL machine and
+  // asserts against the fake os, which is how it started failing.
+  const noTools = (cmd, args, opts, cb) => { cb(new Error("ENOENT")); return { on() {} }; };
+  const m = await collectMachine({ os: fakeOs({ totalmem: 10e9, freemem: 2e9, loadavg: [4, 2, 1], cpus: 4 }), execFile: noTools, powermetrics: false, disk: false });
   assert.equal(m.memUsedPct, 80);
   assert.deepEqual(m.loadavg, [4, 2, 1]);
   assert.equal(m.cpuPct, 100); // load 4 / 4 cpus = 100%
@@ -319,21 +326,25 @@ test("collectPowermetrics passes the arch-correct sampler to the binary", async
   assert.ok(!samplerArg.includes("smc"), "must not send the Intel-only sampler to arm64");
 });
 
-test("collectPowermetrics ALWAYS says why it has no temperature (fail-open, never silent)", async () => {
+test("collectPowermetrics ALWAYS says why it has no answer (fail-open, never silent)", async () => {
   // The unprivileged reality on every seat: the probe yields nothing.
   const pm = await collectPowermetrics({
     execFile: fakeExecFile({}), platform: "darwin", arch: "arm64", sudoPowermetrics: false,
   });
   assert.equal(pm.tempC, undefined);
   assert.match(pm.detail, /requires root/);
-  assert.match(pm.detail, /TELEMETRY_SUDO_POWERMETRICS/);
+  // It no longer tells the operator to edit sudoers to get a temperature. That
+  // advice was futile: powermetrics has no die temperature on Apple Silicon at
+  // ANY privilege level, so following it to the letter still yielded nothing.
+  assert.ok(!/TELEMETRY_SUDO_POWERMETRICS/.test(pm.detail));
+  assert.ok(!/sudoers/.test(pm.detail));
 
   // A non-darwin host is a DIFFERENT absence and says so.
   const linux = await collectPowermetrics({ platform: "linux" });
   assert.match(linux.detail, /platform linux/);
 });
 
-test("collectMachine surfaces the temperature failure as machine.tempDetail", async () => {
+test("collectMachine surfaces a thermal failure as a NOTE, never as the value", async () => {
   const m = await collectMachine({
     os: fakeOs({}),
     execFile: fakeExecFile({}),
@@ -346,7 +357,11 @@ test("collectMachine surfaces the temperature failure as machine.tempDetail", as
   });
   assert.equal(m.tempC, undefined);
   assert.equal(m.tempSource, undefined);
-  assert.match(m.tempDetail, /requires root/);
+  assert.equal(m.thermalState, undefined);
+  assert.equal(typeof m.tempDetail, "string");
+  // The note must not blame the operator's sudoers file for a limitation of the
+  // hardware — that is the message this whole rewrite exists to delete.
+  assert.ok(!/sudoers/.test(m.tempDetail));
   // And the cpuPct it DOES have is labelled as the estimate it is.
   assert.equal(m.cpuSource, "loadavg");
 });
@@ -380,6 +395,191 @@ test("collectPowermetrics tries the opt-in sudo path only when enabled", async (
   assert.ok(sudoCall >= 0);
 });
 
+// ---------------------------------------------------------------------------
+// Thermal — real °C without root, and a coarse state word that never fails.
+//
+// Every test here stubs the exec seam. NOTHING in this file forks `swift`,
+// `powermetrics` or `pmset`; the probe's real output is pinned as fixture text.
+// ---------------------------------------------------------------------------
+
+/** Verbatim stdout of `swift lib/telemetry/thermal-probe.swift` on a Mac16,11. */
+const PROBE_OK = "thermalState=nominal\ntempC=60.76\ntempSensors=42\n";
+
+/** Verbatim `sudo powermetrics --samplers thermal` — note: NO temperature line. */
+const PM_THERMAL = [
+  "Machine model: Mac16,11",
+  "",
+  "**** Thermal pressure ****",
+  "",
+  "Current pressure level: Nominal",
+  "",
+].join("\n");
+
+/** Route stdout by binary, so a test can say which rung of the ladder answers. */
+const SWIFT = "/usr/bin/swift";
+const SUDO = "/usr/bin/sudo";
+const PMSET = "/usr/bin/pmset";
+
+test("collectThermal reads a REAL °C unprivileged, via the IOHID probe", async () => {
+  // The headline fix. powermetrics cannot produce a die temperature on Apple
+  // Silicon even as root; IOHIDEventSystem hands it over to any uid.
+  const seen = [];
+  const spy = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    seen.push(cmd);
+    queueMicrotask(() => cb(null, cmd === SWIFT ? PROBE_OK : "", ""));
+    return { on() {} };
+  };
+  const th = await collectThermal({ execFile: spy, platform: "darwin", arch: "arm64" });
+  assert.equal(th.tempC, 60.76);
+  assert.equal(th.tempSource, "iohid");
+  assert.equal(th.thermalState, "nominal");
+  assert.equal(th.thermalSource, "processinfo");
+  // No root, no sudo, on the happy path.
+  assert.ok(!seen.includes(SUDO), "the °C path must never need sudo");
+  // The detail is an EXPLANATION riding alongside a real reading, not a
+  // substitute for one — the drawer prints the number and demotes this.
+  assert.equal(typeof th.detail, "string");
+  assert.match(th.detail, /IOHIDEventSystem/);
+});
+
+test("collectThermal falls back to the coarse STATE when there is no °C", async () => {
+  // The probe ran but the sensors gave nothing: state alone is still a useful,
+  // honest answer, and the field semantics say so rather than faking a number.
+  const spy = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    queueMicrotask(() => cb(null, cmd === SWIFT ? "thermalState=serious\n" : "", ""));
+    return { on() {} };
+  };
+  const th = await collectThermal({ execFile: spy, platform: "darwin", arch: "arm64" });
+  assert.equal(th.tempC, undefined);
+  assert.equal(th.thermalState, "serious");
+  assert.equal(th.thermalSource, "processinfo");
+  assert.match(th.detail, /no numeric °C/);
+});
+
+test("collectThermal uses sudo powermetrics for PRESSURE only, and only when opted in", async () => {
+  const seen = [];
+  const spy = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    seen.push(cmd);
+    // swift is absent on this host; sudo powermetrics answers.
+    queueMicrotask(() => cb(null, cmd === SUDO ? PM_THERMAL : "", ""));
+    return { on() {} };
+  };
+  const off = await collectThermal({
+    execFile: spy, platform: "darwin", arch: "arm64", sudoPowermetrics: false,
+  });
+  assert.ok(!seen.includes(SUDO), "must not shell out to sudo unless asked");
+  assert.equal(off.thermalState, undefined);
+
+  seen.length = 0;
+  const on = await collectThermal({
+    execFile: spy, platform: "darwin", arch: "arm64", sudoPowermetrics: true,
+  });
+  assert.ok(seen.includes(SUDO));
+  assert.equal(on.thermalState, "nominal");
+  assert.equal(on.thermalSource, "sudo-powermetrics");
+  // Root buys a pressure LEVEL and nothing more. If this ever starts asserting a
+  // tempC from powermetrics on arm64, the fixture is lying.
+  assert.equal(on.tempC, undefined);
+  assert.match(on.detail, /pressure/i);
+});
+
+test("collectThermal will NOT dress 'no thermal events on record' up as a live reading", async () => {
+  // `pmset -g therm` on a healthy Mac reports history, not temperature. Calling
+  // that "nominal" would be inventing a measurement nobody took.
+  const spy = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    const out = cmd === PMSET ? "Note: No thermal warning level has been recorded\n" : "";
+    queueMicrotask(() => cb(null, out, ""));
+    return { on() {} };
+  };
+  const th = await collectThermal({ execFile: spy, platform: "darwin", arch: "arm64" });
+  assert.equal(th.thermalState, undefined);
+  assert.equal(th.tempC, undefined);
+  assert.match(th.detail, /xcode-select --install/);
+
+  // But a REAL throttle event — a speed limit below 100 — is a live signal.
+  const throttling = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    const out = cmd === PMSET ? "CPU_Speed_Limit \t= 60\n" : "";
+    queueMicrotask(() => cb(null, out, ""));
+    return { on() {} };
+  };
+  const hot = await collectThermal({ execFile: throttling, platform: "darwin", arch: "arm64" });
+  assert.equal(hot.thermalState, "serious");
+  assert.equal(hot.thermalSource, "pmset");
+});
+
+test("parseThermalProbe rejects the sensor that lies (-9201.14 °C)", () => {
+  // `PMU tdev2` returns garbage on real hardware. The probe filters it, and the
+  // collector band-checks AGAIN — trusting a subprocess's arithmetic is how an
+  // absurd number reaches a dashboard.
+  assert.deepEqual(parseThermalProbe("thermalState=nominal\ntempC=-9201.14\n"), {
+    thermalState: "nominal",
+  });
+  assert.equal(parseThermalProbe("tempC=900\n").tempC, undefined);
+  assert.equal(parseThermalProbe("tempC=61.2\n").tempC, 61.2);
+  // Unknown keys are ignored so the probe can grow a field without breaking us.
+  assert.equal(parseThermalProbe("tempC=61.2\nfuture=x\n").tempC, 61.2);
+  assert.deepEqual(parseThermalProbe(""), {});
+  assert.deepEqual(parseThermalProbe(null), {});
+  // Only the four real levels are accepted.
+  assert.equal(parseThermalProbe("thermalState=toasty\n").thermalState, undefined);
+});
+
+test("parseThermalPressure reads the only thermal fact powermetrics has on arm64", () => {
+  assert.equal(parseThermalPressure(PM_THERMAL), "nominal");
+  assert.equal(parseThermalPressure("Current pressure level: Critical"), "critical");
+  assert.equal(parseThermalPressure("no such block"), undefined);
+  assert.equal(parseThermalPressure(""), undefined);
+});
+
+test("parsePowermetrics computes CPU% from the PER-CLUSTER arm64 shape", () => {
+  // The old regex wanted a bare `CPU idle residency:` line. Apple Silicon never
+  // prints one — only per-core and per-cluster lines — so on every seat in the
+  // fleet this silently fell through and then blamed the operator's sudoers.
+  const arm = [
+    "**** Processor usage ****",
+    "E-Cluster idle residency:  78.34%",
+    "CPU 0 idle residency:  90.00%",
+    "CPU 1 idle residency:  80.00%",
+    "CPU 2 idle residency:  70.00%",
+    "CPU 3 idle residency:  60.00%",
+  ].join("\n");
+  const parsed = parsePowermetrics(arm);
+  assert.equal(parsed.cpuPct, 25); // 100 - mean(90,80,70,60)
+  // The Intel/total shape still wins outright where it exists.
+  assert.equal(parsePowermetrics("CPU Average idle residency: 78.0 %").cpuPct, 22);
+});
+
+test("collectMachine ships tempC + thermalState + the note TOGETHER", async () => {
+  // The old collector made the reading and the explanation MUTUALLY EXCLUSIVE
+  // (`else if`), which left hq with nothing to put in the value slot — so it
+  // printed a 118-character sudoers sentence in a column of "62°C" and "41%".
+  const spy = (cmd, args, optsOrCb, maybeCb) => {
+    const cb = typeof optsOrCb === "function" ? optsOrCb : maybeCb;
+    queueMicrotask(() => cb(null, cmd === SWIFT ? PROBE_OK : "", ""));
+    return { on() {} };
+  };
+  const m = await collectMachine({
+    os: fakeOs({}),
+    execFile: spy,
+    platform: "darwin",
+    arch: "arm64",
+    sudoPowermetrics: false,
+    disk: false,
+    inventory: false,
+    spend: false,
+  });
+  assert.equal(m.tempC, 60.8); // round1
+  assert.equal(m.tempSource, "iohid");
+  assert.equal(m.thermalState, "nominal");
+  assert.equal(m.thermalSource, "processinfo");
+  assert.equal(typeof m.tempDetail, "string");
+});
+
 // ---------------------------------------------------------------------------
 // spend24h
 // ---------------------------------------------------------------------------
@@ -466,3 +666,75 @@ test("collectStatus ships the FULL machine vocabulary the Fleet row reads", asyn
     rmSync(root, { recursive: true, force: true });
   }
 });
+
+// ---------------------------------------------------------------------------
+// memUsedPct — pressure, not occupancy
+//
+// This was `1 - os.freemem()/os.totalmem()`, which is not a pressure measure on
+// macOS: os.freemem() counts only entirely-free pages, and macOS keeps nearly
+// all RAM occupied with file cache, purgeable and compressed pages that are
+// reclaimable on demand. Measured on a healthy idle Mac mini M4 Pro at one
+// instant: os.freemem() → 99.2% "used", memory_pressure → 68% free. The seat
+// showed a standing CRITICAL while the watchdog, reading memory_pressure,
+// correctly did nothing.
+// ---------------------------------------------------------------------------
+
+test("memUsedPct comes from memory_pressure, not from free/total", async () => {
+  const seen = [];
+  const execFile = (cmd, args, opts, cb) => {
+    seen.push(cmd);
+    if (cmd === "/usr/bin/memory_pressure") {
+      cb(null, "System-wide memory free percentage: 68%\n");
+    } else {
+      cb(new Error("not stubbed"));
+    }
+    return { on() {} };
+  };
+  // An os whose free/total ratio would say 99% used — the old, wrong answer.
+  const osImpl = {
+    loadavg: () => [1, 1, 1],
+    totalmem: () => 24 * 1024 ** 3,
+    freemem: () => 0.2 * 1024 ** 3,
+    cpus: () => [],
+    uptime: () => 3600,
+  };
+
+  const m = await collectMachine({ execFile, os: osImpl, platform: "darwin" });
+
+  assert.equal(m.memUsedPct, 32, "100 - 68 free, i.e. what the watchdog thresholds on");
+  assert.notEqual(m.memUsedPct, 99, "the free/total ratio must not be the answer on darwin");
+  assert.ok(seen.includes("/usr/bin/memory_pressure"), "memory_pressure is consulted");
+});
+
+test("memUsedPct falls back to free/total when memory_pressure yields nothing", async () => {
+  // Linux, or a macOS where the binary is missing/erroring. There the ratio
+  // does mean what it says, so a fallback is right — silence is not.
+  const execFile = (cmd, args, opts, cb) => { cb(new Error("ENOENT")); return { on() {} }; };
+  const osImpl = {
+    loadavg: () => [0, 0, 0],
+    totalmem: () => 1000,
+    freemem: () => 250,
+    cpus: () => [],
+    uptime: () => 1,
+  };
+
+  const m = await collectMachine({ execFile, os: osImpl, platform: "darwin" });
+  assert.equal(m.memUsedPct, 75, "1 - 250/1000");
+});
+
+test("memUsedPct ignores an out-of-range memory_pressure reading", async () => {
+  const execFile = (cmd, args, opts, cb) => {
+    cb(null, "System-wide memory free percentage: 8000%\n");
+    return { on() {} };
+  };
+  const osImpl = {
+    loadavg: () => [0, 0, 0],
+    totalmem: () => 1000,
+    freemem: () => 400,
+    cpus: () => [],
+    uptime: () => 1,
+  };
+
+  const m = await collectMachine({ execFile, os: osImpl, platform: "darwin" });
+  assert.equal(m.memUsedPct, 60, "a nonsense percentage falls back rather than propagating");
+});

package/lib/telemetry/thermal-probe.swift

@@ -0,0 +1,119 @@
+// thermal-probe.swift — the ONLY unprivileged thermal reading on Apple Silicon.
+//
+// ── WHY THIS FILE EXISTS ────────────────────────────────────────────────────
+// The telemetry collector spent two months telling every operator that
+// `powermetrics` "requires root … allowlist it NOPASSWD in sudoers". That
+// advice was both futile and wrong:
+//
+//   1. `powermetrics` emits NO die temperature on Apple Silicon AT ALL — not
+//      unprivileged, not as root, not with `-A` (every sampler). Verified on a
+//      Mac mini M4 Pro (Mac16,11, macOS 24D70): `sudo powermetrics -A` returns
+//      242 lines and `grep -i temperature` finds nothing. The only thermal fact
+//      in that output is `**** Thermal pressure **** / Current pressure level:
+//      Nominal`. So an operator who followed the advice to the letter would
+//      still get an em-dash.
+//   2. Real °C IS available WITHOUT root — through IOHIDEventSystem, which is
+//      where the SoC's PMU temperature sensors actually publish. `ioreg` shows
+//      those sensors as HID service nodes but carries no values, which is why
+//      the registry looked like a dead end.
+//
+// This probe is invoked as `/usr/bin/swift thermal-probe.swift` and prints two
+// `key=value` lines that `parseThermalProbe()` reads. It requires no root, no
+// entitlement, and triggers no TCC prompt; verified working under `env -i` and
+// detached from any tty, which is the LaunchAgent case the daemon actually runs
+// in.
+//
+// ── THE TWO SIGNALS, AND WHY BOTH ───────────────────────────────────────────
+// `thermalState` comes from ProcessInfo — a PUBLIC, always-present API. It is
+// the floor: it cannot fail, needs no Xcode toolchain beyond running this
+// script, and is a genuinely useful coarse answer (nominal/fair/serious/
+// critical). `tempC` comes from the PRIVATE IOHIDEventSystem symbols; it is the
+// precise answer when it is available. If the private path ever stops working
+// on a future macOS, the state line still prints and the collector degrades to
+// a coarse-but-honest reading rather than to nothing.
+//
+// Output (tempC/tempSensors omitted entirely when the sensor read yields nothing):
+//   thermalState=nominal
+//   tempC=61.24
+//   tempSensors=42
+
+import Foundation
+import IOKit
+
+// ── 1. Thermal state: public API, cannot fail ───────────────────────────────
+let word: String
+switch ProcessInfo.processInfo.thermalState {
+case .nominal: word = "nominal"
+case .fair: word = "fair"
+case .serious: word = "serious"
+case .critical: word = "critical"
+@unknown default: word = "unknown"
+}
+print("thermalState=\(word)")
+
+// ── 2. Die temperature: private IOHIDEventSystem, unprivileged ──────────────
+// Matching dictionary {PrimaryUsagePage: 0xff00, PrimaryUsage: 5} selects the
+// AppleARMPMUTempSensor HID services; kIOHIDEventTypeTemperature = 15 and the
+// float field is (type << 16).
+let kTemperatureEvent: Int32 = 15
+let kHIDPageAppleVendor = 0xff00
+let kHIDUsageAppleVendorTemperatureSensor = 5
+
+guard let iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW) else {
+  exit(0) // state line already printed; that is a complete, honest answer
+}
+
+typealias ClientCreateFn = @convention(c) (CFAllocator?) -> Unmanaged<AnyObject>?
+typealias SetMatchingFn = @convention(c) (AnyObject, CFDictionary) -> Void
+typealias CopyServicesFn = @convention(c) (AnyObject) -> Unmanaged<CFArray>?
+typealias CopyEventFn = @convention(c) (AnyObject, Int32, Int32, Int32) -> Unmanaged<AnyObject>?
+typealias FloatValueFn = @convention(c) (AnyObject, Int32) -> Double
+typealias CopyPropertyFn = @convention(c) (AnyObject, CFString) -> Unmanaged<AnyObject>?
+
+guard
+  let pCreate = dlsym(iokit, "IOHIDEventSystemClientCreate"),
+  let pSetMatching = dlsym(iokit, "IOHIDEventSystemClientSetMatching"),
+  let pCopyServices = dlsym(iokit, "IOHIDEventSystemClientCopyServices"),
+  let pCopyEvent = dlsym(iokit, "IOHIDServiceClientCopyEvent"),
+  let pFloatValue = dlsym(iokit, "IOHIDEventGetFloatValue"),
+  let pCopyProperty = dlsym(iokit, "IOHIDServiceClientCopyProperty")
+else { exit(0) }
+
+let clientCreate = unsafeBitCast(pCreate, to: ClientCreateFn.self)
+let setMatching = unsafeBitCast(pSetMatching, to: SetMatchingFn.self)
+let copyServices = unsafeBitCast(pCopyServices, to: CopyServicesFn.self)
+let copyEvent = unsafeBitCast(pCopyEvent, to: CopyEventFn.self)
+let floatValue = unsafeBitCast(pFloatValue, to: FloatValueFn.self)
+let copyProperty = unsafeBitCast(pCopyProperty, to: CopyPropertyFn.self)
+
+guard let client = clientCreate(kCFAllocatorDefault)?.takeRetainedValue() else { exit(0) }
+setMatching(client, [
+  "PrimaryUsagePage": kHIDPageAppleVendor,
+  "PrimaryUsage": kHIDUsageAppleVendorTemperatureSensor,
+] as CFDictionary)
+guard let services = copyServices(client)?.takeRetainedValue() as? [AnyObject] else { exit(0) }
+
+// SOME SENSORS LIE. `PMU tdev2` reliably returns -9201.14 on this hardware, so
+// every reading is band-checked before it is allowed to influence the answer.
+// Of 70 matching services, 67 return a plausible value.
+var dieTemps: [Double] = []
+var anyTemps: [Double] = []
+for service in services {
+  guard let event = copyEvent(service, kTemperatureEvent, 0, 0)?.takeRetainedValue() else { continue }
+  let value = floatValue(event, kTemperatureEvent << 16)
+  guard value.isFinite, value > 0, value < 150 else { continue }
+  anyTemps.append(value)
+  let name = (copyProperty(service, "Product" as CFString)?.takeRetainedValue() as? String) ?? ""
+  if name.contains("tdie") { dieTemps.append(value) }
+}
+
+// Prefer the SoC die sensors ("PMU tdie<n>"); fall back to whatever plausible
+// sensors exist so a future chip with different naming still reports something.
+let pool = dieTemps.isEmpty ? anyTemps : dieTemps
+guard let hottest = pool.max() else { exit(0) }
+
+// The MAX, not the mean: a fleet dashboard wants the worst core, because that
+// is what throttles. Averaging 42 die sensors hides exactly the spike the
+// operator is looking for.
+print(String(format: "tempC=%.2f", hottest))
+print("tempSensors=\(pool.count)")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cohortapp/agent-sdk",
-  "version": "2.7.0",
+  "version": "2.8.1",
   "description": "Cohort Agent SDK \u2014 autonomous AI colleague runtime. Deploy senior AI colleagues on dedicated Mac minis, wired to the Cohort operating surface.",
   "type": "module",
   "bin": {