agent-dag 1.48.0 → 3.0.0

package/src/server/system-metrics.mjs

@@ -18,7 +18,7 @@
 // readout the loudest producer in the application. So this is a plain poll
 // endpoint, exactly like /api/quota and /api/codex-usage already are.
 import { spawn } from "node:child_process";
-import { readFile } from "node:fs/promises";
+import { readdir, readFile } from "node:fs/promises";
 import os from "node:os";
 
 /** CPU is the metric with spikes, so it is sampled often enough to catch one. */
@@ -41,6 +41,18 @@ let swap = null;
 const cpuHistory = [];
 let memory = null;
 let memInFlight = false;
+let thermal = null;
+let thermalTimer = null;
+let thermalInFlight = false;
+/** Consecutive readings that came back with nothing. See THERMAL_GIVE_UP. */
+let thermalMisses = 0;
+/** Whether this machine has EVER answered. See sampleThermal. */
+let thermalEverAnswered = false;
+/** Minute buckets, oldest first, for every section that keeps a history.
+ *  See HISTORY_MINUTES. */
+const history = [];
+/** When sampling started, so a modal can say what "since" means. */
+let historySince = 0;
 
 /** Total and idle jiffies across every core, as one pair. */
 function readTicks() {
@@ -112,13 +124,14 @@ function cpuPercent() {
 }
 
 /**
- * The locale every child of this module is parsed in.
+ * The locale every child of this module runs in — and only the part of it that
+ * had to be forced.
  *
- * Not a preference — a correctness requirement. Every command spawned here has
- * its output read by a regex, and every one of those regexes reads a number
- * with a `.` in it. `ps` and `sysctl` honour LC_NUMERIC, so on a machine set to
- * de_DE, fr_FR, ru_RU or pt_BR — comma is the decimal separator for most of
- * Europe and Latin America — the same commands print:
+ * Not a preference. Every command spawned here has its output read by a regex
+ * and every one of those regexes reads a number with a `.` in it. `ps` and
+ * `sysctl` honour LC_NUMERIC, so on a machine set to de_DE, fr_FR, ru_RU or
+ * pt_BR — comma is the decimal separator for most of Europe and Latin America —
+ * the same commands print:
  *
  *     ps      1   0,2  0,0 /sbin/launchd
  *     sysctl  total = 8192,00M  used = 7189,75M  free = 1002,25M
@@ -126,20 +139,42 @@ function cpuPercent() {
  * and the parsers matched nothing at all. Not partially: `parsePsProcesses`
  * `continue`s on every row, so the process panel was permanently empty, and
  * `swapFromSysctl` returned null, so the macOS swap meter was permanently
- * blank. Silently, with nothing in the log, on a machine where everything else
- * worked.
+ * blank. Silently, on a machine where everything else worked.
  *
- * Forcing the locale rather than teaching the parsers to read a comma is the
- * fix that scales: it makes the OUTPUT invariant, which is what every parser
- * here was written against and what every parser added later will assume. The
- * comma tolerance below is defence in depth for the case where a sandbox strips
- * the environment, not the primary answer.
+ * THIS USED TO FORCE THE WHOLE LOCALE, and forcing the whole locale also forces
+ * the CHARACTER SET. Under `LC_ALL=C`, `ps` escapes every byte it cannot render
+ * as ASCII, so an application named in Cyrillic came back as
+ *
+ *     M-PM-/M-PM-=M-PM-4M-PM-5M-PM-:M-QM^A M-PM^\M-QM^CM-PM-7M-QM^KM-PM-:M-PM-0
+ *
+ * where the name is `Яндекс Музыка`. The panel truncates at 164px so it read as
+ * noise; the process list added in #738 has room for all ninety-one characters
+ * of it, which is how it was found. Every reader with a non-Latin application
+ * on their machine was being shown that.
+ *
+ * So only the numeric is forced now, and the character set is inherited. All
+ * four measured on this machine:
+ *
+ *     LC_ALL=C                        name mangled,  0.7
+ *     LC_ALL="" LC_NUMERIC=C          Яндекс Музыка, 0.7
+ *     LANG=de_DE + our override       Яндекс Музыка, 0.7   ← the comma case
+ *     LC_ALL=de_DE + our empty LC_ALL                0.7   ← a hostile env
+ *
+ * The empty `LC_ALL` is doing real work in the last of those: POSIX ignores an
+ * empty LC_ALL, so clearing it is what stops a user's own LC_ALL from
+ * outranking the LC_NUMERIC below. Setting LC_NUMERIC alone would not survive
+ * it.
+ *
+ * A reader whose own locale is C still sees the escaped form — but so does
+ * their terminal, so that is their machine being consistent rather than this
+ * module choosing for them.
  *
  * Meaningless on Windows, where the branches are PowerShell piped through
  * ConvertTo-Json and already culture-invariant — and harmless there for the
- * same reason.
+ * same reason. The comma tolerance in the parsers stays as defence in depth for
+ * a sandbox that strips the environment, not as the primary answer.
  */
-const C_LOCALE = { LC_ALL: "C", LANG: "C" };
+const C_LOCALE = { LC_ALL: "", LC_NUMERIC: "C" };
 
 /** Run a command and resolve its stdout, or null. Never rejects, never inherits
  *  a shell, never inherits a locale, and is killed rather than allowed to hang
@@ -299,9 +334,48 @@ async function readSwap(platform = process.platform) {
   return null;
 }
 
-/** How many rows the process list keeps. Enough to see what is eating the
- *  machine, few enough that the panel never becomes a scroll. */
-const TOP_N = 8;
+/**
+ * How far down each ranking the payload reaches.
+ *
+ * The panel draws eight rows and decides their order on the client (#739), so
+ * whichever column it ranks by has to be rankable from the rows it was handed.
+ * `ps` returns a CPU-sorted list, and cutting that at eight and then sorting
+ * those eight by memory produced a table that was honest about its rows and
+ * wrong about its question: the machine's heaviest memory consumer need never
+ * have appeared in a CPU top eight at all. Same shape as #492 — a list that is
+ * never empty, never errors, and is not the rows being asked for.
+ *
+ * So what goes over the wire is a candidate SET rather than a ranking.
+ * pickCandidates takes this many by CPU and this many by memory and sends the
+ * union, which makes the true top eight of either column present by
+ * construction. Wide enough that drawing more rows later cannot quietly
+ * re-break that, small enough that the whole thing stays a few kilobytes of
+ * four-field rows, fetched only while the panel is open.
+ */
+const CANDIDATE_N = 40;
+
+/**
+ * The rows worth sending, given that the ordering happens on the client.
+ *
+ * A union of two rankings, deduplicated by object identity rather than by pid:
+ * parseGetProcessJson falls back to pid 0 for a row whose `Id` did not parse,
+ * more than one row can do that, and a pid-keyed set would drop real processes
+ * in order to deduplicate a placeholder.
+ *
+ * An unknown CPU sorts last here, for the same reason the column prints a dash
+ * rather than a zero — a Windows first reading has no percentage yet, and
+ * unknown is not idle and is not busiest either. Such a row still reaches the
+ * payload through the memory half, which is the reading that does exist on that
+ * pass.
+ */
+export function pickCandidates(rows, limit = CANDIDATE_N) {
+  const byCpu = [...rows].sort((a, b) => (b.cpu ?? -1) - (a.cpu ?? -1) || b.mem - a.mem);
+  const byMem = [...rows].sort((a, b) => b.mem - a.mem || (b.cpu ?? -1) - (a.cpu ?? -1));
+  const out = byCpu.slice(0, limit);
+  const seen = new Set(out);
+  for (const r of byMem.slice(0, limit)) if (!seen.has(r)) out.push(r);
+  return out;
+}
 
 /**
  * The `ps` argument list, which is not the same list on both Unixes.
@@ -349,10 +423,17 @@ export function psArgs(platform = process.platform) {
  *
  * There is no row limit in the query because neither `ps` has one and `run`
  * deliberately never inherits a shell, so there is no `| head` to pipe into.
- * The loop below stops at `limit` instead, which costs one parse of a string
- * we have already paid to read.
+ *
+ * And there is none by default here either, which is a change: the loop used to
+ * stop at eight, and stopping at eight is what made the memory ranking a lie
+ * once the panel could ask for one (#739). Selection belongs to pickCandidates,
+ * which cannot rank a column out of rows this function has already thrown away.
+ * The cost is a regex over every line of `ps` — a few hundred on a busy
+ * machine, once every four seconds and only while the panel is open — against a
+ * string that has already been read and allocated. `limit` stays for callers
+ * that do want a truncation, which is now only the tests.
  */
-export function parsePsProcesses(text, limit = TOP_N) {
+export function parsePsProcesses(text, limit = Infinity) {
   const lines = String(text ?? "").trim().split("\n");
   const out = [];
   for (const line of lines.slice(1)) {           // drop the header row
@@ -430,7 +511,7 @@ export function parseGetProcessJson(json, totalMem) {
  * 0-100 convention (see cpuPercent) is a different question with a different
  * answer, and the only way this drifts back is if a core count is in reach.
  */
-export function cpuFromDeltas(rows, prev, elapsedMs, limit = TOP_N) {
+export function cpuFromDeltas(rows, prev, elapsedMs, limit = Infinity) {
   const secs = elapsedMs / 1000;
   const out = rows.map(r => {
     const before = prev instanceof Map ? prev.get(r.pid) : undefined;
@@ -460,7 +541,7 @@ let prevProcAt = 0;
 
 /** The run producing the next list, and the last one that finished. */
 let procInFlight = null;
-let procLast = null; // { at, procs }
+let procLast = null; // { at, read: { procs, total } }
 
 /**
  * How old a finished reading may be and still answer a caller.
@@ -494,7 +575,7 @@ const PROC_MIN_GAP_MS = 1_500;
  */
 export async function readProcesses(platform = process.platform) {
   const now = Date.now();
-  if (procLast && now - procLast.at < PROC_MIN_GAP_MS) return procLast.procs;
+  if (procLast && now - procLast.at < PROC_MIN_GAP_MS) return procLast.read;
   if (procInFlight) return procInFlight;
   // Only a real reading is remembered. Every failure inside readProcessesNow —
   // a spawn that never started, a non-zero exit, the timeout — resolves to an
@@ -504,9 +585,9 @@ export async function readProcesses(platform = process.platform) {
   // instead. The in-flight share still applies, so a burst arriving during a
   // failing read is one failing child, not a burst of them.
   procInFlight = readProcessesNow(platform)
-    .then(procs => {
-      if (procs.length) procLast = { at: Date.now(), procs };
-      return procs;
+    .then(read => {
+      if (read.procs.length) procLast = { at: Date.now(), read };
+      return read;
     })
     .finally(() => { procInFlight = null; });
   return procInFlight;
@@ -521,13 +602,593 @@ async function readProcessesNow(platform) {
     if (!out) return [];
     const rows = parseGetProcessJson(out.trim(), os.totalmem());
     const now = Date.now();
-    const result = cpuFromDeltas(rows, prevProcCpu, now - prevProcAt);
+    const ranked = cpuFromDeltas(rows, prevProcCpu, now - prevProcAt);
     prevProcCpu = new Map(rows.filter(r => r.cpuSec != null).map(r => [r.pid, r.cpuSec]));
     prevProcAt = now;
-    return result;
+    return { procs: pickCandidates(ranked), total: ranked.length };
   }
   const out = await run("ps", psArgs(platform), 4_000);
-  return out ? parsePsProcesses(out) : [];
+  if (!out) return { procs: [], total: 0 };
+  const all = parsePsProcesses(out);
+  return { procs: pickCandidates(all), total: all.length };
+}
+
+// ---------------------------------------------------------------------------
+// Thermal: is this machine getting hot, and is it being held back for it.
+//
+// The section the load average cannot answer. A saturated machine that is cool
+// is a machine doing work; a saturated machine that is thermally limited is one
+// where the next agent you launch makes everything slower, and `67.27 82.98
+// 74.19` reads identically in both cases.
+//
+// THREE PLATFORMS ANSWER THREE DIFFERENT QUESTIONS, and on one of them the
+// honest answer is not a temperature at all. Everything below was measured on
+// the machines available rather than taken from documentation, and the negative
+// results are recorded here because they are the reason the shape is what it
+// is:
+//
+//   Linux    /sys/class/hwmon/hwmon*/temp*_input, millidegrees Celsius, with
+//            the chip in `name` and the sensor in `temp*_label`. A plain file
+//            read, exactly like /proc/meminfo — no subprocess, and the chip
+//            publishes its own `temp*_max` and `temp*_crit`, so the warning
+//            bands are the hardware's rather than ones invented here.
+//            /sys/class/thermal/thermal_zone*/ is the coarser fallback.
+//
+//   macOS    No CPU degrees without root, verified: `powermetrics --samplers
+//            smc` answers "powermetrics must be invoked as the superuser", and
+//            `ioreg -c AppleSMC -r -d 1` publishes no temperature key at all to
+//            an unprivileged process. Asking a dashboard for a password every
+//            ten seconds is not an option, and this deck does not ship a
+//            kernel driver.
+//
+//            But the GPU driver does publish one, and nothing said so: the
+//            accelerator's PerformanceStatistics carries "Temperature(C)"
+//            beside its clock, its activity and its power. Read live on an
+//            Intel Mac with an AMD card — 60, 60, 61 over four seconds, from
+//            `ioreg -r -k PerformanceStatistics` in 51ms. Apple Silicon's
+//            AGXAccelerator publishes the same dictionary WITHOUT that key, so
+//            there the parser finds nothing and no row is drawn, which is the
+//            correct outcome rather than a special case.
+//
+//            And `pmset -g therm` is unprivileged, instant, and present on
+//            both architectures. What it reports is not heat but the
+//            consequence of heat: CPU_Speed_Limit, the share of the CPU's speed
+//            the thermal manager is currently allowing. That is arguably the
+//            more useful of the two readings — a temperature is a number you
+//            have to interpret, a speed limit is the thing you were trying to
+//            interpret it into.
+//
+//            `sysctl machdep.xcpm.cpu_thermal_level` is deliberately unused. It
+//            is live (33, then 42, then 41 over three seconds) but it is
+//            Intel-only and an undocumented scale, and printing it as though it
+//            were degrees would be exactly the lie this module refuses.
+//
+//   Windows  MSAcpi_ThermalZoneTemperature in root/wmi, CurrentTemperature in
+//            TENTHS OF A KELVIN. Published by the firmware and genuinely absent
+//            on a large share of desktop boards — the same lesson
+//            parseGetProcessJson learned about perflib, which is why absent is
+//            ordinary here rather than an error.
+//
+// NEVER INVENT A READING. No sensor means no row, and no rows at all means the
+// section is not rendered: not 0°C, not a dash, not a grey empty bar. Same rule
+// that keeps `cpu` null until two samples exist.
+
+/** How often the thermal reading is refreshed. Heat moves on the scale of
+ *  seconds, and the two platforms that cost a subprocess to ask cost 51ms
+ *  (`ioreg`) and rather less (`pmset`), measured. Linux costs a file read. */
+const THERMAL_INTERVAL_MS = 10_000;
+
+/**
+ * How many consecutive empty readings before this machine is left alone.
+ *
+ * The reason is Windows, where MSAcpi_ThermalZoneTemperature is absent on a
+ * large share of desktops: without this, every one of those machines would pay
+ * a `Get-CimInstance` child every ten seconds, forever, to render a section it
+ * can never render. Three rather than one because a single failure can be a
+ * hiccup — a timeout, a machine mid-wake — and giving up on a hiccup would lose
+ * a reading the machine does have.
+ *
+ * Not persisted. A restart asks again, which is what should happen after the
+ * user installs a driver or changes a firmware setting.
+ */
+const THERMAL_GIVE_UP = 3;
+
+/** Bands used where the hardware publishes none of its own. Linux sensors
+ *  carry `temp*_max` and `temp*_crit` and those win: a laptop package sensor
+ *  and an NVMe drive do not share a comfortable range, and one scale for both
+ *  would be a number this module made up. */
+const WARN_C = 75;
+const CRIT_C = 90;
+
+/**
+ * How much history the panel keeps, and why it is bucketed by minute.
+ *
+ * Each section answers "what is it now"; the chart behind it answers "what did
+ * it do while that build was running", which is a different question and the
+ * reason a section opens one at all. 1440 minutes is a day, which covers "since
+ * the deck started" for every session anybody actually has.
+ *
+ * A bucket holds the MAXIMUM of its minute, never the mean, and that choice is
+ * the same one for every series here. A machine that touched 94°C for twenty
+ * seconds and sat at 60 for the rest of the minute averages to 66 and reads as
+ * calm; a load average that spiked to 114 between two quiet stretches averages
+ * away entirely. The spike is what somebody opens a chart to find.
+ *
+ * Kept out of systemSnapshot deliberately. That endpoint is polled every three
+ * seconds by a topbar meter that draws none of this; a day of buckets on every
+ * one of those responses would be the largest thing the deck sends, for charts
+ * that are usually closed. It has its own route, like the process list.
+ */
+const HISTORY_MINUTES = 1440;
+const BUCKET_MS = 60_000;
+
+/** A reading that is not a temperature is not a misparse to be shown anyway.
+ *  Silicon does not run below freezing or above 130°C, and both ends of that
+ *  have been produced by reading the right file with the wrong unit. */
+const plausible = c => Number.isFinite(c) && c > 0 && c < 130;
+
+/**
+ * Millidegrees Celsius out of a hwmon `temp*_input`, or null.
+ *
+ * The kernel writes an integer; the divide is the whole conversion. Exported
+ * and pure for the reason every parser here is: a Linux answer has to be
+ * checkable from a Mac.
+ */
+export function celsiusFromMilli(text) {
+  const n = Number(String(text ?? "").trim());
+  const c = Math.round(n / 1000);
+  return plausible(c) ? c : null;
+}
+
+/**
+ * Which of a machine's sensors the panel names, out of every sensor found.
+ *
+ * A real machine publishes a lot of them: the package, one per core, the NVMe
+ * drive, the wireless card, the chipset. Two rows is what the panel has room
+ * for and two rows is what somebody watching a build wants, so this picks the
+ * CPU and the GPU by the chip that published them and leaves the rest alone.
+ *
+ * Preference inside a chip matters as much as the chip does. coretemp exposes
+ * `Package id 0` beside `Core 0`..`Core N`, and the package is the reading
+ * — a single core's number is noisier and lower than the die it sits on.
+ * k10temp exposes `Tctl` and, on parts that have it, `Tdie`: Tctl is Tdie plus
+ * a vendor offset that exists for fan control, so Tdie is the temperature and
+ * Tctl is the fallback. amdgpu's `edge` is the die edge and `junction` is the
+ * hotspot; edge is what every other tool calls the GPU temperature.
+ *
+ * Where a chip publishes nothing recognisable, the hottest of its sensors is
+ * taken, because the question is "is it getting hot" and the hottest sensor is
+ * the one that answers it.
+ */
+const CPU_CHIPS = ["coretemp", "k10temp", "zenpower", "cpu_thermal", "soc_thermal"];
+const GPU_CHIPS = ["amdgpu", "nouveau", "i915", "xe", "radeon"];
+
+export function pickThermalRows(sensors) {
+  const hottest = rows => rows.reduce((a, b) => (b.celsius > a.celsius ? b : a));
+  const pick = (chips, prefer, label) => {
+    const mine = (sensors ?? []).filter(s => chips.includes(s.chip) && plausible(s.celsius));
+    if (!mine.length) return null;
+    for (const re of prefer) {
+      const hit = mine.find(s => re.test(s.label ?? ""));
+      if (hit) return { ...hit, label };
+    }
+    return { ...hottest(mine), label };
+  };
+  return [
+    pick(CPU_CHIPS, [/^package id/i, /^tdie$/i, /^tctl$/i], "CPU"),
+    pick(GPU_CHIPS, [/^edge$/i, /^junction$/i], "GPU"),
+  ].filter(Boolean);
+}
+
+/**
+ * Every temperature sensor under /sys/class/hwmon, with the chip that owns it
+ * and the bands that chip publishes for it.
+ *
+ * `root` is a parameter so this can be pointed at a tree on disk. There is no
+ * Linux machine here and no container runtime, so the alternative would be a
+ * directory walk nobody has ever run — and a walk is exactly the kind of code
+ * that a fixture of its OUTPUT cannot check, because the walk is the part that
+ * is wrong.
+ */
+export async function readHwmon(root = "/sys/class/hwmon", deps = {}) {
+  const dir = deps.readdir ?? readdir;
+  const file = deps.readFile ?? readFile;
+  const read = async path => { try { return String(await file(path, "utf8")).trim(); } catch { return null; } };
+  let chips;
+  try { chips = await dir(root); } catch { return []; }
+  const out = [];
+  for (const hwmon of chips) {
+    const base = `${root}/${hwmon}`;
+    const chip = (await read(`${base}/name`)) ?? hwmon;
+    let entries;
+    try { entries = await dir(base); } catch { continue; }
+    for (const entry of entries) {
+      const m = /^(temp\d+)_input$/.exec(entry);
+      if (!m) continue;
+      const celsius = celsiusFromMilli(await read(`${base}/${entry}`));
+      if (celsius == null) continue;
+      out.push({
+        chip,
+        label: await read(`${base}/${m[1]}_label`),
+        celsius,
+        // The hardware's own bands where it has them. `max` is where the chip
+        // says it is unhappy and `crit` is where it says it will act.
+        warnAt: celsiusFromMilli(await read(`${base}/${m[1]}_max`)) ?? WARN_C,
+        critAt: celsiusFromMilli(await read(`${base}/${m[1]}_crit`)) ?? CRIT_C,
+      });
+    }
+  }
+  return out;
+}
+
+/**
+ * The coarser Linux fallback, for a machine whose sensors have no hwmon driver.
+ *
+ * One row, and it is labelled with the zone's own `type` rather than "CPU",
+ * because a thermal zone is not a claim about what was measured. `acpitz` is
+ * the motherboard's idea of ambient on a lot of hardware and calling that the
+ * CPU would be the same lie in a different place.
+ */
+const ZONE_ORDER = ["x86_pkg_temp", "cpu-thermal", "cpu_thermal", "soc_thermal"];
+
+export async function readThermalZones(root = "/sys/class/thermal", deps = {}) {
+  const dir = deps.readdir ?? readdir;
+  const file = deps.readFile ?? readFile;
+  const read = async path => { try { return String(await file(path, "utf8")).trim(); } catch { return null; } };
+  let zones;
+  try { zones = (await dir(root)).filter(n => /^thermal_zone\d+$/.test(n)); } catch { return []; }
+  const found = [];
+  for (const zone of zones) {
+    const celsius = celsiusFromMilli(await read(`${root}/${zone}/temp`));
+    if (celsius == null) continue;
+    found.push({ label: (await read(`${root}/${zone}/type`)) ?? zone, celsius, warnAt: WARN_C, critAt: CRIT_C });
+  }
+  if (!found.length) return [];
+  const known = found.find(z => ZONE_ORDER.includes(z.label));
+  return [known ?? found.reduce((a, b) => (b.celsius > a.celsius ? b : a))];
+}
+
+/**
+ * GPU degrees out of `ioreg -r -k PerformanceStatistics`.
+ *
+ * The macOS reading nothing documented: the accelerator publishes
+ * "Temperature(C)" in the same dictionary as its clock and its power. The
+ * maximum across accelerators, because a machine with two cards is asking
+ * whether it is getting hot, and the hotter card is the answer.
+ */
+export function gpuFromIoreg(text) {
+  let best = null;
+  for (const m of String(text ?? "").matchAll(/"Temperature\(C\)"\s*=\s*(-?\d+)/g)) {
+    const c = Number(m[1]);
+    if (plausible(c) && (best == null || c > best)) best = c;
+  }
+  return best;
+}
+
+/**
+ * The share of the CPU's speed the thermal manager is allowing, out of
+ * `pmset -g therm`, or null when this Mac has never recorded one.
+ *
+ * `CPU_Scheduler_Limit` sits beside it and is deliberately not read: it limits
+ * scheduling rather than clock, so folding the two into one percentage would
+ * produce a number that is neither. If scheduler throttling turns out to matter
+ * it is a second row, not a redefinition of this one.
+ */
+export function throttleFromPmset(text) {
+  const m = /CPU_Speed_Limit\s*=\s*(\d+)/.exec(String(text ?? ""));
+  if (!m) return null;
+  const pct = Number(m[1]);
+  if (!Number.isFinite(pct) || pct < 0 || pct > 100) return null;
+  return { speedLimit: pct };
+}
+
+/**
+ * Windows thermal zones out of MSAcpi_ThermalZoneTemperature.
+ *
+ * CurrentTemperature is in tenths of a Kelvin, which is the single detail this
+ * whole branch turns on: reading it as anything else gives a number that is
+ * plausible-looking and wrong.
+ *
+ * The zone is labelled "Thermal zone" when there is one and by its own name
+ * when there are several, because ACPI does not say which zone is the CPU and
+ * this module does not guess. `TZ00` is not a friendly label; it is an honest
+ * one, and it only appears on a machine that has more than one.
+ */
+export function tempFromMsAcpiJson(json) {
+  let rows;
+  try { rows = typeof json === "string" ? JSON.parse(json) : json; }
+  catch { return []; }
+  if (!rows) return [];
+  if (!Array.isArray(rows)) rows = [rows];
+  const found = [];
+  for (const r of rows) {
+    const k = Number(r?.CurrentTemperature);
+    if (!Number.isFinite(k)) continue;
+    const celsius = Math.round(k / 10 - 273.15);
+    if (!plausible(celsius)) continue;
+    const name = String(r?.InstanceName ?? "").split("\\").pop().replace(/_\d+$/, "");
+    found.push({ label: name || "Thermal zone", celsius, warnAt: WARN_C, critAt: CRIT_C });
+  }
+  if (found.length === 1) found[0].label = "Thermal zone";
+  return found.slice(0, 2);
+}
+
+/**
+ * What /api/system carries, or null when this machine says nothing at all.
+ *
+ * Two fields rather than one list, because they are two different readings and
+ * collapsing them would let a throttle percentage be drawn under a °C heading
+ * — the thing the label rule exists to prevent. `swapLabel` earned that rule
+ * once already.
+ */
+export async function readThermal(platform = process.platform) {
+  if (platform === "linux") {
+    const sensors = await readHwmon();
+    const celsius = pickThermalRows(sensors);
+    const rows = celsius.length ? celsius : await readThermalZones();
+    return rows.length ? { celsius: rows, throttle: null } : null;
+  }
+
+  if (platform === "darwin") {
+    // Scoped by key rather than dumped whole: `ioreg -l` is 217KB and just
+    // under two seconds on this machine, `-r -k PerformanceStatistics` is 83KB
+    // and 51ms for the same number.
+    const [gpu, therm] = await Promise.all([
+      run("ioreg", ["-r", "-k", "PerformanceStatistics", "-w", "0"], 3_000),
+      run("pmset", ["-g", "therm"]),
+    ]);
+    const celsius = [];
+    const c = gpu ? gpuFromIoreg(gpu) : null;
+    if (c != null) celsius.push({ label: "GPU", celsius: c, warnAt: WARN_C, critAt: CRIT_C });
+    const throttle = therm ? throttleFromPmset(therm) : null;
+    return celsius.length || throttle ? { celsius, throttle } : null;
+  }
+
+  if (platform === "win32") {
+    const out = await run("powershell.exe", [
+      "-NoProfile", "-NonInteractive", "-Command",
+      "Get-CimInstance -Namespace root/wmi -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | Select-Object InstanceName,CurrentTemperature | ConvertTo-Json -Compress",
+    ], 6_000);
+    if (!out) return null;
+    const celsius = tempFromMsAcpiJson(out.trim());
+    return celsius.length ? { celsius, throttle: null } : null;
+  }
+
+  return null;
+}
+
+/**
+ * Fold one reading into the minute it belongs to, under a namespaced key.
+ *
+ * Keys are namespaced by section (`thermal:GPU`, `cpu:all`, `mem:swap`) rather
+ * than kept in four rings, because they all share one clock: a bucket is a
+ * minute of this machine, and every series that has something to say about that
+ * minute says it in the same place. The sections sample at different rates —
+ * CPU every three seconds, thermal every ten, memory every thirty — and folding
+ * by maximum makes that difference invisible to the reader, which is what it
+ * should be.
+ */
+function record(key, value, nowMs = Date.now()) {
+  if (!Number.isFinite(value)) return;
+  const minute = Math.floor(nowMs / BUCKET_MS);
+  let last = history[history.length - 1];
+  if (!last || last.m !== minute) {
+    last = { m: minute, v: {} };
+    history.push(last);
+    while (history.length > HISTORY_MINUTES) history.shift();
+  }
+  const prev = last.v[key];
+  last.v[key] = prev == null ? value : Math.max(prev, value);
+}
+
+/**
+ * The thermal reading, whose series are not known until the machine answers.
+ *
+ * Keyed by the row's own label rather than by position, because the rows are
+ * not the same on every platform and a machine can start reporting a sensor it
+ * was not reporting before — a GPU driver loads, a laptop is docked. A series
+ * that appears late simply has no points before it appeared, which is the truth
+ * and draws correctly.
+ */
+function recordThermal(reading, nowMs = Date.now()) {
+  if (!reading) return;
+  for (const r of reading.celsius ?? []) record(`thermal:${r.label}`, r.celsius, nowMs);
+  // Stored as the share TAKEN AWAY, the same way the panel draws it, so the
+  // chart and the row cannot disagree about which direction is bad.
+  if (reading.throttle) {
+    record(`thermal:${THROTTLE_LABEL}`, Math.max(0, 100 - reading.throttle.speedLimit), nowMs);
+  }
+}
+
+/** The one thermal row that is not degrees. Named once so the recorder, the
+ *  route and the panel cannot drift apart on the spelling. */
+export const THROTTLE_LABEL = "Throttling";
+
+/**
+ * The scale a series is drawn against.
+ *
+ * Fixed at 100 wherever the PANEL draws the same number against a 0-100 track,
+ * because two pictures of one reading that disagree about how alarming it is
+ * would be worse than either alone. Load average is the exception and gets a
+ * fitted top: it is genuinely unbounded — measured at 114 on a twelve-core
+ * machine — and the section that shows it draws no track at all, so there is no
+ * competing picture for a fitted scale to contradict. Rounded up to something a
+ * person would choose, and floored at one and a half times the core count so a
+ * quiet machine is not drawn as a dramatic climb.
+ */
+function loadTop(points, coreCount) {
+  const peak = points.reduce((a, p) => Math.max(a, p.v), 0);
+  const floor = Math.max(4, Math.ceil(coreCount * 1.5));
+  const want = Math.max(floor, peak * 1.15);
+  const step = want <= 20 ? 5 : want <= 100 ? 10 : 50;
+  return Math.ceil(want / step) * step;
+}
+
+/**
+ * What each section's chart is made of.
+ *
+ * One entry per series, each carrying its own unit, its own bands and its own
+ * scale, because a percentage, a temperature and a queue depth share nothing —
+ * drawing them against one axis would invite a reading of one shape against
+ * another that means nothing.
+ */
+function seriesFor(group) {
+  const at = key => history.filter(b => b.v[key] != null)
+    // Timestamps rather than indices: a bucket only exists for a minute that was
+    // sampled, so a gap — the machine asleep, the process paused — stays a gap
+    // rather than becoming a straight line across it.
+    .map(b => ({ t: b.m * BUCKET_MS, v: b.v[key] }));
+  const coreCount = os.cpus().length;
+
+  if (group === "thermal") {
+    const bands = new Map((thermal?.celsius ?? []).map(r => [r.label, r]));
+    const labels = [];
+    for (const b of history) {
+      for (const k of Object.keys(b.v)) {
+        if (!k.startsWith("thermal:")) continue;
+        const label = k.slice("thermal:".length);
+        if (!labels.includes(label)) labels.push(label);
+      }
+    }
+    return labels.map(label => ({
+      label,
+      unit: label === THROTTLE_LABEL ? "%" : "C",
+      top: 100,
+      // Throttling is the one reading here whose normal value is zero, so it
+      // is the one that does not need a full-height box to be read. Said by the
+      // series rather than inferred from "has no bands", which was the first
+      // rule and was wrong: CPU has no bands DELIBERATELY and uses the whole
+      // scale, so it was getting the short box for a reason that is not true
+      // of it.
+      restsAtZero: label === THROTTLE_LABEL,
+      warnAt: label === THROTTLE_LABEL ? null : (bands.get(label)?.warnAt ?? WARN_C),
+      critAt: label === THROTTLE_LABEL ? null : (bands.get(label)?.critAt ?? CRIT_C),
+      points: at(`thermal:${label}`),
+    }));
+  }
+
+  if (group === "cores") {
+    // Not one line per core: twelve lines in a 620px dialog is a picture nobody
+    // can read. These two answer what the columns cannot answer over time —
+    // "all cores" at 20 with "busiest" at 100 is ONE core pinned, which is a
+    // different machine from twelve at 20.
+    //
+    // No bands, deliberately, and the reason is written at the top of
+    // SystemMeter: a CPU at 90% is the machine doing the work you asked for. An
+    // indicator that alarms during the normal case teaches you to stop reading
+    // it.
+    return [
+      { label: "All cores", unit: "%", top: 100, warnAt: null, critAt: null, points: at("cpu:all"), restsAtZero: false },
+      { label: "Busiest core", unit: "%", top: 100, warnAt: null, critAt: null, points: at("cpu:busiest"), restsAtZero: false },
+    ].filter(s => s.points.length);
+  }
+
+  if (group === "memory") {
+    const swapLabel = process.platform === "win32" ? "Commit" : "Swap";
+    return [
+      // One band, not two. A `critAt` of 100 draws a rule along the top of a
+      // chart whose scale ends at 100 — it is the ceiling, drawn again in red,
+      // and it says nothing the edge did not.
+      { label: "Physical", unit: "%", top: 100, warnAt: 90, critAt: null, restsAtZero: false, points: at("mem:physical") },
+      { label: swapLabel, unit: "%", top: 100, warnAt: 90, critAt: null, restsAtZero: false, points: at("mem:swap") },
+    ].filter(s => s.points.length);
+  }
+
+  if (group === "load") {
+    // One series, not three. 1m, 5m and 15m are three views of one number —
+    // the longer two are the short one smoothed — so charting the 1m over an
+    // hour says everything the other two would, at the resolution they hide.
+    const points = at("load:1m");
+    if (!points.length) return [];
+    return [{
+      label: "Queued work",
+      unit: "",
+      top: loadTop(points, coreCount),
+      // Where the queue exceeds the cores there are to run it, which is the one
+      // number the section's own note already draws the line at.
+      warnAt: coreCount,
+      critAt: null,
+      restsAtZero: false,
+      points,
+    }];
+  }
+
+  return [];
+}
+
+/**
+ * Whether this machine has been held back AT ALL since the deck started, and
+ * when it last was.
+ *
+ * The row reports the current sample, and on a desktop that current sample is
+ * `0%` essentially always — measured here: ninety seconds of AES-NI on twelve
+ * cores never moved `CPU_Speed_Limit` off 100. Which is the truth, and which
+ * reads as "this readout does not work" the second time somebody looks at it.
+ * It was reported that way twice.
+ *
+ * So the note under the row gets to say the other thing. Nothing new is
+ * sampled for it: the minute buckets already hold the peak of every minute, and
+ * this is a scan of what is already there. A machine that has never been
+ * throttled says so; one that was at lunchtime says when.
+ */
+function heldBackSoFar() {
+  const key = `thermal:${THROTTLE_LABEL}`;
+  let peak = 0;
+  let lastMs = 0;
+  for (const b of history) {
+    const v = b.v[key];
+    if (v == null || v <= 0) continue;
+    if (v > peak) peak = v;
+    lastMs = b.m * BUCKET_MS;
+  }
+  return peak > 0 ? { peak, lastMs } : null;
+}
+
+/** What /api/system/history answers, for one section. */
+export function historySnapshot(group) {
+  return { ok: true, sinceMs: historySince, stepMs: BUCKET_MS, series: seriesFor(group) };
+}
+
+/**
+ * One reading at a time, and a machine that cannot answer is asked three times
+ * rather than for the life of the process.
+ *
+ * `deps.read` is a seam rather than a convenience: the rule this function
+ * exists for only fires on a machine that answers with nothing, and the machine
+ * this was written on answers with something, so there is no other way to run
+ * the branch that matters.
+ */
+export async function sampleThermal(deps = {}) {
+  if (thermalInFlight) return;
+  // Giving up is only ever for a machine that has NEVER answered — the Windows
+  // desktop with no MSAcpi class, which would otherwise pay a PowerShell child
+  // every ten seconds for the life of the process. A machine that answered once
+  // has a sensor, and it keeps being asked however long the silence runs.
+  if (!thermalEverAnswered && thermalMisses >= THERMAL_GIVE_UP) return;
+  const read = deps.read ?? readThermal;
+  thermalInFlight = true;
+  try {
+    const next = await read();
+    if (next) { thermal = next; thermalMisses = 0; thermalEverAnswered = true; recordThermal(next); }
+    else if (++thermalMisses >= THERMAL_GIVE_UP) {
+      // DROP THE LAST READING. It used to be kept, and that is a number from
+      // four minutes ago printed as though it were now — the one thing this
+      // whole section refuses. A GPU driver unloads, a laptop is docked, a
+      // sensor goes away: the honest answer is that the section stops being
+      // drawn, not that it freezes.
+      thermal = null;
+      // But keep ASKING on a machine that has answered before. The cost
+      // argument for giving up was only ever about a machine that can never
+      // answer — a Windows desktop with no MSAcpi class paying a PowerShell
+      // child every ten seconds forever. One that answered has a sensor, and a
+      // silence is a gap rather than an absence.
+      if (!thermalEverAnswered && thermalTimer) {
+        clearInterval(thermalTimer);
+        thermalTimer = null;
+      }
+    }
+  } catch { thermalMisses++; }
+  finally { thermalInFlight = false; }
 }
 
 async function sampleMemory() {
@@ -544,6 +1205,8 @@ async function sampleMemory() {
     // Same 30s cadence as memory, and for the same reason: it moves in minutes
     // and costs a subprocess on two of the three platforms.
     swap = await readSwap();
+    record("mem:physical", memory.usedPct);
+    if (swap && swap.total > 0) record("mem:swap", Math.round((swap.used / swap.total) * 1000) / 10);
   } catch { /* keep the previous reading rather than blanking the meter */ }
   finally { memInFlight = false; }
 }
@@ -555,6 +1218,13 @@ function sampleCpu() {
   cores = per;
   cpuHistory.push(pct);
   while (cpuHistory.length > HISTORY) cpuHistory.shift();
+  record("cpu:all", pct);
+  if (per?.length) record("cpu:busiest", Math.max(...per));
+  // Free — os.loadavg() reads a kernel value, no syscall worth the name — so it
+  // rides the CPU tick rather than earning a timer. Windows returns [0,0,0],
+  // which is not a reading and is not recorded as one.
+  const load = os.loadavg();
+  if (process.platform !== "win32" && load.some(n => n > 0)) record("load:1m", Math.round(load[0] * 100) / 100);
 }
 
 /**
@@ -566,16 +1236,26 @@ export function startSystemMetrics() {
   prevTicks = readTicks();          // baseline, so the first tick has a delta
   prevCoreTicks = readCoreTicks();
   sampleMemory();
+  historySince = Date.now();
+  sampleThermal();
   cpuTimer = setInterval(sampleCpu, CPU_INTERVAL_MS);
   memTimer = setInterval(sampleMemory, MEM_INTERVAL_MS);
+  thermalTimer = setInterval(sampleThermal, THERMAL_INTERVAL_MS);
   cpuTimer.unref?.();
   memTimer.unref?.();
+  thermalTimer.unref?.();
 }
 
 export function stopSystemMetrics() {
   if (cpuTimer) clearInterval(cpuTimer);
   if (memTimer) clearInterval(memTimer);
-  cpuTimer = memTimer = null;
+  if (thermalTimer) clearInterval(thermalTimer);
+  cpuTimer = memTimer = thermalTimer = null;
+  thermal = null;
+  thermalMisses = 0;
+  thermalEverAnswered = false;
+  history.length = 0;
+  historySince = 0;
   prevTicks = null;
   prevCoreTicks = null;
   cpuHistory.length = 0;
@@ -610,6 +1290,9 @@ export function systemSnapshot() {
     memory,
     swap,
     perCore: cores,
+    // Null on a machine that publishes nothing, and the panel draws no section
+    // at all for it rather than an empty one.
+    thermal: thermal ? { ...thermal, heldBack: heldBackSoFar() } : null,
     uptimeSec: Math.round(os.uptime()),
     platform: process.platform,
     loadavg: hasLoad ? load.map(n => Math.round(n * 100) / 100) : null,