@hienlh/ppm 0.23.12 → 0.23.13

package/src/services/supervisor.ts

@@ -7,11 +7,12 @@
 import type { Subprocess } from "bun";
 import { resolve } from "node:path";
 import {
-  readFileSync, writeFileSync, existsSync, mkdirSync, openSync, closeSync, appendFileSync,
+  readFileSync, writeFileSync, existsSync, mkdirSync, openSync, closeSync,
   unlinkSync, statSync,
 } from "node:fs";
 import { getPpmDir } from "./ppm-dir.ts";
-import { stdioIsLogFile, consumeStdioIsLogEnv, rotateIfOversized, MAX_LOG_BYTES, STDIO_IS_LOG_ENV } from "./log-rotate.ts";
+import { rotateIfOversized, MAX_LOG_BYTES, STDIO_IS_LOG_ENV } from "./log-rotate.ts";
+import { createLogger, installFileLogSink, type LogLevel } from "./logger.ts";
 import { isCompiledBinary } from "./autostart-generator.ts";
 import { cleanupStaleBinaryUpgradeArtifacts } from "./binary-upgrade-swap.ts";
 import {
@@ -29,7 +30,7 @@ import { decideNamedProbeAction, publicHostnameIsOurs } from "./named-tunnel/nam
 import { isCloudflaredPid } from "./tunnel-registry.service.ts";
 import { getQuickTunnelArgs } from "./cloudflared.service.ts";
 import { startStoppedPage, stopStoppedPage } from "./supervisor-stopped-page.ts";
-import { sdNotify } from "./sd-notify.ts";
+import { sdNotify, withoutNotifySocket } from "./sd-notify.ts";
 import {
   killProcessTree, snapshotServerDescendants, reapTrackedDescendants,
   findPortListenerPid, isPpmProcess, collectProcessTree, terminateTree,
@@ -38,6 +39,7 @@ import { reapZombiePortOrphans } from "./windows-zombie-port-reaper.ts";
 import {
   SERVER_PORT_FILE, resolveTargetPort, _resetTargetCache,
 } from "./edge-target-resolver.ts";
+import { EDGE_PRIOR_DEATHS_ENV } from "./edge-forwarder.ts";
 import { PLIST_LABEL } from "./autostart-generator.ts";
 
 // ─── Constants ─────────────────────────────────────────────────────────
@@ -77,6 +79,10 @@ let adoptedTunnelPid: number | null = null; // PID of tunnel kept alive across u
 // survives self-replace, so a new supervisor adopts it rather than respawning.
 let edgePid: number | null = null;
 let edgeProbeTimer: ReturnType<typeof setInterval> | null = null;
+// Deaths the edge probe saw in a row. An edge that cannot bind dies again within a second of
+// every respawn, so without this the same lines repeat every 10s while the public port is dark.
+let edgeDeathStreak = 0;
+const EDGE_DEATHS_BEFORE_ERROR = 3;
 // The loopback port our own server child published. Cleared on every respawn so
 // a stale value never fights the incoming generation. Once set, it makes the
 // supervisor the authority on what `.server-port` should contain.
@@ -162,28 +168,16 @@ let originalArgv: string[] = [];
 
 // ─── Logging ───────────────────────────────────────────────────────────
 
+const supervisorLog = createLogger("supervisor");
+
 /**
- * Whether this process's own stderr already lands in ppm.log.
- *
- * Normally it does not — under systemd stderr is the journal, and the
- * supervisor's lines belong there as well as in the log. But the replacement
- * supervisor of a self-upgrade is spawned with `stdio: ["ignore", newLogFd,
- * newLogFd]`, and from then on every line would be written to the file twice.
- * Resolved once, lazily: fd 2 does not change underneath a running process.
+ * The supervisor's own lines, through the shared logger (and so its level and redaction).
+ * Under systemd its stderr is the journal, and the sink installed at the entry point below
+ * echoes each line there as well; when stderr already is ppm.log (the replacement supervisor
+ * of a self-upgrade is spawned that way) the echo is skipped rather than written twice.
  */
-let stderrIsLogFile: boolean | null = null;
-
-function log(level: string, msg: string) {
-  const ts = new Date().toISOString();
-  const line = `[${ts}] [${level}] [supervisor] ${msg}\n`;
-  try { appendFileSync(logFile(), line); } catch {}
-  if (stderrIsLogFile === null) {
-    stderrIsLogFile = stdioIsLogFile(2, logFile());
-    consumeStdioIsLogEnv();
-  }
-  // Write supervisor logs to stderr so journalctl captures them — unless
-  // stderr is the log file itself, where that is the same line again.
-  if (!stderrIsLogFile) { try { process.stderr.write(line); } catch {} }
+function log(level: "DEBUG" | "INFO" | "WARN" | "ERROR" | "FATAL", msg: string) {
+  supervisorLog[level.toLowerCase() as LogLevel](msg);
 }
 
 let logRotateTimer: ReturnType<typeof setInterval> | null = null;
@@ -533,11 +527,14 @@ async function spawnEdge(publicPort: number, host: string, logFd: number): Promi
     detached: true,
     windowsHide: true,
     stdio: ["ignore", "ignore", logFd] as ["ignore", "ignore", number],
+    // Told that its stderr is ppm.log (Windows cannot work that out, see logger.ts), and how
+    // many edges died before it, so a bind failure on every respawn is not logged as news.
+    env: { ...withoutNotifySocket(process.env), [STDIO_IS_LOG_ENV]: "1", [EDGE_PRIOR_DEATHS_ENV]: String(edgeDeathStreak) },
   }));
   proc.unref();
   edgePid = proc.pid ?? null;
   updateStatus({ edgePid });
-  log("INFO", `Edge forwarder started on ${host}:${publicPort} (PID: ${edgePid}, detached)`);
+  log(edgeDeathStreak > 1 ? "DEBUG" : "INFO", `Edge forwarder started on ${host}:${publicPort} (PID: ${edgePid}, detached)`);
 }
 
 /**
@@ -588,8 +585,15 @@ function startEdgeProbe(publicPort: number, host: string, logFd: number) {
     if (shuttingDown || getState() === "upgrading" || !edgePid) return;
     try {
       process.kill(edgePid, 0);
+      if (edgeDeathStreak >= EDGE_DEATHS_BEFORE_ERROR) {
+        log("INFO", `Edge forwarder up again (PID: ${edgePid}) after ${edgeDeathStreak} deaths in a row`);
+      }
+      edgeDeathStreak = 0;
     } catch {
-      log("WARN", `Edge forwarder (PID: ${edgePid}) died — respawning`);
+      edgeDeathStreak++;
+      if (edgeDeathStreak === 1) log("WARN", `Edge forwarder (PID: ${edgePid}) died — respawning`);
+      else if (edgeDeathStreak === EDGE_DEATHS_BEFORE_ERROR) log("ERROR", `Edge forwarder died ${edgeDeathStreak} times in a row — public port ${publicPort} is dark`);
+      else log("DEBUG", `Edge forwarder (PID: ${edgePid}) died again (#${edgeDeathStreak} in a row) — respawning`);
       edgePid = null;
       void spawnEdge(publicPort, host, logFd).catch((e) =>
         log("ERROR", `Edge respawn failed: ${e}`));
@@ -689,7 +693,7 @@ export async function spawnServer(
     cmd,
     stdio: ["ignore", logFd, logFd],
     // The child cannot work out on Windows that fd 1 is already the log, so it is told.
-    env: { ...process.env, [STDIO_IS_LOG_ENV]: "1" },
+    env: { ...withoutNotifySocket(process.env), [STDIO_IS_LOG_ENV]: "1" },
     // No visible console window. Critical on Windows after an upgrade: the new
     // supervisor is spawned consoleless (detached), so without this its console
     // children — and the Claude SDK grandchildren they spawn — pop blank windows.
@@ -702,8 +706,13 @@ export async function spawnServer(
   log("INFO", `Server started (PID: ${childPid})`);
   void mirrorServerPort();
 
+  // Kept past the null below: a crash line without the signal and uptime cannot tell an
+  // OOM kill or a health-check kill from a crash at boot.
+  const exitedChild = serverChild;
+  const startedAt = Date.now();
   const exitCode = await serverChild.exited;
   serverChild = null;
+  const exitSignal = exitedChild.signalCode ?? "none";
 
   // Don't respawn if in stopped state (soft stop)
   if (getState() === "stopped") {
@@ -737,7 +746,7 @@ export async function spawnServer(
   serverRestarts++;
 
   if (serverRestarts > MAX_RESTARTS) {
-    log("WARN", `Server exceeded ${MAX_RESTARTS} restarts, pausing`);
+    log("FATAL", `Server crashed ${serverRestarts} times in a row, under ${STABLE_WINDOW_MS / 60_000} min apart (last exit ${exitCode}, signal ${exitSignal}) — giving up; server paused until 'ppm restart --force' or a Cloud resume`);
     notifyStateChange("running", "paused", "max_restarts_exceeded");
     setState("paused");
     updateStatus({
@@ -760,7 +769,7 @@ export async function spawnServer(
   }
 
   const delay = backoffDelay(serverRestarts);
-  log("WARN", `Server crashed (exit ${exitCode}), restarting in ${delay}ms (#${serverRestarts})`);
+  log("ERROR", `Server crashed (exit ${exitCode}, signal ${exitSignal}, up ${Math.round((Date.now() - startedAt) / 1000)}s) — restart #${serverRestarts}/${MAX_RESTARTS}, restarting in ${delay}ms`);
   await Bun.sleep(delay);
 
   if (!shuttingDown) return spawnServer(serverArgs, logFd);
@@ -769,6 +778,33 @@ export async function spawnServer(
 // ─── Tunnel management ─────────────────────────────────────────────────
 const cloudflaredLogPath = () => resolve(getPpmDir(), "cloudflared.log");
 
+/**
+ * Why cloudflared failed, in its own words: the last ERR line it wrote after `fromByteOffset`,
+ * preferring one that names an `error=`. Read before the next spawn deletes the file. Left out
+ * whenever it could carry a credential — a connector's settings line can hold the run token.
+ * A visitor's failed request is not the tunnel failing, and its `dest=` is the URL that was
+ * asked for, a capability path or `?token=` among them: those lines are never picked.
+ */
+function lastCloudflaredError(fromByteOffset = 0): string | null {
+  try {
+    const lines = readFileSync(cloudflaredLogPath()).subarray(fromByteOffset).toString("utf8").split(/\r?\n/)
+      .filter((l) => / ERR /.test(l) && !/\b(dest|originService|cfRay)=/.test(l));
+    const line = lines.findLast((l) => /error=/.test(l)) ?? lines.at(-1);
+    if (!line) return null;
+    const msg = line.replace(/^\S*\s*ERR\s+/, "").replace(/https?:\/\/\S+/g, "<url>").trim();
+    if (/token|secret|password|credential/i.test(msg) || /[A-Za-z0-9+/_=-]{40,}/.test(msg)) return null;
+    return msg.slice(0, 240);
+  } catch {
+    return null;
+  }
+}
+
+/** `" — cloudflared: <last ERR line>"`, or nothing when there is none worth showing. */
+function cloudflaredCause(fromByteOffset = 0): string {
+  const line = lastCloudflaredError(fromByteOffset);
+  return line ? ` — cloudflared: ${line}` : "";
+}
+
 /**
  * Wait for the quick-mode trycloudflare URL in the (offset-anchored) log —
  * thin wrapper over phase 2a's `waitForLogLine` so quick mode keeps its own
@@ -788,7 +824,7 @@ async function syncUrlToCloud(url: string) {
     if (getCloudDevice()) {
       const ok = await sendHeartbeat(url);
       if (ok) log("INFO", `Cloud synced: ${url}`);
-      else log("WARN", "Cloud sync failed (non-blocking)");
+      else log("DEBUG", "Cloud sync failed (non-blocking)"); // sendHeartbeat warned, with the reason
     }
   } catch {}
 }
@@ -802,7 +838,7 @@ async function syncUrlToCloud(url: string) {
  * just slower — so named mode's extra fallback attempt never changes the
  * retry cadence quick-only installs already depend on.
  */
-async function retryTunnelAfterFailure(generation: number): Promise<void> {
+async function retryTunnelAfterFailure(generation: number, failure?: string): Promise<void> {
   if (shuttingDown) return;
   const now = Date.now();
   if (now - lastTunnelCrash > STABLE_WINDOW_MS) tunnelRestarts = 0;
@@ -810,7 +846,9 @@ async function retryTunnelAfterFailure(generation: number): Promise<void> {
   tunnelRestarts++;
   if (tunnelRestarts > MAX_RESTARTS) tunnelRestarts = MAX_RESTARTS;
   const delay = backoffDelay(tunnelRestarts) + Math.floor(Math.random() * 1000);
-  log("WARN", `Tunnel failed, retry in ${delay}ms (#${tunnelRestarts})`);
+  // One line per failed round: what failed (when the caller knows) and when it is tried again.
+  // A warning, not an error — an offline laptop goes round this loop all night.
+  log("WARN", failure ? `${failure} — retry #${tunnelRestarts} in ${delay}ms` : `Tunnel failed, retry in ${delay}ms (#${tunnelRestarts})`);
   await Bun.sleep(delay);
   if (generation !== tunnelGeneration) return; // superseded during backoff
   // Re-read the live server port: spawnServer may have moved it since this
@@ -932,6 +970,7 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
     let successfulMode: TunnelMode | null = null;
     let resolvedUrl: string | null = null;
     let downgraded = false;
+    let lastFailure: string | undefined;
 
     for (let i = 0; i < attempts.length; i++) {
       const attempt = attempts[i]!;
@@ -961,7 +1000,9 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
         downgraded = i > 0;
         break;
       } catch (err) {
-        log("ERROR", `${attempt.mode} tunnel failed to start: ${err}`);
+        lastFailure = `${attempt.mode} tunnel failed to start (${err})${cloudflaredCause(offset)}`;
+        const next = attempts[i + 1];
+        if (next) log("WARN", `${lastFailure} — falling back to ${next.mode}`);
         try { if (pid) process.kill(pid, "SIGKILL"); } catch {}
         tunnelUrl = null;
         if (generation !== tunnelGeneration) return; // superseded — don't bother with the next attempt
@@ -969,7 +1010,7 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
     }
 
     if (winPid === null || !successfulMode || resolvedUrl === null) {
-      return retryTunnelAfterFailure(generation);
+      return retryTunnelAfterFailure(generation, lastFailure);
     }
 
     // A newer authoritative (re)start superseded us while we extracted the URL —
@@ -1018,6 +1059,8 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
   let successfulMode: TunnelMode | null = null;
   let resolvedUrl: string | null = null;
   let downgraded = false;
+  let lastFailure: string | undefined;
+  let readyOffset = 0; // where the live connector's own log lines start
 
   for (let i = 0; i < attempts.length; i++) {
     const attempt = attempts[i]!;
@@ -1032,7 +1075,10 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
     let attemptChild: Subprocess;
     try {
       attemptChild = await withProbeSpawnGate(() =>
-        Bun.spawn(buildCmd(attempt.args), { stderr: attemptLogFd, stdout: "ignore", stdin: "ignore" }));
+        // An explicit env: without one, Bun hands over the environment we started with.
+        Bun.spawn(buildCmd(attempt.args), {
+          stderr: attemptLogFd, stdout: "ignore", stdin: "ignore", env: withoutNotifySocket(process.env),
+        }));
       tunnelChild = attemptChild; // publish so restartTunnel/killStaleTunnel can reach the live child
     } finally {
       try { closeSync(attemptLogFd); } catch {} // cloudflared keeps its own via dup2
@@ -1050,9 +1096,12 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
       child = attemptChild;
       successfulMode = attempt.mode;
       downgraded = i > 0;
+      readyOffset = offset;
       break;
     } catch (err) {
-      log("ERROR", `${attempt.mode} tunnel failed to start: ${err}`);
+      lastFailure = `${attempt.mode} tunnel failed to start (${err})${cloudflaredCause(offset)}`;
+      const next = attempts[i + 1];
+      if (next) log("WARN", `${lastFailure} — falling back to ${next.mode}`);
       try { attemptChild.kill(); } catch {}
       if (tunnelChild === attemptChild) tunnelChild = null;
       tunnelUrl = null;
@@ -1061,7 +1110,7 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
   }
 
   if (!child || !successfulMode || resolvedUrl === null) {
-    return retryTunnelAfterFailure(generation);
+    return retryTunnelAfterFailure(generation, lastFailure);
   }
 
   // A newer authoritative (re)start superseded us while we extracted the URL —
@@ -1102,8 +1151,10 @@ export async function spawnTunnel(port: number, generation: number = ++tunnelGen
     return;
   }
 
-  log("WARN", `Tunnel process exited (code=${exitCode}, url=${deadUrl}), applying backoff`);
-  return retryTunnelAfterFailure(generation);
+  return retryTunnelAfterFailure(
+    generation,
+    `Tunnel process exited (code=${exitCode}, signal=${child.signalCode ?? "none"}, url=${deadUrl})${cloudflaredCause(readyOffset)}`,
+  );
 }
 
 // ─── Config-database snapshots ─────────────────────────────────────────
@@ -1176,15 +1227,21 @@ function startServerHealthCheck() {
       healthFailCount = 0;
       return;
     }
+    let lastProbe: string;
     try {
       const res = await fetch(`http://127.0.0.1:${checkPort}/api/health`, {
         signal: AbortSignal.timeout(5000),
       });
       if (res.ok) { healthFailCount = 0; return; }
-    } catch {}
+      lastProbe = `HTTP ${res.status}`;
+    } catch (e) {
+      const err = e as { name?: string; code?: string; message?: string } | null;
+      lastProbe = err?.name === "TimeoutError" ? "timeout after 5s" : err?.code ?? err?.message ?? String(e);
+    }
     healthFailCount++;
+    log("DEBUG", `Server health probe on port ${checkPort} failed (${healthFailCount}/${SERVER_HEALTH_FAIL_THRESHOLD}): ${lastProbe}`);
     if (healthFailCount >= SERVER_HEALTH_FAIL_THRESHOLD && serverChild) {
-      log("WARN", `Server unresponsive (${healthFailCount} failures), killing`);
+      log("ERROR", `Server unresponsive for ${healthFailCount}×${SERVER_HEALTH_INTERVAL_MS / 1000}s (last probe: ${lastProbe}) — killing PID ${serverChild.pid}`);
       const pid = serverChild.pid;
       if (process.platform === "win32") {
         killProcessTree(pid);
@@ -1588,6 +1645,7 @@ async function selfReplace(): Promise<{ success: boolean; error?: string }> {
     const proc = await withProbeSpawnGate(() => nodeSpawn(cmd[0]!, cmd.slice(1), {
       detached: true,
       stdio: ["ignore", newLogFd, newLogFd] as any,
+      // The one child that keeps NOTIFY_SOCKET: it becomes the main process (MAINPID= below).
       env: { ...process.env, [STDIO_IS_LOG_ENV]: "1" } as NodeJS.ProcessEnv,
       windowsHide: true,
     }));
@@ -1654,6 +1712,9 @@ async function notifyStateChange(from: string, to: string, reason: string) {
   } catch {}
 }
 
+/** The actions `connectCloud`'s command switch handles; anything else is refused with a warning. */
+const KNOWN_CLOUD_COMMANDS = new Set(["start", "restart", "resume", "stop", "shutdown", "status"]);
+
 /** Connect supervisor to Cloud via WebSocket (if device is linked) */
 async function connectCloud(opts: { port: number }, serverArgs: string[], logFd: number): Promise<boolean> {
   try {
@@ -1714,6 +1775,10 @@ async function connectCloud(opts: { port: number }, serverArgs: string[], logFd:
           data,
           timestamp: new Date().toISOString(),
         });
+        // The outcome otherwise reaches only Cloud: the log would show the command arriving
+        // and nothing about whether it was carried out.
+        if (success) log("INFO", `Cloud command ${cmd.action} id=${cmd.id} → ok`);
+        else log(KNOWN_CLOUD_COMMANDS.has(cmd.action) ? "INFO" : "WARN", `Cloud command ${cmd.action} id=${cmd.id} → rejected: ${error}`);
       };
 
       log("INFO", `Cloud command received: ${cmd.action}`);
@@ -1816,8 +1881,9 @@ function startCloudMonitor(opts: { port: number }, serverArgs: string[], logFd:
         log("INFO", "Cloud monitor: device linked detected, connecting to cloud");
         await connectCloud(opts, serverArgs, logFd);
       } else if (device && cloudConnected && !isConnected()) {
-        // Device linked, we attempted connection but WS is dead — reconnect
-        log("WARN", "Cloud monitor: WS disconnected, reconnecting");
+        // Device linked, we attempted connection but WS is dead — reconnect. Debug: during an
+        // outage this fires every minute, and cloud-ws already reported the disconnect.
+        log("DEBUG", "Cloud monitor: WS not connected — reconnecting");
         const { disconnect } = await import("./cloud-ws.service.ts");
         disconnect();
         cloudConnected = false;
@@ -2381,7 +2447,8 @@ export async function runSupervisor(opts: {
 
   // Spawn server + (fresh) tunnel in parallel
   const promises: Promise<void>[] = [spawnServer(serverArgs, logFd)];
-  if (tunnelSharingEnabled && !tunnelAdopted) promises.push(spawnTunnel(_opts.port));
+  const tunnelLoopStarted = tunnelSharingEnabled && !tunnelAdopted;
+  if (tunnelLoopStarted) promises.push(spawnTunnel(_opts.port));
 
   await Promise.all(promises);
 
@@ -2392,7 +2459,8 @@ export async function runSupervisor(opts: {
   }
 
   // If we get here, both loops exited (shutdown or max restarts)
-  log("INFO", "Supervisor exiting");
+  if (shuttingDown) log("INFO", "Supervisor exiting");
+  else log("FATAL", `Supervisor exiting unexpectedly (exit 1, state=${getState()}, server loop ended, tunnel loop ${tunnelLoopStarted ? "ended" : "never started here (adopted or switched off)"})`);
   process.exit(shuttingDown ? 0 : 1);
 }
 
@@ -2407,11 +2475,17 @@ if (process.argv.includes("__supervise__")) {
   // (see src/server/index.ts:227, src/index.ts:27). Supervisor must not gate on it.
   const share = true;
 
+  // Every line of this process — its own and those of the tunnel, upgrade and cloud code it
+  // hosts, which log through `console.*` — goes to ppm.log, and to the journal through stderr.
+  installFileLogSink({ echo: "stderr", routeConsole: true });
+
   // Set DB profile for supervisor (needed to read config)
   if (profile) {
     const { setDbProfile } = await import("./db.service.ts");
     setDbProfile(profile);
   }
+  const { startLogLevelSync } = await import("./log-level-config.ts");
+  startLogLevelSync();
 
   runSupervisor({ port, host, profile, share });
 }

package/src/services/system-metrics/gpu-collector-nvidia.ts

@@ -7,6 +7,9 @@
 import type { GpuMetrics } from "../../types/system-metrics.ts";
 import type { Runner } from "../host-info/spawn-runner.ts";
 import { defaultRunner } from "../host-info/spawn-runner.ts";
+import { createLogger } from "../logger.ts";
+
+const log = createLogger("SystemMetrics");
 
 export const NVIDIA_SMI_ARGV = [
   "nvidia-smi",
@@ -64,13 +67,20 @@ export function createNvidiaGpuCollector(run: Runner = defaultRunner): GpuCollec
       if (disabled) return [];
       try {
         const r = await run(NVIDIA_SMI_ARGV, SPAWN_TIMEOUT_MS);
-        if (r.code !== 0 || r.timedOut) throw new Error(r.stderr || `exit ${r.code}`);
+        if (r.code !== 0 || r.timedOut) throw new Error(r.timedOut ? "timed out" : r.stderr || `exit ${r.code}`);
         failures = 0;
         return parseNvidiaSmiCsv(r.stdout);
-      } catch {
+      } catch (e) {
         // ENOENT (no NVIDIA driver) throws synchronously from the spawn; a
         // present-but-broken nvidia-smi fails a few times, then is given up on.
-        if (++failures >= MAX_FAILURES) disabled = true;
+        if (++failures >= MAX_FAILURES) {
+          disabled = true;
+          // Only the broken case is news: no driver means no nvidia-smi, on most machines.
+          const reason = (e as Error)?.message ?? String(e);
+          if (!/ENOENT|Executable not found/i.test(reason)) {
+            log.warn(`nvidia-smi disabled after ${failures} failures: ${reason.replace(/\s+/g, " ").trim().slice(0, 300)}`);
+          }
+        }
         return [];
       }
     },

package/src/services/system-metrics/kill-request-handler.ts

@@ -10,15 +10,16 @@ import type { ProcessCollector } from "./process-collector-types.ts";
 import { identityMatches, resolveLiveProcess } from "./kill-identity-resolver.ts";
 import { checkKillAllowed } from "./kill-guard.ts";
 import type { ProtectedPids } from "./ppm-protected-pids.ts";
+import type { Logger } from "../logger.ts";
 
 export interface KillHandlerDeps {
   platform: MetricsPlatform;
   collector: ProcessCollector;
   resolveProtected: (isAlive: (pid: number) => boolean, nameOf: (pid: number) => string | undefined) => ProtectedPids;
   execute: (pid: number, tree: boolean) => Promise<KillProcessResult>;
-  /** Audit line: pid + name + result ONLY. `~/.ppm/ppm.log`'s tail is served
-   *  unauthenticated by `/api/logs/recent`, so a command line must never land here. */
-  log: (line: string) => void;
+  /** Audit line: pid + name + result ONLY. `~/.ppm/ppm.log` is read back into
+   *  bug reports, so a command line must never land here. */
+  log: Logger;
 }
 
 export type KillStatus = 200 | 400 | 403 | 404 | 409 | 500;
@@ -60,20 +61,20 @@ export async function handleKillRequest(body: unknown, deps: KillHandlerDeps): P
     ppidOf: maps.ppidOf,
     startedAtOf: maps.startedAtOf,
   });
-  const prefix = `[SystemMetrics] kill pid=${live.pid} name=${live.name} tree=${tree}`;
+  const prefix = `kill pid=${live.pid} name=${live.name} tree=${tree}`;
   if (!verdict.allowed) {
-    deps.log(`${prefix} → refused: ${verdict.reason}`);
+    deps.log.warn(`${prefix} → refused: ${verdict.reason}`);
     return { status: 403, body: err(verdict.reason ?? "Refused") };
   }
 
-  deps.log(`${prefix} → allowed`);
+  deps.log.debug(`${prefix} → allowed`);
   try {
     const result = await deps.execute(live.pid, tree);
-    deps.log(`${prefix} → done (${result.method}, ${result.killed.length} signalled)`);
+    deps.log.info(`${prefix} → done (${result.method}, ${result.killed.length} signalled)`);
     return { status: 200, body: ok(result) };
   } catch (e) {
     const message = (e as Error)?.message ?? String(e);
-    deps.log(`${prefix} → failed: ${message}`);
+    deps.log.error(`${prefix} → failed: ${message}`);
     return { status: 500, body: err(`Failed to end PID ${live.pid}: ${message}`) };
   }
 }

package/src/services/system-metrics/powershell-session.ts

@@ -9,6 +9,7 @@
  * CIM provider and spawn hundreds of times a day.
  */
 import { defaultPsSpawner, encodeRequestLine, type PsChild, type PsSpawner } from "./powershell-session-bootstrap.ts";
+import { createLogger, type Logger } from "../logger.ts";
 
 export { POWERSHELL_BOOTSTRAP, defaultPsSpawner, type PsChild, type PsSpawner } from "./powershell-session-bootstrap.ts";
 
@@ -22,6 +23,8 @@ export interface PowerShellSessionOptions {
   /** Child working set creeps over hours; a scheduled recycle is free of the budget. */
   recycleAfterMs?: number;
   now?: () => number;
+  /** The child's start, failures, stop and the session being given up on. */
+  log?: Logger;
 }
 
 export const PS_DISABLED_WARNING = "Windows process collection disabled after repeated PowerShell failures";
@@ -33,6 +36,8 @@ interface Pending { id: string; resolve: (s: string) => void; reject: (e: Error)
 
 /** Anything larger than this without an end marker is not a reply we want. */
 const MAX_BUFFER_CHARS = 8 * 1024 * 1024;
+/** How much of the child's stderr a failure line quotes. */
+const STDERR_TAIL_CHARS = 300;
 
 export class PowerShellSession {
   private readonly spawn: PsSpawner;
@@ -40,6 +45,7 @@ export class PowerShellSession {
   private readonly maxRestarts: number;
   private readonly recycleAfterMs: number;
   private readonly now: () => number;
+  private readonly log: Logger;
 
   private child: PsChild | null = null;
   private generation = 0;
@@ -49,6 +55,8 @@ export class PowerShellSession {
   private previousDiedAbnormally = false;
   private disabled = false;
   private buffer = "";
+  /** The end of the current child's stderr, which says why it died when anything does. */
+  private stderrTail = "";
   private pending: Pending | null = null;
   private nextId = 1;
 
@@ -58,6 +66,7 @@ export class PowerShellSession {
     this.maxRestarts = opts.maxRestarts ?? 5;
     this.recycleAfterMs = opts.recycleAfterMs ?? 6 * 60 * 60 * 1000;
     this.now = opts.now ?? Date.now;
+    this.log = opts.log ?? createLogger("SystemMetrics");
   }
 
   isHealthy(): boolean { return this.child !== null && !this.disabled; }
@@ -69,9 +78,13 @@ export class PowerShellSession {
   request(script: string): Promise<string> {
     if (this.disabled) return Promise.reject(new PsSessionDisabledError());
     if (this.pending) return Promise.reject(new PsSessionBusyError());
-    if (this.child && this.now() - this.spawnedAt >= this.recycleAfterMs) this.stop();
+    let reason: "resume" | "recycle" = "resume";
+    if (this.child && this.now() - this.spawnedAt >= this.recycleAfterMs) {
+      this.stop();
+      reason = "recycle";
+    }
     if (!this.child) {
-      try { this.start(); } catch (e) { return Promise.reject(e as Error); }
+      try { this.start(reason); } catch (e) { return Promise.reject(e as Error); }
     }
     const child = this.child!;
     const id = String(this.nextId++);
@@ -89,6 +102,7 @@ export class PowerShellSession {
 
   /** Write nothing, kill, clear. Safe to call repeatedly. No subscribers must mean no child. */
   stop(): void {
+    if (this.child) this.log.info(`PowerShell session stopped pid=${this.child.pid}`);
     this.teardown(new Error("PowerShell session stopped"));
   }
 
@@ -101,10 +115,15 @@ export class PowerShellSession {
     if (child) { try { child.kill(); } catch { /* already gone */ } }
   }
 
-  private start(): void {
+  /** `reason` is why a healthy session needs a child: the first one, or a restart after a
+   *  failure, are told apart here. */
+  private start(reason: "resume" | "recycle"): void {
+    const why = !this.everStarted ? "first" : this.previousDiedAbnormally ? "restart" : reason;
     if (this.everStarted && this.previousDiedAbnormally) {
       if (this.restarts >= this.maxRestarts) {
         this.disabled = true;
+        // For the rest of the server's life: the Processes page stays empty from here on.
+        this.log.error(`PowerShell collector disabled after ${this.restarts} restarts`);
         throw new PsSessionDisabledError();
       }
       this.restarts++;
@@ -116,8 +135,12 @@ export class PowerShellSession {
     this.child = child;
     this.spawnedAt = this.now();
     this.buffer = "";
+    this.stderrTail = "";
+    this.log.info(`PowerShell session started pid=${child.pid} reason=${why}`);
     void this.pump(child, gen);
-    void drain(child.stderr);
+    void drain(child.stderr, (text) => {
+      if (gen === this.generation) this.stderrTail = (this.stderrTail + text).slice(-STDERR_TAIL_CHARS);
+    });
     child.exited.then(
       () => { if (gen === this.generation) this.onFailure("PowerShell exited unexpectedly"); },
       () => { if (gen === this.generation) this.onFailure("PowerShell exited unexpectedly"); },
@@ -159,6 +182,11 @@ export class PowerShellSession {
 
   /** Timeout, exit or write failure: kill, mark for a counted restart, reject. */
   private onFailure(reason: string): void {
+    const stderr = this.stderrTail.replace(/\s+/g, " ").trim();
+    this.log.warn(
+      `PowerShell session failed: ${reason} pid=${this.child?.pid ?? "?"} restarts=${this.restarts}/${this.maxRestarts}` +
+        (stderr ? `: ${stderr}` : ""),
+    );
     this.previousDiedAbnormally = true;
     this.teardown(new Error(reason));
   }
@@ -172,11 +200,16 @@ export class PowerShellSession {
   }
 }
 
-/** A full stderr pipe would block the child; read and discard it. */
-async function drain(stream: ReadableStream<Uint8Array> | null | undefined): Promise<void> {
+/** A full stderr pipe would block the child; read it, handing each chunk's text on. */
+async function drain(stream: ReadableStream<Uint8Array> | null | undefined, onText: (text: string) => void): Promise<void> {
   if (!stream) return;
+  const decoder = new TextDecoder();
   try {
     const reader = stream.getReader();
-    while (!(await reader.read()).done) { /* discard */ }
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) return;
+      onText(decoder.decode(value, { stream: true }));
+    }
   } catch { /* closed with the child */ }
 }

package/src/services/system-metrics/process-collector-windows.ts

@@ -12,6 +12,7 @@ import { createStickyColumns } from "./process-collector-types.ts";
 import { PowerShellSession, PsSessionDisabledError, PS_DISABLED_WARNING } from "./powershell-session.ts";
 import { parseWindowsTick, type WindowsCommandLine } from "./process-collector-windows-parse.ts";
 import { computeGpuPercents, memBytesToMB, type GpuUsageState } from "./gpu-process-usage-windows.ts";
+import { createLogger, type Logger } from "../logger.ts";
 
 /** `.ToUniversalTime()` here, not in JS: only .NET on this machine knows which
  *  DST rule applied at the instant each process started. */
@@ -49,7 +50,7 @@ export interface WindowsProcessCollectorOptions {
   now?: () => number;
   commandRefreshMs?: number;
   /** Seam for the one-shot startup timing line, so tests stay silent. */
-  log?: (message: string) => void;
+  log?: Logger;
 }
 
 export interface WindowsProcessCollector extends ProcessCollector {
@@ -60,7 +61,7 @@ export function createWindowsProcessCollector(opts: WindowsProcessCollectorOptio
   const session = opts.session ?? new PowerShellSession();
   const now = opts.now ?? Date.now;
   const refreshMs = opts.commandRefreshMs ?? COMMANDLINE_REFRESH_MS;
-  const log = opts.log ?? ((m: string) => console.log(m));
+  const log = opts.log ?? createLogger("SystemMetrics");
   const observeColumns = createStickyColumns();
   let commandByPid = new Map<number, WindowsCommandLine>();
   let lastCommandFetchAt = Number.NEGATIVE_INFINITY;
@@ -89,7 +90,7 @@ export function createWindowsProcessCollector(opts: WindowsProcessCollectorOptio
         logged = true;
         // One line, once: the round trip now also carries the two GPU classes,
         // and the 2 s tick budget is the thing that would break first.
-        log(`[system-metrics] windows tick round trip (incl. GPU pass): ${now() - startedRequestAt} ms`);
+        log.info(`windows tick round trip (incl. GPU pass): ${now() - startedRequestAt} ms`);
       }
 
       const parsed = parseWindowsTick(text);