@hienlh/ppm 0.23.19 → 0.23.21

package/src/services/event-loop-lag.ts

@@ -9,7 +9,7 @@
  * a busy instance against 0.12-0.65ms on an idle one of the same binary), but
  * from outside it names no culprit.
  *
- * A late tick means one of two things, and they are fixed in different places:
+ * A late tick means one of three things, and they are fixed in different places:
  *
  *   - **self** — the loop was busy with our own synchronous work, or with GC.
  *     Worth hunting: a blocking call, a whole-file read, a synchronous DB write.
@@ -18,10 +18,17 @@
  *     and one `claude` process per session. Nothing in this repository fixes
  *     that; moving work to another thread least of all, since the CPU was never
  *     ours to move it to.
+ *   - **blocked** — we got little CPU while the machine had plenty to spare, so
+ *     the thread was *waiting*: on a page-in, a disk read, a lock, a synchronous
+ *     child process. This is what every 20-60s stall on 26/10/2026 turned out to
+ *     be — the main thread in `PageIn` waits with the machine at 13% CPU, because
+ *     the process ran at memory priority 2 and I/O priority Low (see
+ *     `process-priority.ts`). They were logged as "starved" until this cause existed,
+ *     which pointed at the one explanation that was wrong.
  *
- * Sampling `process.cpuUsage()` across the same gap tells them apart: time we
- * burned is ours, time we did not is somebody else's. Without that, "the server
- * stalled" is a fact with two opposite conclusions.
+ * Sampling `process.cpuUsage()` across the same gap separates our work from not
+ * ours; the machine's CPU across it separates starved from blocked. Without both,
+ * "the server stalled" is a fact with opposite conclusions.
  *
  * The monitor costs one timer and two counter reads per tick. It holds its
  * samples in memory, and only a stall of `LOG_STALL_MS` or more also reaches
@@ -31,6 +38,7 @@
  * left nothing behind once the process restarted.
  */
 import { createLogger } from "./logger.ts";
+import { machineBusyFraction, readMachineCpuTimes, type MachineCpuTimes } from "./machine-cpu.ts";
 
 const log = createLogger("event-loop");
 
@@ -55,7 +63,13 @@ export const SELF_RATIO = 0.7;
 /** CPU/wall at or below this: the process was not running for most of the gap. */
 export const STARVED_RATIO = 0.25;
 
-export type LagCause = "self" | "starved" | "mixed";
+/**
+ * Machine-wide busy share at or above which a process not running counts as starved.
+ * Below it there were free cores, so not running means the thread was waiting.
+ */
+export const MACHINE_BUSY_FRACTION = 0.8;
+
+export type LagCause = "self" | "starved" | "blocked" | "mixed";
 
 export interface LagSample {
   /** Epoch ms at which the late tick finally ran. */
@@ -66,6 +80,8 @@ export interface LagSample {
   cpuMs: number;
   /** Resident set at that moment, to spot a stall that is really GC pressure. */
   rssMb: number;
+  /** Share of every core on the machine that was busy across the gap, 0-100; absent when unknown. */
+  machineCpuPct?: number;
   cause: LagCause;
 }
 
@@ -96,12 +112,17 @@ export interface LagReport {
  * The ratio can exceed 1: `cpuUsage` counts every thread, and Bun runs file
  * system work on a pool, so a gap spent in parallel IO bills more CPU than wall
  * time passed. That is still our own work, which is what the answer says.
+ *
+ * Without a machine reading a gap we spent off the CPU stays "starved", the only
+ * answer there was before the machine was measured.
  */
-export function classifyLag(lagMs: number, cpuMs: number): LagCause {
+export function classifyLag(lagMs: number, cpuMs: number, machineBusy?: number | null): LagCause {
   if (lagMs <= 0) return "self";
   const ratio = cpuMs / lagMs;
   if (ratio >= SELF_RATIO) return "self";
-  if (ratio <= STARVED_RATIO) return "starved";
+  if (ratio <= STARVED_RATIO) {
+    return machineBusy != null && machineBusy < MACHINE_BUSY_FRACTION ? "blocked" : "starved";
+  }
   return "mixed";
 }
 
@@ -116,18 +137,22 @@ export function sampleFromTick(args: {
   elapsedMs: number;
   cpuMicros: number;
   rssBytes: number;
+  /** Machine-wide busy share across the same gap, 0-1, when it was measured. */
+  machineBusy?: number | null;
   thresholdMs?: number;
 }): LagSample | null {
   const lagMs = args.elapsedMs - TICK_MS;
   if (lagMs < (args.thresholdMs ?? REPORT_THRESHOLD_MS)) return null;
   const cpuMs = args.cpuMicros / 1000;
-  return {
+  const sample: LagSample = {
     at: args.now,
     lagMs: Math.round(lagMs),
     cpuMs: Math.round(cpuMs),
     rssMb: Math.round(args.rssBytes / 1048576),
-    cause: classifyLag(lagMs, cpuMs),
+    cause: classifyLag(lagMs, cpuMs, args.machineBusy),
   };
+  if (args.machineBusy != null) sample.machineCpuPct = Math.round(args.machineBusy * 100);
+  return sample;
 }
 
 // ─── Monitor ───────────────────────────────────────────────────────────
@@ -137,6 +162,7 @@ let samples: LagSample[] = [];
 let startedAt = 0;
 let lastAt = 0;
 let lastCpu: { user: number; system: number } | null = null;
+let lastMachine: MachineCpuTimes | null = null;
 let ticks = 0;
 let stalls = 0;
 let blockedMs = 0;
@@ -156,7 +182,8 @@ export function logLongStall(sample: LagSample): void {
     return;
   }
   const more = longStallsNotLogged ? ` (+${longStallsNotLogged} more of ${LOG_STALL_MS}ms+ since the last one logged)` : "";
-  log.warn(`Event loop stalled ${sample.lagMs}ms (cause=${sample.cause}, cpu=${sample.cpuMs}ms, rss=${sample.rssMb}MB)${more}`);
+  const machine = sample.machineCpuPct === undefined ? "" : `, machineCpu=${sample.machineCpuPct}%`;
+  log.warn(`Event loop stalled ${sample.lagMs}ms (cause=${sample.cause}, cpu=${sample.cpuMs}ms${machine}, rss=${sample.rssMb}MB)${more}`);
   lastStallLoggedAt = sample.at;
   longStallsNotLogged = 0;
 }
@@ -168,8 +195,11 @@ function tick(): void {
   const cpuMicros = lastCpu
     ? cpu.user - lastCpu.user + (cpu.system - lastCpu.system)
     : 0;
+  const machine = readMachineCpuTimes();
+  const machineBusy = machineBusyFraction(lastMachine, machine);
   lastAt = now;
   lastCpu = cpu;
+  lastMachine = machine;
   ticks++;
 
   const sample = sampleFromTick({
@@ -177,6 +207,7 @@ function tick(): void {
     elapsedMs,
     cpuMicros,
     rssBytes: process.memoryUsage.rss(),
+    machineBusy,
   });
   if (!sample) return;
 
@@ -194,6 +225,7 @@ export function startLagMonitor(): void {
   startedAt = Date.now();
   lastAt = startedAt;
   lastCpu = process.cpuUsage();
+  lastMachine = readMachineCpuTimes();
   ticks = 0;
   stalls = 0;
   blockedMs = 0;
@@ -211,11 +243,12 @@ export function stopLagMonitor(): void {
   clearInterval(timer);
   timer = null;
   lastCpu = null;
+  lastMachine = null;
 }
 
 export function lagReport(): LagReport {
   const uptimeMs = startedAt ? Date.now() - startedAt : 0;
-  const byCause: Record<LagCause, number> = { self: 0, starved: 0, mixed: 0 };
+  const byCause: Record<LagCause, number> = { self: 0, starved: 0, blocked: 0, mixed: 0 };
   for (const s of samples) byCause[s.cause]++;
   return {
     running: timer !== null,

package/src/services/machine-cpu.ts

@@ -0,0 +1,40 @@
+/**
+ * How busy the whole machine's CPUs were between two readings.
+ *
+ * A process that got little CPU during a stall was either *starved* (every core was
+ * busy with someone else) or *blocked* (cores were free, and it was waiting on a disk
+ * read, a page-in, a lock or a child process). Its own CPU time cannot tell the two
+ * apart; the machine's can.
+ */
+import { cpus } from "node:os";
+
+export interface MachineCpuTimes {
+  /** Sum over every core of time spent not idle, in ms. */
+  busy: number;
+  /** Sum over every core of all time, in ms. */
+  total: number;
+}
+
+export function readMachineCpuTimes(): MachineCpuTimes | null {
+  try {
+    let busy = 0;
+    let total = 0;
+    for (const c of cpus()) {
+      const t = c.times;
+      const all = t.user + t.nice + t.sys + t.idle + t.irq;
+      total += all;
+      busy += all - t.idle;
+    }
+    return total > 0 ? { busy, total } : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Share of all cores that were busy between `a` and `b`, 0-1; null when it cannot be told. */
+export function machineBusyFraction(a: MachineCpuTimes | null, b: MachineCpuTimes | null): number | null {
+  if (!a || !b) return null;
+  const total = b.total - a.total;
+  if (total <= 0) return null;
+  return Math.min(1, Math.max(0, (b.busy - a.busy) / total));
+}

package/src/services/process-priority.ts

@@ -0,0 +1,153 @@
+/**
+ * This process's CPU, memory and I/O priority on Windows, and raising them back to normal.
+ *
+ * A process started by a Task Scheduler task at the default priority 7 — which is how
+ * PPM's autostart task was registered until it was changed to 5 — runs below normal on
+ * CPU, at memory priority 2 instead of 5, and at I/O priority Low instead of Normal.
+ * Every child inherits that, the server included. Windows takes pages from a
+ * low-memory-priority process first, and reading them back happens at that process's
+ * I/O priority. Measured on 26/10/2026: the server's main thread sat in `PageIn` waits
+ * for 23s on an idle NVMe disk with the machine at 13% CPU, and stalls like it, 20-60s
+ * long and several an hour, were what the supervisor's health check kept killing it for.
+ * Raising the three priorities to normal on the running process needs no privilege.
+ *
+ * Only ever raises: a priority someone set higher on purpose is left alone.
+ * Every read is `undefined` when it fails, and nothing here throws.
+ */
+
+export interface ProcessPriority {
+  /** Priority class name, e.g. "normal", "below-normal". */
+  cpu?: string;
+  /** Memory priority, 1 (very low) to 5 (normal). */
+  memory?: number;
+  /** I/O priority, 0 (very low) to 2 (normal); 3 is high. */
+  io?: number;
+}
+
+const CLASS_NAMES: Record<number, string> = {
+  0x40: "idle",
+  0x4000: "below-normal",
+  0x20: "normal",
+  0x8000: "above-normal",
+  0x80: "high",
+  0x100: "realtime",
+};
+const NORMAL_PRIORITY_CLASS = 0x20;
+/** Classes below normal, the only ones this raises. */
+const LOW_CLASSES = new Set(["idle", "below-normal"]);
+export const MEMORY_PRIORITY_NORMAL = 5;
+export const IO_PRIORITY_NORMAL = 2;
+
+/** `GetProcessInformation` / `SetProcessInformation` class. */
+const PROCESS_MEMORY_PRIORITY = 0;
+/** `NtQueryInformationProcess` / `NtSetInformationProcess` class `ProcessIoPriority`. */
+const PROCESS_IO_PRIORITY = 33;
+/** `(HANDLE)-1`, the pseudo handle for the calling process. */
+const CURRENT_PROCESS = -1n;
+
+/** What to change to bring `p` up to normal; an empty object when nothing is below it. Exported for tests. */
+export function priorityRaisePlan(p: ProcessPriority): { cpu?: true; memory?: true; io?: true } {
+  const plan: { cpu?: true; memory?: true; io?: true } = {};
+  if (p.cpu !== undefined && LOW_CLASSES.has(p.cpu)) plan.cpu = true;
+  if (p.memory !== undefined && p.memory < MEMORY_PRIORITY_NORMAL) plan.memory = true;
+  if (p.io !== undefined && p.io < IO_PRIORITY_NORMAL) plan.io = true;
+  return plan;
+}
+
+/** "cpu=below-normal memory=2/5 io=1/2", for one log line. */
+export function describePriority(p: ProcessPriority): string {
+  const v = (x: unknown) => (x === undefined ? "?" : String(x));
+  return `cpu=${v(p.cpu)} memory=${v(p.memory)}/${MEMORY_PRIORITY_NORMAL} io=${v(p.io)}/${IO_PRIORITY_NORMAL}`;
+}
+
+type Kernel32 = {
+  GetPriorityClass: (process: bigint) => number;
+  SetPriorityClass: (process: bigint, cls: number) => number;
+  GetProcessInformation: (process: bigint, cls: number, info: number, size: number) => number;
+  SetProcessInformation: (process: bigint, cls: number, info: number, size: number) => number;
+};
+type Ntdll = {
+  NtQueryInformationProcess: (process: bigint, cls: number, info: number, size: number, ret: number | null) => number;
+  NtSetInformationProcess: (process: bigint, cls: number, info: number, size: number) => number;
+};
+
+let libs: { k32: Kernel32; nt: Ntdll } | null | undefined;
+
+async function load(): Promise<{ k32: Kernel32; nt: Ntdll } | null> {
+  if (libs !== undefined) return libs;
+  try {
+    const { dlopen, FFIType } = await import("bun:ffi");
+    const k32 = dlopen("kernel32.dll", {
+      GetPriorityClass: { args: [FFIType.i64], returns: FFIType.u32 },
+      SetPriorityClass: { args: [FFIType.i64, FFIType.u32], returns: FFIType.i32 },
+      GetProcessInformation: { args: [FFIType.i64, FFIType.i32, FFIType.ptr, FFIType.u32], returns: FFIType.i32 },
+      SetProcessInformation: { args: [FFIType.i64, FFIType.i32, FFIType.ptr, FFIType.u32], returns: FFIType.i32 },
+    }).symbols as unknown as Kernel32;
+    const nt = dlopen("ntdll.dll", {
+      NtQueryInformationProcess: { args: [FFIType.i64, FFIType.i32, FFIType.ptr, FFIType.u32, FFIType.ptr], returns: FFIType.i32 },
+      NtSetInformationProcess: { args: [FFIType.i64, FFIType.i32, FFIType.ptr, FFIType.u32], returns: FFIType.i32 },
+    }).symbols as unknown as Ntdll;
+    libs = { k32, nt };
+  } catch {
+    libs = null;
+  }
+  return libs;
+}
+
+/** This process's priorities. Empty off Windows or when the calls are unavailable. */
+export async function readProcessPriority(platform: NodeJS.Platform = process.platform): Promise<ProcessPriority> {
+  if (platform !== "win32") return {};
+  try {
+    const l = await load();
+    if (!l) return {};
+    const { ptr } = await import("bun:ffi");
+    const out: ProcessPriority = {};
+    const cls = l.k32.GetPriorityClass(CURRENT_PROCESS);
+    if (cls) out.cpu = CLASS_NAMES[cls] ?? `0x${cls.toString(16)}`;
+    const mem = new Uint32Array(1);
+    if (l.k32.GetProcessInformation(CURRENT_PROCESS, PROCESS_MEMORY_PRIORITY, ptr(mem), 4) !== 0) out.memory = mem[0];
+    const io = new Uint32Array(1);
+    if (l.nt.NtQueryInformationProcess(CURRENT_PROCESS, PROCESS_IO_PRIORITY, ptr(io), 4, null) === 0) out.io = io[0];
+    return out;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Raise whatever is below normal to normal. Returns the reading before and after, so the
+ * caller can log what it found — a below-normal reading is the evidence, whether or not
+ * the raise worked.
+ */
+export async function raiseToNormalPriority(
+  platform: NodeJS.Platform = process.platform,
+): Promise<{ before: ProcessPriority; after: ProcessPriority; changed: boolean }> {
+  const before = await readProcessPriority(platform);
+  const plan = priorityRaisePlan(before);
+  if (!plan.cpu && !plan.memory && !plan.io) return { before, after: before, changed: false };
+  try {
+    const l = await load();
+    if (!l) return { before, after: before, changed: false };
+    const { ptr } = await import("bun:ffi");
+    if (plan.cpu) l.k32.SetPriorityClass(CURRENT_PROCESS, NORMAL_PRIORITY_CLASS);
+    if (plan.memory) {
+      const mem = new Uint32Array([MEMORY_PRIORITY_NORMAL]);
+      l.k32.SetProcessInformation(CURRENT_PROCESS, PROCESS_MEMORY_PRIORITY, ptr(mem), 4);
+    }
+    if (plan.io) {
+      const io = new Uint32Array([IO_PRIORITY_NORMAL]);
+      l.nt.NtSetInformationProcess(CURRENT_PROCESS, PROCESS_IO_PRIORITY, ptr(io), 4);
+    }
+  } catch { /* the reading after says what took */ }
+  return { before, after: await readProcessPriority(platform), changed: true };
+}
+
+/**
+ * Raise to normal and log it — only when something was below normal, since that reading
+ * is the one worth keeping: it says the process ran demoted until this moment.
+ */
+export async function logPriorityRaise(role: string, warn: (message: string) => void): Promise<void> {
+  const { before, after, changed } = await raiseToNormalPriority();
+  if (!changed) return;
+  warn(`${role} started below normal priority (${describePriority(before)}), raised to ${describePriority(after)}`);
+}

package/src/services/process-thread-snapshot.ts

@@ -0,0 +1,193 @@
+/**
+ * What another process's main thread is doing right now, read from outside it.
+ *
+ * When the server stops answering, nothing inside it can say why: its event loop is
+ * the thing that stopped. The supervisor can still look, and one reading taken while
+ * the health probe is timing out names the wait — `WrPageIn` (memory being read back
+ * from disk), `Executive` (a kernel wait such as a synchronous file read), `WrQueue`
+ * (idle, so the loop is fine and the probe was slow elsewhere), or running (our own
+ * synchronous work). That one word is what located the 20-60s stalls of 26/10/2026,
+ * which were invisible to every log PPM kept: an outside watcher found the main
+ * thread in `PageIn` for 23s with the machine at 13% CPU.
+ *
+ * Also returned: the process's hard page faults and CPU time, so two readings give a
+ * rate. Windows reads all of it with one `NtQuerySystemInformation` call; Linux reads
+ * `/proc/<pid>/stat` and the main task's `wchan`. Every field is `undefined` when it
+ * cannot be read, and nothing here throws.
+ */
+import { readFileSync } from "node:fs";
+
+export interface ThreadSnapshot {
+  /** Main thread scheduler state: "running", "waiting", "ready", … (Linux: R, S, D, … spelled out). */
+  state?: string;
+  /** Windows `KWAIT_REASON` name while waiting; Linux `wchan` (the kernel function it sleeps in). */
+  waitReason?: string;
+  threads?: number;
+  /** Page faults that went to disk, since the process started. */
+  hardFaults?: number;
+  /** User + kernel CPU of the whole process since it started, in ms. */
+  cpuMs?: number;
+}
+
+const THREAD_STATES = [
+  "initialized", "ready", "running", "standby", "terminated", "waiting", "transition",
+  "deferred-ready", "gate-wait", "waiting-for-outswap",
+];
+
+/** `KWAIT_REASON`, in order. */
+const WAIT_REASONS = [
+  "Executive", "FreePage", "PageIn", "PoolAllocation", "DelayExecution", "Suspended", "UserRequest",
+  "WrExecutive", "WrFreePage", "WrPageIn", "WrPoolAllocation", "WrDelayExecution", "WrSuspended",
+  "WrUserRequest", "WrSpare0", "WrQueue", "WrLpcReceive", "WrLpcReply", "WrVirtualMemory", "WrPageOut",
+  "WrRendezvous", "WrKeyedEvent", "WrTerminated", "WrProcessInSwap", "WrCpuRateControl",
+  "WrCalloutStack", "WrKernel", "WrResource", "WrPushLock", "WrMutex", "WrQuantumEnd",
+  "WrDispatchInt", "WrPreempted", "WrYieldExecution", "WrFastMutex", "WrGuardedMutex", "WrRundown",
+  "WrAlertByThreadId", "WrDeferredPreempt", "WrPhysicalFault",
+];
+
+// SYSTEM_PROCESS_INFORMATION on 64-bit Windows.
+const P_NEXT = 0x00;
+const P_THREADS = 0x04;
+const P_HARD_FAULTS = 0x10;
+const P_USER_TIME = 0x28;
+const P_KERNEL_TIME = 0x30;
+const P_PID = 0x50;
+const P_SIZE = 0x100;
+// SYSTEM_THREAD_INFORMATION, which follow it one after another.
+const T_CREATE_TIME = 0x10;
+const T_STATE = 0x44;
+const T_WAIT_REASON = 0x48;
+const T_SIZE = 0x50;
+/** Process and thread times are in 100 ns units. */
+const TICKS_PER_MS = 10_000n;
+
+/**
+ * Find `pid` in a `SystemProcessInformation` buffer and describe its main thread — the
+ * one created first. Exported for tests.
+ */
+export function parseWindowsProcessInformation(view: DataView, pid: number): ThreadSnapshot | null {
+  let at = 0;
+  for (;;) {
+    if (at + P_SIZE > view.byteLength) return null;
+    if (Number(view.getBigUint64(at + P_PID, true)) === pid) {
+      const threads = view.getUint32(at + P_THREADS, true);
+      const cpu = view.getBigInt64(at + P_USER_TIME, true) + view.getBigInt64(at + P_KERNEL_TIME, true);
+      const out: ThreadSnapshot = {
+        threads,
+        hardFaults: view.getUint32(at + P_HARD_FAULTS, true),
+        cpuMs: Number(cpu / TICKS_PER_MS),
+      };
+      let main = -1;
+      let mainCreated = 0n;
+      for (let i = 0; i < threads; i++) {
+        const t = at + P_SIZE + i * T_SIZE;
+        if (t + T_SIZE > view.byteLength) break;
+        const created = view.getBigInt64(t + T_CREATE_TIME, true);
+        if (main === -1 || created < mainCreated) { main = t; mainCreated = created; }
+      }
+      if (main !== -1) {
+        const state = view.getUint32(main + T_STATE, true);
+        out.state = THREAD_STATES[state] ?? `state-${state}`;
+        if (out.state === "waiting") {
+          const reason = view.getUint32(main + T_WAIT_REASON, true);
+          out.waitReason = WAIT_REASONS[reason] ?? `reason-${reason}`;
+        }
+      }
+      return out;
+    }
+    const next = view.getUint32(at + P_NEXT, true);
+    if (next === 0) return null;
+    at += next;
+  }
+}
+
+type Ntdll = { NtQuerySystemInformation: (cls: number, buf: number, len: number, ret: number) => number };
+let ntdll: Ntdll | null | undefined;
+
+const SYSTEM_PROCESS_INFORMATION = 5;
+const STATUS_INFO_LENGTH_MISMATCH = 0xc0000004 | 0;
+/** Grows on a length mismatch; a busy machine needs ~1-2 MB. */
+let bufferSize = 2 * 1024 * 1024;
+
+async function readWindows(pid: number): Promise<ThreadSnapshot | null> {
+  if (ntdll === undefined) {
+    try {
+      const { dlopen, FFIType } = await import("bun:ffi");
+      ntdll = dlopen("ntdll.dll", {
+        NtQuerySystemInformation: { args: [FFIType.i32, FFIType.ptr, FFIType.u32, FFIType.ptr], returns: FFIType.i32 },
+      }).symbols as unknown as Ntdll;
+    } catch {
+      ntdll = null;
+    }
+  }
+  if (!ntdll) return null;
+  const { ptr } = await import("bun:ffi");
+  for (let attempt = 0; attempt < 4; attempt++) {
+    const buf = new Uint8Array(bufferSize);
+    const needed = new Uint32Array(1);
+    const status = ntdll.NtQuerySystemInformation(SYSTEM_PROCESS_INFORMATION, ptr(buf), buf.byteLength, ptr(needed));
+    if (status === STATUS_INFO_LENGTH_MISMATCH) {
+      bufferSize = Math.max(bufferSize * 2, (needed[0] ?? 0) + 256 * 1024);
+      continue;
+    }
+    if (status !== 0) return null;
+    return parseWindowsProcessInformation(new DataView(buf.buffer), pid);
+  }
+  return null;
+}
+
+const LINUX_STATES: Record<string, string> = {
+  R: "running", S: "sleeping", D: "disk-wait", T: "stopped", t: "traced", Z: "zombie", X: "dead",
+  I: "idle", W: "paging", P: "parked",
+};
+
+/** `/proc/<pid>/stat`: state, threads, major faults, CPU. Exported for tests. */
+export function parseLinuxStat(text: string, clockTicksPerSecond = 100): ThreadSnapshot | null {
+  // The command name is in parentheses and may itself hold spaces or parentheses.
+  const close = text.lastIndexOf(")");
+  if (close === -1) return null;
+  const f = text.slice(close + 2).trim().split(/\s+/);
+  // f[0] is field 3 (state); field N is f[N - 3].
+  const state = f[0];
+  const majflt = Number(f[12 - 3]);
+  const utime = Number(f[14 - 3]);
+  const stime = Number(f[15 - 3]);
+  const threads = Number(f[20 - 3]);
+  const out: ThreadSnapshot = {};
+  if (state) out.state = LINUX_STATES[state] ?? state;
+  if (Number.isFinite(majflt)) out.hardFaults = majflt;
+  if (Number.isFinite(utime) && Number.isFinite(stime)) out.cpuMs = Math.round(((utime + stime) * 1000) / clockTicksPerSecond);
+  if (Number.isFinite(threads)) out.threads = threads;
+  return out;
+}
+
+function readLinux(pid: number): ThreadSnapshot | null {
+  const out = parseLinuxStat(readFileSync(`/proc/${pid}/stat`, "utf8"));
+  if (!out) return null;
+  // The process's own stat reports the main task's state; wchan says where it sleeps.
+  try {
+    const wchan = readFileSync(`/proc/${pid}/task/${pid}/wchan`, "utf8").trim();
+    if (wchan && wchan !== "0") out.waitReason = wchan;
+  } catch { /* wchan is optional, and hidden on some kernels */ }
+  return out;
+}
+
+/** One reading of `pid`. Null when the platform cannot be read or the process is gone. */
+export async function readThreadSnapshot(pid: number, platform: NodeJS.Platform = process.platform): Promise<ThreadSnapshot | null> {
+  try {
+    if (platform === "win32") return await readWindows(pid);
+    if (platform === "linux") return readLinux(pid);
+  } catch { /* fall through */ }
+  return null;
+}
+
+/** "main thread waiting/WrPageIn, hard faults +38, cpu +120ms" — relative to `previous` when given. */
+export function describeThreadSnapshot(now: ThreadSnapshot, previous?: ThreadSnapshot | null): string {
+  const parts: string[] = [];
+  if (now.state) parts.push(`main thread ${now.state}${now.waitReason ? `/${now.waitReason}` : ""}`);
+  if (now.hardFaults !== undefined) {
+    parts.push(previous?.hardFaults !== undefined ? `hard faults +${now.hardFaults - previous.hardFaults}` : `hard faults ${now.hardFaults}`);
+  }
+  if (now.cpuMs !== undefined && previous?.cpuMs !== undefined) parts.push(`server cpu +${now.cpuMs - previous.cpuMs}ms`);
+  return parts.join(", ");
+}

package/src/services/supervisor.ts

@@ -41,6 +41,9 @@ import {
 } from "./edge-target-resolver.ts";
 import { EDGE_PRIOR_DEATHS_ENV } from "./edge-forwarder.ts";
 import { PLIST_LABEL } from "./autostart-generator.ts";
+import { logPriorityRaise } from "./process-priority.ts";
+import { readThreadSnapshot, describeThreadSnapshot, type ThreadSnapshot } from "./process-thread-snapshot.ts";
+import { readMachineCpuTimes, machineBusyFraction, type MachineCpuTimes } from "./machine-cpu.ts";
 
 // ─── Constants ─────────────────────────────────────────────────────────
 const MAX_RESTARTS = 10;
@@ -1193,6 +1196,41 @@ async function runDbBackupTick(reason: "hourly" | "start"): Promise<void> {
 }
 
 // ─── Health checks ─────────────────────────────────────────────────────
+
+/**
+ * The server and the machine as of the last probe, so a failed probe can report rates
+ * over the 30s before it rather than totals since boot.
+ */
+let serverBaseline: { pid: number; thread: ThreadSnapshot | null; machine: MachineCpuTimes | null } | null = null;
+
+async function recordServerBaseline(pid: number | undefined): Promise<void> {
+  if (!pid) return;
+  serverBaseline = { pid, thread: await readThreadSnapshot(pid), machine: readMachineCpuTimes() };
+}
+
+/**
+ * What the server is doing while it does not answer: its main thread's wait, the
+ * page faults that went to disk and its CPU since the last probe, and how busy the
+ * machine was. A failed probe used to say only "timeout", which left a 60s stall with
+ * no cause on record once the server was restarted.
+ */
+async function describeUnresponsiveServer(pid: number | undefined): Promise<string> {
+  if (!pid) return "";
+  try {
+    const thread = await readThreadSnapshot(pid);
+    const machine = readMachineCpuTimes();
+    const before = serverBaseline?.pid === pid ? serverBaseline : null;
+    const parts: string[] = [];
+    if (thread) parts.push(describeThreadSnapshot(thread, before?.thread));
+    const busy = machineBusyFraction(before?.machine ?? null, machine);
+    if (busy !== null) parts.push(`machine cpu ${Math.round(busy * 100)}%`);
+    serverBaseline = { pid, thread, machine };
+    return parts.filter(Boolean).join(", ");
+  } catch {
+    return "";
+  }
+}
+
 function startServerHealthCheck() {
   healthTimer = setInterval(async () => {
     if (shuttingDown || getState() === "stopped") return;
@@ -1232,14 +1270,16 @@ function startServerHealthCheck() {
       const res = await fetch(`http://127.0.0.1:${checkPort}/api/health`, {
         signal: AbortSignal.timeout(5000),
       });
-      if (res.ok) { healthFailCount = 0; return; }
+      if (res.ok) { healthFailCount = 0; await recordServerBaseline(serverChild?.pid); return; }
       lastProbe = `HTTP ${res.status}`;
     } catch (e) {
       const err = e as { name?: string; code?: string; message?: string } | null;
       lastProbe = err?.name === "TimeoutError" ? "timeout after 5s" : err?.code ?? err?.message ?? String(e);
     }
     healthFailCount++;
-    log("DEBUG", `Server health probe on port ${checkPort} failed (${healthFailCount}/${SERVER_HEALTH_FAIL_THRESHOLD}): ${lastProbe}`);
+    // Read while the server is still not answering: the main thread's wait is the cause.
+    const waitingOn = await describeUnresponsiveServer(serverChild?.pid);
+    log("WARN", `Server health probe on port ${checkPort} failed (${healthFailCount}/${SERVER_HEALTH_FAIL_THRESHOLD}): ${lastProbe}${waitingOn ? ` — ${waitingOn}` : ""}`);
     if (healthFailCount >= SERVER_HEALTH_FAIL_THRESHOLD && serverChild) {
       log("ERROR", `Server unresponsive for ${healthFailCount}×${SERVER_HEALTH_INTERVAL_MS / 1000}s (last probe: ${lastProbe}) — killing PID ${serverChild.pid}`);
       const pid = serverChild.pid;
@@ -2020,6 +2060,8 @@ export async function runSupervisor(opts: {
   const logFd = openSync(logFile(), "a");
   startLogRotation();
   log("INFO", `Supervisor started (PID: ${process.pid}, port: ${opts.port}, share: ${opts.share})`);
+  // Before any child is spawned: every child inherits these, the server included.
+  await logPriorityRaise("supervisor", (m) => log("WARN", m));
 
   // ── Systemd self-heal: if the unit file is stale (e.g. still has
   // Restart=on-failure), we were likely spawned by the old Bun.spawn()

package/src/web/components/chat/question-card.tsx

@@ -134,14 +134,20 @@ function useQuestionKeyboard(config: {
     if (el) {
       el.addEventListener("keydown", handleKeyDown);
       el.setAttribute("tabindex", "0");
-      // Same reason: asking the main document whether focus is already inside the card answers
-      // "no" for a card living in another document, and this effect re-runs on every render —
-      // which would yank focus out of the custom input after each keystroke.
-      if (config.autoFocus && !el.contains(el.ownerDocument.activeElement)) el.focus();
     }
     return () => { el?.removeEventListener("keydown", handleKeyDown); };
   }, [config, focusedOption]);
 
+  // Focus the card once, when it appears. Doing it on every render took focus back whenever the
+  // card re-rendered for any reason, so with two chats side by side each waiting on a question,
+  // the other card's chat updating pulled focus out of the card being answered. The owner
+  // document is asked rather than the global one, for a card rendered in a picture-in-picture window.
+  const autoFocusOnMount = useRef(config.autoFocus);
+  useEffect(() => {
+    const el = containerRef.current;
+    if (autoFocusOnMount.current && el && !el.contains(el.ownerDocument.activeElement)) el.focus();
+  }, []);
+
   return { focusedOption, setFocusedOption, containerRef };
 }