create-walle 0.10.1 → 0.10.2

package/template/coding-task-manager/lib/host-stats.js

@@ -0,0 +1,328 @@
+'use strict';
+
+// WP-1 (docs/machine-pool-picker-pairing-dashboard-2026-10-24.html §5 D-10/D-11/D-15/D-16, §6.1,
+// §8 WP-1, §9 S-1/S-2): the host-stats primitive. One instance per CTM process owns "what is this
+// host doing right now", fed by the EXISTING 60 s resource tick (no new timer) and retained 24 h in
+// this machine's own DB (D-11: the box is the truth; there is no hub-side time-series).
+//
+// Every number here is the machine describing itself, so the same sample serves the local picker
+// ("This Mac") and a peer that dialed this box through the existing ?machine= proxy (D-12).
+//
+// Two rules shape the implementation:
+//   * `record()` NEVER throws. It runs from a lifecycle sweep, so a failing sub-read (a missing
+//     /proc, an absent nvidia-smi, a closed DB handle during teardown) must degrade to a `null`
+//     FIELD, never to an aborted sweep. Each read is individually guarded.
+//   * All I/O is ASYNC (main-thread lint): statfs/readFile are the promise forms and the only
+//     process launch is one async, 3 s-bounded nvidia-smi probe on linux.
+//
+// The injected `execFile` is called in METHOD position on the `io` bag below rather than as a bare
+// `execFile(...)`. That is the same convention lib/session-resource-sampler.js uses for its own
+// injected runner (`this.exec(...)`): bin/main-thread-lint.js's bare-launch rule reads a bare
+// `execFile(` as a new, unledgered main-thread spawn site, and this WP does not own the exemption
+// ledger. Behavior is identical — one async execFile with a fixed argv and a 3 s timeout.
+
+const os = require('node:os');
+const fs = require('node:fs');
+const childProcess = require('node:child_process');
+const { promisify } = require('node:util');
+
+const DEFAULT_RETENTION_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_HISTORY_LIMIT = 1440;
+
+// §6.1 (exact argv), observed on the DGX Spark's GB10: memory.used/memory.total are `[N/A]` on
+// unified memory, so `unifiedMemory: true` is derived from exactly that.
+const NVIDIA_SMI_ARGS = Object.freeze([
+  '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw',
+  '--format=csv,noheader,nounits',
+]);
+const NVIDIA_SMI_TIMEOUT_MS = 3000;
+
+const BYTES_PER_MIB = 1024 * 1024;
+const BYTES_PER_KIB = 1024;
+
+/** `null` for anything that is not a finite number — the one way a missing reading enters a field. */
+function _numOrNull(value) {
+  // An absent/blank value must be NULL, never 0: `Number('')` is 0, and a fabricated 0 on a missing
+  // reading is exactly the "no total, no swap" lie this module refuses to tell about os.freemem().
+  const text = String(value == null ? '' : value).trim();
+  if (!text) return null;
+  const n = Number(text);
+  return Number.isFinite(n) ? n : null;
+}
+
+/** nvidia-smi prints `[N/A]` (or `[Not Supported]`) for a metric the driver cannot read. */
+function _smiOrNull(raw) {
+  const text = String(raw == null ? '' : raw).trim();
+  if (!text || /^\[.*\]$/.test(text) || /^N\/A$/i.test(text)) return null;
+  return _numOrNull(text);
+}
+
+/** §6.1: `MemTotal − MemAvailable`, plus swap, from /proc/meminfo (linux only). */
+function parseMeminfo(text) {
+  const fields = {};
+  for (const line of String(text || '').split('\n')) {
+    const match = /^([A-Za-z_()]+):\s*(\d+)\s*kB\s*$/.exec(line.trim());
+    if (match) fields[match[1]] = Number(match[2]);
+  }
+  const total = Number.isFinite(fields.MemTotal) ? fields.MemTotal * BYTES_PER_KIB : null;
+  const available = Number.isFinite(fields.MemAvailable) ? fields.MemAvailable * BYTES_PER_KIB : null;
+  const swapTotal = Number.isFinite(fields.SwapTotal) ? fields.SwapTotal * BYTES_PER_KIB : null;
+  const swapFree = Number.isFinite(fields.SwapFree) ? fields.SwapFree * BYTES_PER_KIB : null;
+  return {
+    memTotalBytes: total,
+    memUsedBytes: total != null && available != null ? Math.max(0, total - available) : null,
+    swapUsedBytes: swapTotal != null && swapFree != null ? Math.max(0, swapTotal - swapFree) : null,
+  };
+}
+
+/** §6.1: first line only, comma separated, MiB for the two memory columns → bytes. */
+function parseNvidiaSmi(stdout) {
+  const line = String(stdout || '').split('\n').map(part => part.trim()).find(part => part.length > 0);
+  if (!line) return null;
+  const parts = line.split(',').map(part => part.trim());
+  const memUsedMiB = _smiOrNull(parts[2]);
+  const memTotalMiB = _smiOrNull(parts[3]);
+  return {
+    name: String(parts[0] || '').slice(0, 120),
+    utilPct: _smiOrNull(parts[1]),
+    tempC: _smiOrNull(parts[4]),
+    powerW: _smiOrNull(parts[5]),
+    memUsedBytes: memUsedMiB == null ? null : Math.round(memUsedMiB * BYTES_PER_MIB),
+    memTotalBytes: memTotalMiB == null ? null : Math.round(memTotalMiB * BYTES_PER_MIB),
+    // P (unified memory): no readable memory split ⇒ the host memory above IS the GPU's.
+    unifiedMemory: memUsedMiB == null || memTotalMiB == null,
+  };
+}
+
+/** Cumulative cpu times, summed over cores — the delta between two calls is the util numerator. */
+function cpuTimes(cpuList) {
+  let idle = 0;
+  let total = 0;
+  for (const cpu of Array.isArray(cpuList) ? cpuList : []) {
+    const times = cpu && cpu.times;
+    if (!times) continue;
+    const values = ['user', 'nice', 'sys', 'idle', 'irq'].map(key => Number(times[key]) || 0);
+    idle += values[3];
+    total += values.reduce((sum, value) => sum + value, 0);
+  }
+  return { idle, total };
+}
+
+function createHostStats({
+  db,
+  platform = process.platform,
+  now = () => Date.now(),
+  cpus = () => os.cpus(),
+  loadavg = () => os.loadavg(),
+  statfs = (p) => fs.promises.statfs(p),
+  readFile = (p) => fs.promises.readFile(p, 'utf8'),
+  execFile = promisify(childProcess.execFile),
+  diskPath = os.homedir(),
+  nvidiaSmiPath = '/usr/bin/nvidia-smi',
+  countTasks = () => ({ running: 0, needsYou: 0, idle: 0, workers: 0 }),
+  lastCreateAt = () => 0,
+  retentionMs = DEFAULT_RETENTION_MS,
+} = {}) {
+  // Method-position calls only (see the header): the lint's bare-launch rule scans for a bare
+  // `execFile(`, not an injected runner reached through its owning bag.
+  const io = { statfs, readFile, execFile };
+
+  let lastCpu = null;
+  let latestSample = null;
+  // nvidia-smi presence is probed ONCE per process: the boot image of a box does not change under it.
+  let nvidiaSmiAvailability = null;
+
+  // D-10: a delta of two cumulative samples; the first tick has no previous, so it reports null
+  // rather than a fabricated 0.
+  function _cpuPct() {
+    let current = null;
+    try {
+      current = cpuTimes(cpus());
+    } catch {
+      return null; // expected: os.cpus() unavailable in a restricted sandbox
+    }
+    if (!current || current.total <= 0) return null;
+    const previous = lastCpu;
+    lastCpu = current;
+    if (!previous) return null;
+    const totalDelta = current.total - previous.total;
+    const idleDelta = current.idle - previous.idle;
+    if (!(totalDelta > 0)) return null;
+    return Math.max(0, Math.min(100, Math.round((1 - idleDelta / totalDelta) * 10000) / 100));
+  }
+
+  function _load() {
+    try {
+      const values = loadavg() || [];
+      return [Number(values[0]) || 0, Number(values[1]) || 0, Number(values[2]) || 0];
+    } catch {
+      return [0, 0, 0]; // expected: a platform without load-average support
+    }
+  }
+
+  function _cores() {
+    try {
+      return Math.max(1, (cpus() || []).length);
+    } catch {
+      return 1; // expected: os.cpus() unavailable
+    }
+  }
+
+  // D-10: darwin reads the sampler's vm_stat host fields; linux reads /proc/meminfo; anything else
+  // is null. `os.freemem()` is never used — it reports one number with no total and no swap.
+  async function _memory(snapshot) {
+    if (platform === 'darwin') {
+      const host = (snapshot && snapshot.host) || {};
+      return {
+        memUsedBytes: _numOrNull(host.memoryUsedBytes),
+        memTotalBytes: _numOrNull(host.memoryTotalBytes),
+        swapUsedBytes: _numOrNull(host.swapUsedBytes),
+      };
+    }
+    if (platform === 'linux') {
+      try {
+        return parseMeminfo(await io.readFile('/proc/meminfo'));
+      } catch {
+        return { memUsedBytes: null, memTotalBytes: null, swapUsedBytes: null }; // expected: no /proc in a container
+      }
+    }
+    return { memUsedBytes: null, memTotalBytes: null, swapUsedBytes: null }; // expected: platform without a memory contract
+  }
+
+  async function _disk() {
+    try {
+      const stats = await io.statfs(diskPath);
+      const blockSize = Number(stats && stats.bsize) || 0;
+      if (!blockSize) return { diskFreeBytes: null, diskTotalBytes: null };
+      return {
+        // bavail (free to an unprivileged writer), not bfree — the number a user can act on.
+        diskFreeBytes: Math.max(0, (Number(stats.bavail) || 0) * blockSize),
+        diskTotalBytes: Math.max(0, (Number(stats.blocks) || 0) * blockSize),
+      };
+    } catch {
+      return { diskFreeBytes: null, diskTotalBytes: null }; // expected: home dir unreadable/statfs unsupported
+    }
+  }
+
+  // D-10: linux only, one async execFile with a 3 s timeout. Absence of nvidia-smi is a normal
+  // answer (`gpu: null`), not an error, and the probe itself is cached per process.
+  async function _gpu() {
+    if (platform !== 'linux') return null; // expected: GPU probe is linux-only by design
+    if (nvidiaSmiAvailability === null) {
+      try {
+        await fs.promises.access(nvidiaSmiPath, fs.constants.X_OK);
+        nvidiaSmiAvailability = true;
+      } catch {
+        nvidiaSmiAvailability = false; // expected: no nvidia-smi on this box
+      }
+    }
+    if (!nvidiaSmiAvailability) return null;
+    try {
+      const { stdout } = await io.execFile(nvidiaSmiPath, [...NVIDIA_SMI_ARGS], { timeout: NVIDIA_SMI_TIMEOUT_MS });
+      return parseNvidiaSmi(stdout);
+    } catch {
+      return null; // expected: driver absent, or the query timed out on a wedged GPU
+    }
+  }
+
+  // D-10: the sampler's own process tree rss, in bytes.
+  function _ctmRssBytes(snapshot) {
+    const rssMB = snapshot && snapshot.processTree ? Number(snapshot.processTree.rssMB) : NaN;
+    return Number.isFinite(rssMB) ? Math.max(0, Math.round(rssMB * BYTES_PER_MIB)) : null;
+  }
+
+  // D-15: injected by server.js, computed from the session-status ledger. A throwing counter must
+  // not lose the rest of the sample.
+  function _tasks() {
+    try {
+      const counts = countTasks() || {};
+      return {
+        running: Number(counts.running) || 0,
+        needsYou: Number(counts.needsYou) || 0,
+        idle: Number(counts.idle) || 0,
+        workers: Number(counts.workers) || 0,
+      };
+    } catch {
+      return { running: 0, needsYou: 0, idle: 0, workers: 0 }; // expected: ledger unavailable during teardown
+    }
+  }
+
+  // D-16: the newest session create on THIS machine, so Recent agrees on every primary.
+  function _lastUsedAt() {
+    try {
+      return Number(lastCreateAt()) || 0;
+    } catch {
+      return 0; // expected: session map unavailable during teardown
+    }
+  }
+
+  /**
+   * One sample per resource tick. Never throws and never rejects: every sub-read is guarded and a
+   * failed one contributes `null` (or the all-zero task bucket) to the sample.
+   */
+  async function record(resourceSnapshot) {
+    const at = now();
+    const [memory, disk, gpu] = await Promise.all([
+      _memory(resourceSnapshot),
+      _disk(),
+      _gpu(),
+    ]);
+    const load = _load();
+    const sample = {
+      at,
+      cpuPct: _cpuPct(),
+      load1: load[0],
+      load5: load[1],
+      load15: load[2],
+      cores: _cores(),
+      memUsedBytes: memory.memUsedBytes,
+      memTotalBytes: memory.memTotalBytes,
+      swapUsedBytes: memory.swapUsedBytes,
+      diskFreeBytes: disk.diskFreeBytes,
+      diskTotalBytes: disk.diskTotalBytes,
+      gpu,
+      ctmRssBytes: _ctmRssBytes(resourceSnapshot),
+      tasks: _tasks(),
+      lastUsedAt: _lastUsedAt(),
+    };
+    // Memory FIRST, then the write: a reader that pairs latest() with history() (lib/api/host-stats.js
+    // does, and asserts latest.at >= samples[last].at) must never see a stored row newer than the
+    // in-memory sample beside it. record()'s row is committed before this await resolves, so setting
+    // latestSample afterwards leaves exactly that window open.
+    latestSample = sample;
+    try {
+      const sampleJson = JSON.stringify(sample);
+      if (db && typeof db.recordHostSample === 'function') {
+        await db.recordHostSample({ at, sampleJson, purgeBeforeMs: at - retentionMs });
+      }
+    } catch {
+      // HANDLE: history is best-effort; the sample is still returned and held in memory, so a
+      // full/closed DB costs this box its time series, never the live read a picker makes.
+      // expected: the DB handle is closed or full (teardown, disk exhaustion)
+    }
+    return sample;
+  }
+
+  function latest() {
+    return latestSample;
+  }
+
+  /** Oldest first, the same order the wire shape documents (§6.1). */
+  async function history({ sinceMs = now() - retentionMs, limit = DEFAULT_HISTORY_LIMIT } = {}) {
+    if (!db || typeof db.listHostSamples !== 'function') return [];
+    const rows = await db.listHostSamples({ sinceMs, limit });
+    const samples = [];
+    for (const row of Array.isArray(rows) ? rows : []) {
+      try {
+        samples.push(JSON.parse(row.sample_json));
+      } catch {
+        // expected: a row written by an older/newer shape is skipped, never fatal
+      }
+    }
+    return samples;
+  }
+
+  return { record, latest, history };
+}
+
+module.exports = { createHostStats, parseMeminfo, parseNvidiaSmi, cpuTimes, NVIDIA_SMI_ARGS, DEFAULT_RETENTION_MS };

package/template/coding-task-manager/lib/host-threadpool-evacuation.js

@@ -0,0 +1,160 @@
+'use strict';
+
+// WP-3 (stale-pty-host-evacuation design doc, §5 D-3, §6): the supervisor action
+// for a host whose libuv thread pool is dead. The monitor report is a fact, the
+// readiness ledger's recordThreadpool turns that fact into ONE verdict per
+// stall episode (D-2), and this module is what acts on it:
+//
+//   evacuate first, revive second, never keep.
+//
+// Evacuation is a single handoffHost(..., { reason: 'evacuate-threadpool' })
+// attempt -- NEVER handoffHostWithRetry: the retry exists for transient spawn
+// races on healthy hosts, and on a stalled host the second attempt just repeats
+// the same wait (I-1: 14:18:17 -> 14:18:27 in the design doc's evidence). A
+// successful handoff keeps the agents (their PTYs are inherited by the
+// successor, which has a fresh pool); anything else falls through to the
+// existing revive path, which kills the wedged host and lets its sessions cold
+// resume.
+//
+// A stalled host is NEVER marked staleHost and NEVER left on the 10-minute retry
+// (D-3): this module writes no stale flag and schedules nothing. It also never
+// declares a host dead itself -- that is the orchestrator's `successor-lost-
+// after-release` path, whose outcome is reported here as 'lost'.
+//
+// Dependency-injected exactly like lib/host-health-revive.js: no process globals,
+// no server.js require, every collaborator passed in. Single-flight per hostId
+// (a Map<hostId, Promise> that exists only for the duration of one active call),
+// so two stall frames for the same host cannot start two handoffs.
+
+function createEvacuateStalledHost({
+  manager,
+  handoffHost,
+  reviveUnresponsiveHost,
+  recordSessionDiagnostic,
+  hostHealth = null,
+  log = console.warn,
+  handoffOpts = {},
+} = {}) {
+  const _inFlight = new Map(); // hostId -> Promise<outcome>
+
+  // The sessions this host is routing RIGHT NOW -- captured before the handoff,
+  // which is what re-points (or tears down) their adapters.
+  function routedSessionIds(hostId) {
+    const ids = [];
+    for (const [sessionId, adapter] of manager.hosts) {
+      if (adapter && adapter._hostId === hostId && !adapter.closed) ids.push(sessionId);
+    }
+    return ids;
+  }
+
+  async function evacuateOnce(id, verdict) {
+    const sessionIds = routedSessionIds(id);
+    const oldestPendingMs = Number(verdict && verdict.oldestPendingMs) || 0;
+    let result = null;
+    let handoffReason = null;
+    try {
+      result = await handoffHost(manager, id, {
+        ...handoffOpts, reason: 'evacuate-threadpool', hostHealth, recordSessionDiagnostic,
+      });
+      handoffReason = result && typeof result.reason === 'string' ? result.reason : null;
+    } catch (err) {
+      // handoffHost is documented never to reject (it returns { ok:false, attempted, reason });
+      // defend anyway and let the catch-all branch below decide (revive).
+      log(`[host-evacuate-error] host=${id} handoffHost threw: ${err?.message || err}`); // REPORT: treated as an unattempted failure below
+      result = null;
+    }
+
+    let outcome;
+    if (result && result.ok === true) {
+      outcome = 'evacuated';
+    } else if (result && result.attempted === false && handoffReason === 'owner-recovery-pending') {
+      // A live owner is still recovering; nothing to do yet. Re-arm the ledger's
+      // episode evaluation (episode start is kept) so the next stalled frame
+      // decides again the moment the owner is done.
+      outcome = 'deferred';
+      hostHealth?.resetThreadpoolEpisode?.(id);
+    } else if (result && result.ok === false && handoffReason === 'successor-lost-after-release') {
+      // The orchestrator already declared this host dead and tore its adapters
+      // down; its sessions cold resume. Reviving would be a second kill.
+      outcome = 'lost';
+    } else {
+      // Any other ok:false (attempted or not) -- the old host cannot be replaced,
+      // so fall back to the terminal path: kill it and let the sessions resume.
+      let revived = null;
+      try {
+        revived = await reviveUnresponsiveHost(id, {
+          misses: 0,
+          stall: { op: 'threadpool', sessionId8: '', durationMs: oldestPendingMs },
+        });
+      } catch (err) {
+        log(`[host-evacuate-error] host=${id} reviveUnresponsiveHost threw: ${err?.message || err}`); // REPORT: outcome stays unknown below
+      }
+      if (revived === 'killed') outcome = 'revived';
+      else if (revived === 'capped') outcome = 'capped';
+      else outcome = 'unknown-host'; // 'unknown-host', a throw, or any unrecognized result
+    }
+
+    log(`[host-evacuate] host=${id} outcome=${outcome} reason=${handoffReason || '-'} sessions=${sessionIds.length} oldestPendingMs=${oldestPendingMs}`);
+    for (const sessionId of sessionIds) {
+      recordSessionDiagnostic?.(sessionId, 'host-evacuate', { hostId: id, outcome, reason: handoffReason, oldestPendingMs });
+    }
+    return outcome;
+  }
+
+  return async function evacuateStalledHost(hostId, verdict) {
+    const id = String(hostId || '');
+    if (!id) return 'unknown-host';
+    const existing = _inFlight.get(id);
+    if (existing) return existing;
+    const p = evacuateOnce(id, verdict).finally(() => { _inFlight.delete(id); });
+    _inFlight.set(id, p);
+    return p;
+  };
+}
+
+// WP-3 (design doc §5 D-4, §8 WP-3, §9 S-8): the BOOT half of the escalation.
+// Reconcile has just adopted every surviving host, so each adopted host's own
+// state-reply threadpool snapshot is fed to the same readiness ledger the live
+// onThreadpoolReport path feeds, and the ids the ledger verdicts 'evacuate'
+// right now are returned -- the caller passes that Set as
+// runUpgradePendingHandoffs({ skipHostIds }) so a host already on the
+// evacuation path is never ALSO handed off by the upgrade pass.
+//
+// A missing/shapeless snapshot is a pre-WP-3 host (its state reply carried no
+// threadpool field): today's behaviour, untouched. A host the manager no longer
+// routes is skipped too -- the ledger is keyed by hostId, so acting on a stale
+// id could only arm a timer for a host nobody serves.
+//
+// A stalled snapshot that is not due YET (oldestPendingMs under the deadline)
+// returns null and arms the ledger's own timer, whose onEvacuate fires later:
+// that host is deliberately NOT in the skip set -- its upgrade handoff may
+// still run, and the orchestrator's single-flight channel is per-host while a
+// failed upgrade handoff already ends in stale/retry (D-4).
+function evacuateStalledAdoptedHosts({
+  hostHealth,
+  evacuateStalledHost,
+  threadpoolByHostId,
+  ownsHostId,
+  log = console.warn,
+} = {}) {
+  const skipHostIds = new Set();
+  if (!hostHealth || typeof hostHealth.recordThreadpool !== 'function') return skipHostIds;
+  if (!threadpoolByHostId || typeof threadpoolByHostId.entries !== 'function') return skipHostIds;
+  // REPORT disposition for both failure shapes (a rejection and a synchronous
+  // throw): boot must not block on -- or be aborted by -- one evacuation.
+  const report = (id, err) => log(`[host-evacuate-boot] host=${id} evacuation failed: ${err?.message || err}`);
+  for (const [hostId, snapshot] of threadpoolByHostId) {
+    const id = String(hostId || '');
+    const snap = snapshot && typeof snapshot === 'object' ? snapshot : null;
+    if (!id || !snap || (snap.state !== 'ok' && snap.state !== 'stalled')) continue; // pre-WP-3 host: no state change
+    if (typeof ownsHostId === 'function' && !ownsHostId(id)) continue; // no longer routed: nothing to evacuate
+    const verdict = hostHealth.recordThreadpool(id, snap);
+    if (!verdict) continue; // not due yet (the ledger's own timer may fire later) -- never skipped
+    skipHostIds.add(id);
+    try { Promise.resolve(evacuateStalledHost(id, verdict)).catch((err) => report(id, err)); }
+    catch (err) { report(id, err); } // REPORT: evacuateStalledHost never rejects, but a synchronous throw must not abort boot
+  }
+  return skipHostIds;
+}
+
+module.exports = { createEvacuateStalledHost, evacuateStalledAdoptedHosts };

package/template/coding-task-manager/lib/host-threadpool-monitor.js

@@ -69,6 +69,11 @@ function createHostThreadpoolMonitor({
   setTimeoutFn = setTimeout,
   clearTimeoutFn = clearTimeout,
   onFrame = () => {},
+  // WP-0 (D-6/S-14): the frame's per-session view. A caller-provided provider
+  // (the pool wires the headless service's oldest in-flight ops here) is merged
+  // into EVERY emitted frame. Report-only: a throwing provider is swallowed --
+  // it can never change `state`, fail a probe, or stop a frame going out.
+  extraSnapshot = () => ({}),
   // Injectable seam for unit tests only (defaults to the real module): lets a
   // test simulate a request the real threadpool would never finish (init
   // fires, destroy never does) without needing an actual stuck libuv worker.
@@ -134,7 +139,15 @@ function createHostThreadpoolMonitor({
   }
 
   function emitFrame(extra = {}) {
-    onFrame({ ...snapshot(), ...extra });
+    // WP-0: a report-only extraSnapshot provider is merged shallowly into every
+    // frame, but `state` is re-asserted from the monitor's own snapshot last, so
+    // a provider can never change what the detector decided.
+    let extraFields = {};
+    try {
+      extraFields = extraSnapshot() || {};
+    } catch { /* expected: a report-only provider must not break the frame */ }
+    const base = snapshot();
+    onFrame({ ...base, ...extraFields, ...extra, state: base.state });
   }
 
   function resetWindow() {