@pleri/olam-cli 0.1.196 → 0.1.199

package/host-cp/src/world-watchdog-probes.mjs

@@ -0,0 +1,271 @@
+/**
+ * world-watchdog-probes.mjs — pure probe functions for the world watchdog.
+ *
+ * Three readers extract raw signals from the Linux /proc filesystem:
+ *   - readWchan(pid, opts)            → string | null
+ *   - readCloseWaitSockets(pid, opts) → Array<{remoteIp, remotePort}>
+ *   - readCpuPercent(pid, windowMs, opts) → number | null
+ *
+ * One pure classifier turns those signals into a verdict:
+ *   - classify({ wchan, closeWaitCount, cpuPercent }) → 'healthy'|'suspect'|'wedged'
+ *
+ * All readers are fail-soft: any I/O error or parse error returns
+ * null / [] / 0 rather than throwing. The classifier treats null inputs as
+ * the signal not firing (conservative — only promotes to 'wedged' when all
+ * three signals are conclusive).
+ *
+ * Test injection: pass `opts.procRoot` to redirect /proc reads to a fixture
+ * directory (e.g. src/__tests__/fixtures/proc-gold-elk-5574/).
+ *
+ * CLOSE_WAIT threshold note (deviation from D2): Decision D2 specifies
+ * filtering CLOSE_WAIT by peer hostname (*.anthropic.com | auth-worker.*).
+ * DNS resolution at every tick is unreliable under network stress (exactly
+ * when the watchdog must be most accurate). The gold-elk-5574 forensic data
+ * shows ≥3 CLOSE_WAIT to ANY peer is already diagnostic — a healthy claude
+ * process has 0-1 CLOSE_WAIT sockets under normal operation. The classifier
+ * therefore uses count ≥ 3 without hostname filtering. This deviation is
+ * documented in docs/architecture/world-watchdog.md Signal 2.
+ *
+ * @see docs/architecture/world-watchdog.md
+ * @see packages/host-cp/src/__tests__/world-watchdog-probes.test.mjs
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+// HZ — Linux scheduler tick rate. Kernel default is 100; can be 250 or 1000
+// on tickless kernels but the /proc/stat jiffies-to-seconds conversion is
+// independent of the actual HZ when the denominator is wall-clock ms.
+// We divide jiffies by HZ to get seconds of CPU time, then compare to the
+// wall-clock window. HZ=100 is correct for virtually all container environments.
+const LINUX_HZ = 100;
+
+// /proc/net/tcp state byte for CLOSE_WAIT.
+const CLOSE_WAIT_STATE = '08';
+
+/**
+ * Read the wchan (wait channel) of a process's main thread.
+ *
+ * @param {number|string} pid   Process ID.
+ * @param {{ procRoot?: string }} [opts]
+ *   `procRoot` defaults to '/proc'; override for tests.
+ * @returns {Promise<string|null>}
+ *   The wchan string (e.g. 'futex_wait_queue', 'epoll_wait') or null on error.
+ */
+export async function readWchan(pid, opts = {}) {
+  const procRoot = opts.procRoot ?? '/proc';
+  const wchanPath = path.join(procRoot, String(pid), 'wchan');
+  try {
+    const content = await fs.readFile(wchanPath, 'utf8');
+    return content.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read CLOSE_WAIT sockets for a process from /proc/<pid>/net/tcp (and tcp6).
+ *
+ * Parses the /proc/net/tcp format (space-separated hex fields). State field
+ * (column index 3, 0-based) == '08' means CLOSE_WAIT. Returns all matching
+ * entries regardless of remote peer — see module JSDoc for rationale.
+ *
+ * @param {number|string} pid   Process ID.
+ * @param {{ procRoot?: string }} [opts]
+ * @returns {Promise<Array<{remoteIp: string, remotePort: number}>>}
+ *   Array of CLOSE_WAIT socket descriptors, empty on error or no matches.
+ */
+export async function readCloseWaitSockets(pid, opts = {}) {
+  const procRoot = opts.procRoot ?? '/proc';
+  const results = [];
+
+  for (const proto of ['tcp', 'tcp6']) {
+    const tcpPath = path.join(procRoot, String(pid), 'net', proto);
+    let content;
+    try {
+      content = await fs.readFile(tcpPath, 'utf8');
+    } catch {
+      // ENOENT: pid gone or proto not available — skip, not an error.
+      continue;
+    }
+
+    const lines = content.split('\n');
+    // Skip header line.
+    for (let i = 1; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (!line) continue;
+      const fields = line.split(/\s+/);
+      // /proc/net/tcp columns (0-based):
+      //   0: sl
+      //   1: local_address (hex IP:port)
+      //   2: rem_address   (hex IP:port)
+      //   3: st            (hex state)
+      if (fields.length < 4) continue;
+      const state = fields[3];
+      if (state !== CLOSE_WAIT_STATE) continue;
+
+      const remAddr = fields[2];
+      const colonIdx = remAddr.lastIndexOf(':');
+      if (colonIdx === -1) continue;
+      const remIpHex = remAddr.slice(0, colonIdx);
+      const remPortHex = remAddr.slice(colonIdx + 1);
+
+      const remIp = parseHexIp(remIpHex);
+      const remPort = parseInt(remPortHex, 16);
+
+      if (remIp !== null && Number.isFinite(remPort)) {
+        results.push({ remoteIp: remIp, remotePort: remPort });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Measure CPU utilisation for a process over a time window.
+ *
+ * Reads /proc/<pid>/stat twice (before + after `windowMs` ms) and computes:
+ *   cpuPercent = (utime+stime delta) / (HZ * windowMs / 1000) * 100
+ *
+ * @param {number|string} pid     Process ID.
+ * @param {number} windowMs       Measurement window in milliseconds.
+ * @param {{ procRoot?: string, sleep?: (ms: number) => Promise<void>, now?: () => number }} [opts]
+ *   `sleep`  — injectable delay function (default: real setTimeout).
+ *   `now`    — injectable clock (default: Date.now).
+ *   `procRoot` — injectable proc root for tests.
+ * @returns {Promise<number|null>}
+ *   CPU percent (0–100+) or null on read/parse error.
+ */
+export async function readCpuPercent(pid, windowMs, opts = {}) {
+  const procRoot = opts.procRoot ?? '/proc';
+  const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  const statPath = path.join(procRoot, String(pid), 'stat');
+
+  const before = await readStatTimes(statPath);
+  if (before === null) return null;
+
+  await sleep(windowMs);
+
+  const after = await readStatTimes(statPath);
+  if (after === null) return null;
+
+  const deltaTicks = (after.utime + after.stime) - (before.utime + before.stime);
+  if (deltaTicks < 0) return null;
+
+  // deltaTicks jiffies / HZ = delta CPU-seconds.
+  // windowMs / 1000 = window in seconds.
+  const windowSec = windowMs / 1000;
+  if (windowSec <= 0) return null;
+
+  const cpuPercent = (deltaTicks / LINUX_HZ / windowSec) * 100;
+  return cpuPercent;
+}
+
+// ── Internal helpers ──────────────────────────────────────────────────────────
+
+/**
+ * Parse utime + stime from /proc/<pid>/stat content.
+ *
+ * @param {string} statPath
+ * @returns {Promise<{utime: number, stime: number}|null>}
+ */
+async function readStatTimes(statPath) {
+  let content;
+  try {
+    content = await fs.readFile(statPath, 'utf8');
+  } catch {
+    return null;
+  }
+
+  // The stat format is: pid (comm) state ppid pgroup session ... utime stime ...
+  // The command name (field 2) can contain spaces and parentheses, so we
+  // find the last ')' to reliably locate the fields that follow.
+  const parenClose = content.lastIndexOf(')');
+  if (parenClose === -1) return null;
+
+  // After the closing ')', fields are space-separated starting with ' state'.
+  // Fields after ')' (0-indexed):
+  //   0: state, 1: ppid, 2: pgrp, 3: session, 4: tty_nr, 5: tpgid,
+  //   6: flags, 7: minflt, 8: cminflt, 9: majflt, 10: cmajflt,
+  //   11: utime, 12: stime (i.e. indices 11+12 from the post-paren split)
+  const afterParen = content.slice(parenClose + 1).trim();
+  const fields = afterParen.split(/\s+/);
+  // utime = fields[11], stime = fields[12]
+  if (fields.length < 13) return null;
+
+  const utime = parseInt(fields[11], 10);
+  const stime = parseInt(fields[12], 10);
+
+  if (!Number.isFinite(utime) || !Number.isFinite(stime)) return null;
+  return { utime, stime };
+}
+
+/**
+ * Parse a hex-encoded IP address from /proc/net/tcp format.
+ *
+ * IPv4: 8 hex chars in little-endian byte order (e.g. "0101007F" → "127.0.0.1").
+ * IPv6: 32 hex chars (4 groups of 8, each in little-endian).
+ *
+ * @param {string} hexIp
+ * @returns {string|null}
+ */
+function parseHexIp(hexIp) {
+  if (hexIp.length === 8) {
+    // IPv4: stored as little-endian 32-bit integer.
+    const b = [
+      parseInt(hexIp.slice(6, 8), 16),
+      parseInt(hexIp.slice(4, 6), 16),
+      parseInt(hexIp.slice(2, 4), 16),
+      parseInt(hexIp.slice(0, 2), 16),
+    ];
+    if (b.some((x) => !Number.isFinite(x))) return null;
+    return b.join('.');
+  }
+  if (hexIp.length === 32) {
+    // IPv6: 4 groups of 8 hex chars, each group little-endian.
+    const groups = [];
+    for (let g = 0; g < 4; g++) {
+      const chunk = hexIp.slice(g * 8, g * 8 + 8);
+      // Reverse byte order within each 32-bit group.
+      const bytes = [
+        chunk.slice(6, 8),
+        chunk.slice(4, 6),
+        chunk.slice(2, 4),
+        chunk.slice(0, 2),
+      ];
+      // Pair bytes into 16-bit groups for IPv6 notation.
+      groups.push(bytes[0] + bytes[1], bytes[2] + bytes[3]);
+    }
+    return groups.join(':');
+  }
+  return null;
+}
+
+// ── Classifier ───────────────────────────────────────────────────────────────
+
+/**
+ * @typedef {'healthy'|'suspect'|'wedged'} WatchdogVerdict
+ */
+
+/**
+ * Classify a set of probe signals into a watchdog verdict.
+ *
+ * AND-gate: all three of (wchan=futex_wait_queue, closeWaitCount≥3, cpuPercent<1)
+ * must fire for 'wedged'. Any subset → 'suspect'. None → 'healthy'.
+ * Null inputs are treated as not-firing (fail-soft).
+ *
+ * @param {{ wchan: string|null, closeWaitCount: number|null, cpuPercent: number|null }} signals
+ * @returns {WatchdogVerdict}
+ */
+export function classify({ wchan, closeWaitCount, cpuPercent }) {
+  const wchanFires = wchan === 'futex_wait_queue';
+  const closeWaitFires = typeof closeWaitCount === 'number' && closeWaitCount >= 3;
+  const cpuFires = typeof cpuPercent === 'number' && cpuPercent < 1;
+
+  const firingCount = (wchanFires ? 1 : 0) + (closeWaitFires ? 1 : 0) + (cpuFires ? 1 : 0);
+
+  if (firingCount === 3) return 'wedged';
+  if (firingCount > 0) return 'suspect';
+  return 'healthy';
+}

package/host-cp/src/world-watchdog-recovery.mjs

@@ -0,0 +1,192 @@
+/**
+ * world-watchdog-recovery.mjs — recovery hook for wedged claude processes.
+ *
+ * Isolated from world-watchdog.mjs so kill + replay logic is independently
+ * mockable in tests without touching the watchdog's ticker.
+ *
+ * API:
+ *   createRecovery({ autoRecoverMode, leakyBucket, broadcaster, persister,
+ *                    replay, processKill, log })
+ *     → { onWedgedVerdict({ worldId, pid }): Promise<void> }
+ *
+ * Three modes (from compute.autoRecover in .olam/config.yaml):
+ *   false      — no-op; recovery never fires even on wedged verdict (DEFAULT)
+ *   'dry-run'  — emits all breadcrumbs, never calls processKill or replay
+ *   true       — SIGKILL pid + read last-dispatch + replay; rate-limited
+ *
+ * Rate-limit: B2 leaky-bucket (3/hour/world). 4th wedge in window emits
+ * world.watchdog.recovery.budget_exhausted and skips all action.
+ *
+ * Replay stub: the `replay` dep is accepted as an injected function. In
+ * server.mjs it is wired to a console.warn stub + breadcrumb until the
+ * operator runs the B3 idempotence probe and signs off. See TODO below.
+ *
+ * @see docs/architecture/world-watchdog.md Recovery section
+ * @see packages/host-cp/src/lib/leaky-bucket.mjs
+ * @see packages/host-cp/src/dispatch-persister.mjs
+ */
+
+/**
+ * @typedef {'false'|true|'dry-run'} AutoRecoverMode
+ */
+
+/**
+ * @typedef {object} RecoveryDeps
+ * @property {false|true|'dry-run'} autoRecoverMode
+ *   Passed from server.mjs which reads config.compute.autoRecover.
+ *   Default false if config unavailable.
+ * @property {{ tryConsume(key: string): { allowed: boolean, retryAfterMs?: number, totalInWindow: number } }} leakyBucket
+ *   B2 leaky-bucket instance. Keyed by worldId.
+ * @property {{ broadcast(type: string, payload: object): void }} [broadcaster]
+ *   Host-stream broadcaster. Optional — when absent, breadcrumbs are skipped.
+ * @property {{ read({ worldId: string }): Promise<{ messageId: string, prompt: string, dispatchedAt: string, source: string } | null> }} persister
+ *   B4 dispatch-persister read function.
+ * @property {(opts: { worldId: string, messageId: string, prompt: string }) => Promise<void>} replay
+ *   Opaque dispatch helper. Injected dep — DO NOT implement dispatch here.
+ *   In server.mjs this is wired to a stub until operator signs off on B3 probe.
+ * @property {(pid: number) => void} [processKill]
+ *   process.kill indirection so tests can spy without actually killing.
+ *   Defaults to process.kill.
+ * @property {(msg: string) => void} [log]
+ *   Logger. Defaults to console.log with [world-watchdog-recovery] prefix.
+ */
+
+/**
+ * @typedef {object} RecoveryHandle
+ * @property {(opts: { worldId: string, pid: number|null }) => Promise<void>} onWedgedVerdict
+ */
+
+/**
+ * Create a recovery handle.
+ *
+ * @param {RecoveryDeps} deps
+ * @returns {RecoveryHandle}
+ */
+export function createRecovery({
+  autoRecoverMode = false,
+  leakyBucket,
+  broadcaster = null,
+  persister,
+  replay,
+  processKill = (pid) => process.kill(pid, 'SIGKILL'),
+  log = (m) => console.log(`[world-watchdog-recovery] ${m}`),
+} = {}) {
+  /**
+   * Emit a breadcrumb via broadcaster (fail-soft).
+   *
+   * @param {string} type
+   * @param {object} payload
+   */
+  function broadcast(type, payload) {
+    if (!broadcaster || typeof broadcaster.broadcast !== 'function') return;
+    try {
+      broadcaster.broadcast(type, payload);
+    } catch (err) {
+      log(`broadcast ${type} failed: ${err?.message ?? err}`);
+    }
+  }
+
+  /**
+   * Handle a 2-tick-confirmed wedged verdict for a world.
+   *
+   * Called by world-watchdog.mjs on verdict-transition only (suspect → wedged),
+   * NOT on steady-state re-wedge.
+   *
+   * @param {{ worldId: string, pid: number|null }} opts
+   * @returns {Promise<void>}
+   */
+  async function onWedgedVerdict({ worldId, pid }) {
+    // mode=false → detection-only; never act.
+    if (autoRecoverMode === false) return;
+
+    // PID null → watchdog hasn't resolved a real PID yet (Phase A stub case);
+    // skip silently — there is nothing to kill.
+    if (pid === null) return;
+
+    // Rate-limit gate.
+    const bucket = leakyBucket.tryConsume(worldId);
+    if (!bucket.allowed) {
+      broadcast('world.watchdog.recovery.budget_exhausted', {
+        worldId,
+        retryAfterMs: bucket.retryAfterMs,
+        totalInWindow: bucket.totalInWindow,
+      });
+      log(`worldId=${worldId}: budget exhausted (${bucket.totalInWindow} in window); skipping recovery`);
+      return;
+    }
+
+    // Read last persisted dispatch for replay.
+    let lastDispatch = null;
+    try {
+      lastDispatch = await persister.read({ worldId });
+    } catch (err) {
+      log(`worldId=${worldId}: persister.read failed: ${err?.message ?? err}`);
+    }
+
+    broadcast('world.watchdog.recovery.start', {
+      worldId,
+      pid,
+      mode: autoRecoverMode,
+      lastDispatchMessageId: lastDispatch?.messageId ?? null,
+    });
+
+    // dry-run — log planned action but do NOT kill.
+    if (autoRecoverMode === 'dry-run') {
+      log(`worldId=${worldId}: dry-run — would SIGKILL pid=${pid}${lastDispatch ? ` + replay messageId=${lastDispatch.messageId}` : ' (no last-dispatch)'}`);
+      broadcast('world.watchdog.recovery.complete', {
+        worldId,
+        pid,
+        mode: 'dry-run',
+        replayed: false,
+      });
+      return;
+    }
+
+    // mode=true — act.
+    try {
+      // 1. SIGKILL the wedged process.
+      processKill(pid);
+      log(`worldId=${worldId}: SIGKILL sent to pid=${pid}`);
+
+      // 2. Replay or note absence of last-dispatch.
+      if (!lastDispatch) {
+        broadcast('world.watchdog.recovery.restart_without_replay', {
+          worldId,
+          pid,
+        });
+        log(`worldId=${worldId}: no last-dispatch; killed without replay`);
+      } else {
+        // TODO: wire real replay once operator has run the B3 idempotence probe
+        // and confirmed dispatch is idempotent for the substrates in use.
+        // Until then this stub logs and emits a breadcrumb so the stub path
+        // is visible in production logs. See B3 probe + operator review gate B6.
+        broadcast('world.watchdog.recovery.replay_stub', {
+          worldId,
+          prompt: lastDispatch.prompt,
+        });
+        log(`worldId=${worldId}: replay stub hit — real replay deferred pending B3 sign-off`);
+        await replay({
+          worldId,
+          messageId: lastDispatch.messageId,
+          prompt: lastDispatch.prompt,
+        });
+      }
+
+      broadcast('world.watchdog.recovery.complete', {
+        worldId,
+        pid,
+        mode: true,
+        replayed: !!lastDispatch,
+      });
+    } catch (err) {
+      log(`worldId=${worldId}: recovery failed: ${err?.message ?? err}`);
+      broadcast('world.watchdog.recovery.failed', {
+        worldId,
+        pid,
+        error: err?.message ?? String(err),
+      });
+    }
+  }
+
+  return { onWedgedVerdict };
+}