@pleri/olam-cli 0.1.195 → 0.1.198

package/host-cp/src/world-watchdog-recovery.mjs

@@ -0,0 +1,192 @@
+/**
+ * world-watchdog-recovery.mjs — recovery hook for wedged claude processes.
+ *
+ * Isolated from world-watchdog.mjs so kill + replay logic is independently
+ * mockable in tests without touching the watchdog's ticker.
+ *
+ * API:
+ *   createRecovery({ autoRecoverMode, leakyBucket, broadcaster, persister,
+ *                    replay, processKill, log })
+ *     → { onWedgedVerdict({ worldId, pid }): Promise<void> }
+ *
+ * Three modes (from compute.autoRecover in .olam/config.yaml):
+ *   false      — no-op; recovery never fires even on wedged verdict (DEFAULT)
+ *   'dry-run'  — emits all breadcrumbs, never calls processKill or replay
+ *   true       — SIGKILL pid + read last-dispatch + replay; rate-limited
+ *
+ * Rate-limit: B2 leaky-bucket (3/hour/world). 4th wedge in window emits
+ * world.watchdog.recovery.budget_exhausted and skips all action.
+ *
+ * Replay stub: the `replay` dep is accepted as an injected function. In
+ * server.mjs it is wired to a console.warn stub + breadcrumb until the
+ * operator runs the B3 idempotence probe and signs off. See TODO below.
+ *
+ * @see docs/architecture/world-watchdog.md Recovery section
+ * @see packages/host-cp/src/lib/leaky-bucket.mjs
+ * @see packages/host-cp/src/dispatch-persister.mjs
+ */
+
+/**
+ * @typedef {'false'|true|'dry-run'} AutoRecoverMode
+ */
+
+/**
+ * @typedef {object} RecoveryDeps
+ * @property {false|true|'dry-run'} autoRecoverMode
+ *   Passed from server.mjs which reads config.compute.autoRecover.
+ *   Default false if config unavailable.
+ * @property {{ tryConsume(key: string): { allowed: boolean, retryAfterMs?: number, totalInWindow: number } }} leakyBucket
+ *   B2 leaky-bucket instance. Keyed by worldId.
+ * @property {{ broadcast(type: string, payload: object): void }} [broadcaster]
+ *   Host-stream broadcaster. Optional — when absent, breadcrumbs are skipped.
+ * @property {{ read({ worldId: string }): Promise<{ messageId: string, prompt: string, dispatchedAt: string, source: string } | null> }} persister
+ *   B4 dispatch-persister read function.
+ * @property {(opts: { worldId: string, messageId: string, prompt: string }) => Promise<void>} replay
+ *   Opaque dispatch helper. Injected dep — DO NOT implement dispatch here.
+ *   In server.mjs this is wired to a stub until operator signs off on B3 probe.
+ * @property {(pid: number) => void} [processKill]
+ *   process.kill indirection so tests can spy without actually killing.
+ *   Defaults to process.kill.
+ * @property {(msg: string) => void} [log]
+ *   Logger. Defaults to console.log with [world-watchdog-recovery] prefix.
+ */
+
+/**
+ * @typedef {object} RecoveryHandle
+ * @property {(opts: { worldId: string, pid: number|null }) => Promise<void>} onWedgedVerdict
+ */
+
+/**
+ * Create a recovery handle.
+ *
+ * @param {RecoveryDeps} deps
+ * @returns {RecoveryHandle}
+ */
+export function createRecovery({
+  autoRecoverMode = false,
+  leakyBucket,
+  broadcaster = null,
+  persister,
+  replay,
+  processKill = (pid) => process.kill(pid, 'SIGKILL'),
+  log = (m) => console.log(`[world-watchdog-recovery] ${m}`),
+} = {}) {
+  /**
+   * Emit a breadcrumb via broadcaster (fail-soft).
+   *
+   * @param {string} type
+   * @param {object} payload
+   */
+  function broadcast(type, payload) {
+    if (!broadcaster || typeof broadcaster.broadcast !== 'function') return;
+    try {
+      broadcaster.broadcast(type, payload);
+    } catch (err) {
+      log(`broadcast ${type} failed: ${err?.message ?? err}`);
+    }
+  }
+
+  /**
+   * Handle a 2-tick-confirmed wedged verdict for a world.
+   *
+   * Called by world-watchdog.mjs on verdict-transition only (suspect → wedged),
+   * NOT on steady-state re-wedge.
+   *
+   * @param {{ worldId: string, pid: number|null }} opts
+   * @returns {Promise<void>}
+   */
+  async function onWedgedVerdict({ worldId, pid }) {
+    // mode=false → detection-only; never act.
+    if (autoRecoverMode === false) return;
+
+    // PID null → watchdog hasn't resolved a real PID yet (Phase A stub case);
+    // skip silently — there is nothing to kill.
+    if (pid === null) return;
+
+    // Rate-limit gate.
+    const bucket = leakyBucket.tryConsume(worldId);
+    if (!bucket.allowed) {
+      broadcast('world.watchdog.recovery.budget_exhausted', {
+        worldId,
+        retryAfterMs: bucket.retryAfterMs,
+        totalInWindow: bucket.totalInWindow,
+      });
+      log(`worldId=${worldId}: budget exhausted (${bucket.totalInWindow} in window); skipping recovery`);
+      return;
+    }
+
+    // Read last persisted dispatch for replay.
+    let lastDispatch = null;
+    try {
+      lastDispatch = await persister.read({ worldId });
+    } catch (err) {
+      log(`worldId=${worldId}: persister.read failed: ${err?.message ?? err}`);
+    }
+
+    broadcast('world.watchdog.recovery.start', {
+      worldId,
+      pid,
+      mode: autoRecoverMode,
+      lastDispatchMessageId: lastDispatch?.messageId ?? null,
+    });
+
+    // dry-run — log planned action but do NOT kill.
+    if (autoRecoverMode === 'dry-run') {
+      log(`worldId=${worldId}: dry-run — would SIGKILL pid=${pid}${lastDispatch ? ` + replay messageId=${lastDispatch.messageId}` : ' (no last-dispatch)'}`);
+      broadcast('world.watchdog.recovery.complete', {
+        worldId,
+        pid,
+        mode: 'dry-run',
+        replayed: false,
+      });
+      return;
+    }
+
+    // mode=true — act.
+    try {
+      // 1. SIGKILL the wedged process.
+      processKill(pid);
+      log(`worldId=${worldId}: SIGKILL sent to pid=${pid}`);
+
+      // 2. Replay or note absence of last-dispatch.
+      if (!lastDispatch) {
+        broadcast('world.watchdog.recovery.restart_without_replay', {
+          worldId,
+          pid,
+        });
+        log(`worldId=${worldId}: no last-dispatch; killed without replay`);
+      } else {
+        // TODO: wire real replay once operator has run the B3 idempotence probe
+        // and confirmed dispatch is idempotent for the substrates in use.
+        // Until then this stub logs and emits a breadcrumb so the stub path
+        // is visible in production logs. See B3 probe + operator review gate B6.
+        broadcast('world.watchdog.recovery.replay_stub', {
+          worldId,
+          prompt: lastDispatch.prompt,
+        });
+        log(`worldId=${worldId}: replay stub hit — real replay deferred pending B3 sign-off`);
+        await replay({
+          worldId,
+          messageId: lastDispatch.messageId,
+          prompt: lastDispatch.prompt,
+        });
+      }
+
+      broadcast('world.watchdog.recovery.complete', {
+        worldId,
+        pid,
+        mode: true,
+        replayed: !!lastDispatch,
+      });
+    } catch (err) {
+      log(`worldId=${worldId}: recovery failed: ${err?.message ?? err}`);
+      broadcast('world.watchdog.recovery.failed', {
+        worldId,
+        pid,
+        error: err?.message ?? String(err),
+      });
+    }
+  }
+
+  return { onWedgedVerdict };
+}

package/host-cp/src/world-watchdog.mjs

@@ -0,0 +1,313 @@
+/**
+ * world-watchdog.mjs — periodic watchdog that probes each active world's
+ * `claude` PID for the three wedge signals (wchan + CLOSE_WAIT + CPU) and
+ * emits `world.watchdog.tick` events on the host-stream broadcaster.
+ *
+ * Design:
+ *   - Mirrors `world-activity-tracker.mjs` shape exactly: `startWorldWatchdog(deps)`
+ *     returns `{ stop, tickNow }`.
+ *   - Per-world 2-tick confirm: a `'wedged'` classification is only emitted
+ *     after TWO consecutive ticks with the wedge signature. A single-tick
+ *     wedge emits `'suspect'`. A healthy tick resets the streak.
+ *   - Per-world fail-soft: a probe error for one world never skips other worlds.
+ *   - `OLAM_WORLD_WATCHDOG_DISABLED=1` → `start()` is a no-op (returns stub).
+ *   - Cadence: `OLAM_WORLD_WATCHDOG_TICK_MS` env or `intervalMs` dep (default 30_000).
+ *
+ * v1 stub: `getClaudePidForWorld(worldId)` returns null for all worlds in
+ * Phase A. When null, the tick still fires but all probe signals are null,
+ * producing `verdict: 'unknown'`. Real PID lookup (docker inspect →
+ * /proc/<hostPid>/status NSpid field) is wired in a follow-up.
+ * This is documented here and in docs/architecture/world-watchdog.md.
+ *
+ * Wire-in: `server.mjs` constructs once after broadcaster is ready and calls
+ * `.stop()` from the SIGTERM/SIGINT handler. Gated on `!SERVE_ONLY`.
+ *
+ * @see docs/architecture/world-watchdog.md
+ * @see packages/host-cp/src/world-watchdog-probes.mjs
+ * @see packages/host-cp/src/world-activity-tracker.mjs  (shape reference)
+ */
+
+import {
+  readWchan,
+  readCloseWaitSockets,
+  readCpuPercent,
+  classify,
+} from './world-watchdog-probes.mjs';
+// Recovery hook (B5). Optional dep — when absent (recovery is null/undefined),
+// the watchdog behaves exactly as Phase A: detection-only, no kill, no replay.
+// Wire via startWorldWatchdog({ recovery: createRecovery({...}) }) in server.mjs.
+
+const DEFAULT_TICK_MS = 30_000;
+// CPU measurement window: shorter than the tick cadence so we don't overlap.
+const CPU_WINDOW_MS = 500;
+
+/**
+ * @typedef {object} WorldWatchdogDeps
+ * @property {object}  [broadcaster]          Object with `.broadcast(type, payload)`.
+ *   Optional — when absent events are skipped but state tracking still works.
+ * @property {number}  [intervalMs]           Tick cadence in ms. Defaults to
+ *   `OLAM_WORLD_WATCHDOG_TICK_MS` env or 30_000.
+ * @property {() => Promise<string[]>} [listActiveWorlds]
+ *   Returns an array of active world IDs to probe each tick.
+ *   Defaults to returning [].
+ * @property {(worldId: string) => Promise<number|null>} [getClaudePidForWorld]
+ *   Returns the host-side PID of the claude process for a world, or null.
+ *   v1 default: always returns null (all worlds → verdict 'unknown').
+ * @property {{ procRoot?: string }} [probes]
+ *   Injectable probe options (procRoot for tests).
+ * @property {{ onWedgedVerdict(opts: { worldId: string, pid: number|null }): Promise<void> }} [recovery]
+ *   Optional recovery handle (from world-watchdog-recovery.mjs). When present,
+ *   called once on verdict-transition to 'wedged' (suspect → wedged), NOT on
+ *   steady-state re-wedge. When absent, detection-only (Phase A behaviour).
+ * @property {(msg: string) => void} [log]    Defaults to `console.log`.
+ * @property {(msg: string) => void} [debug]  Defaults to no-op.
+ * @property {(cb: () => void, ms: number) => any} [setTimer]
+ *   Injectable `setInterval` for tests.
+ * @property {(handle: any) => void} [clearTimer]
+ *   Injectable `clearInterval` for tests.
+ * @property {() => Date} [now]               Clock injection for tests.
+ */
+
+/**
+ * @typedef {object} WorldWatchdogHandle
+ * @property {() => void} stop
+ * @property {() => Promise<number>} tickNow  Run one tick immediately (returns
+ *   the count of worlds processed). Exposed for tests.
+ * @property {(worldId: string) => object|null} getVerdict
+ *   Returns the latest in-memory verdict entry for a world, or null if no tick
+ *   has fired yet. Used by the HTTP endpoint (A5).
+ */
+
+/**
+ * Per-world state tracked between ticks for the 2-tick confirm.
+ *
+ * @typedef {object} WorldWatchdogState
+ * @property {'healthy'|'suspect'|'wedged'|'unknown'} lastClassification
+ *   The raw classification from the previous tick (before 2-tick confirm).
+ * @property {'healthy'|'suspect'|'wedged'|'unknown'} lastVerdict
+ *   The emitted verdict (post-confirm).
+ * @property {string} lastTickAt   ISO-8601 timestamp of last tick.
+ * @property {object|null} lastSignals  The signals from the last tick.
+ * @property {number|null} lastPid  The PID probed last tick.
+ */
+
+/**
+ * Start the world watchdog. Returns a `{ stop, tickNow, getVerdict }` handle.
+ *
+ * Honoring `OLAM_WORLD_WATCHDOG_DISABLED=1`: if the env var is set, returns
+ * a no-op stub immediately without starting the interval or making any probe
+ * calls.
+ *
+ * @param {WorldWatchdogDeps} [deps]
+ * @returns {WorldWatchdogHandle}
+ */
+export function startWorldWatchdog(deps = {}) {
+  // Honour kill switch — return a no-op stub.
+  if (process.env.OLAM_WORLD_WATCHDOG_DISABLED === '1') {
+    return {
+      stop() {},
+      tickNow: async () => 0,
+      getVerdict: () => null,
+    };
+  }
+
+  const log = deps.log ?? ((m) => console.log(`[world-watchdog] ${m}`));
+  const debug = deps.debug ?? (() => {});
+  const setTimer = deps.setTimer ?? ((cb, ms) => setInterval(cb, ms));
+  const clearTimer = deps.clearTimer ?? ((h) => clearInterval(h));
+  const now = deps.now ?? (() => new Date());
+
+  const intervalMs =
+    deps.intervalMs ??
+    parseInt(process.env.OLAM_WORLD_WATCHDOG_TICK_MS ?? `${DEFAULT_TICK_MS}`, 10);
+
+  const broadcaster = deps.broadcaster ?? null;
+  const listActiveWorlds = deps.listActiveWorlds ?? (async () => []);
+  const getClaudePidForWorld = deps.getClaudePidForWorld ?? (async (_id) => null);
+  const probeOpts = deps.probes ?? {};
+  // Recovery hook — null when not configured (Phase A / default-off behaviour).
+  const recovery = deps.recovery ?? null;
+
+  // Per-world state map: worldId → WorldWatchdogState.
+  /** @type {Map<string, WorldWatchdogState>} */
+  const worldState = new Map();
+
+  let stopped = false;
+  let inFlight = false;
+  let intervalHandle = null;
+
+  /**
+   * Probe a single world and update its state. Returns the verdict emitted.
+   *
+   * @param {string} worldId
+   * @returns {Promise<'healthy'|'suspect'|'wedged'|'unknown'>}
+   */
+  async function probeWorld(worldId) {
+    const pid = await getClaudePidForWorld(worldId);
+
+    let wchan = null;
+    let closeWaitSockets = [];
+    let cpuPercent = null;
+
+    if (pid !== null) {
+      // All probes are fail-soft — they return null/[] on I/O error.
+      [wchan, closeWaitSockets, cpuPercent] = await Promise.all([
+        readWchan(pid, probeOpts),
+        readCloseWaitSockets(pid, probeOpts),
+        readCpuPercent(pid, CPU_WINDOW_MS, probeOpts),
+      ]);
+    }
+
+    const closeWaitCount = closeWaitSockets.length;
+    const signals = pid !== null
+      ? { wchan, closeWaitCount, cpuPercent }
+      : null;
+
+    // Classify raw signals.
+    const rawClassification = pid !== null
+      ? classify({ wchan, closeWaitCount, cpuPercent })
+      : 'unknown';
+
+    // 2-tick confirm: only emit 'wedged' if BOTH this tick AND the previous tick
+    // classified as 'wedged'. Otherwise emit the raw classification.
+    const prev = worldState.get(worldId);
+    let verdict;
+    if (rawClassification === 'wedged' && prev?.lastClassification === 'wedged') {
+      verdict = 'wedged';
+    } else if (rawClassification === 'wedged') {
+      // First 'wedged' tick — emit 'suspect' (2-tick confirm pending).
+      verdict = 'suspect';
+    } else {
+      verdict = rawClassification;
+    }
+
+    const tickAt = now().toISOString();
+
+    // Update per-world state.
+    worldState.set(worldId, {
+      lastClassification: rawClassification,
+      lastVerdict: verdict,
+      lastTickAt: tickAt,
+      lastSignals: signals,
+      lastPid: pid,
+    });
+
+    // Recovery hook — fire ONCE on verdict-transition to 'wedged' (not on
+    // steady-state re-wedge). Guard: prev?.lastVerdict !== 'wedged' ensures
+    // only the suspect→wedged transition triggers, not wedged→wedged.
+    if (
+      verdict === 'wedged' &&
+      recovery !== null &&
+      prev?.lastVerdict !== 'wedged'
+    ) {
+      // Fire-and-forget; fail-soft so a recovery error never skips other worlds.
+      void recovery.onWedgedVerdict({ worldId, pid }).catch((err) => {
+        log(`recovery.onWedgedVerdict ${worldId} failed: ${err?.message ?? err}`);
+      });
+    }
+
+    // Emit broadcaster event.
+    if (broadcaster && typeof broadcaster.broadcast === 'function') {
+      try {
+        broadcaster.broadcast('world.watchdog.tick', {
+          worldId,
+          verdict,
+          signals,
+          pid,
+          lastTickAt: tickAt,
+        });
+      } catch (err) {
+        log(`broadcast ${worldId} failed: ${err?.message ?? err}`);
+      }
+    }
+
+    return verdict;
+  }
+
+  /**
+   * One tick: get active worlds, probe each, return count processed.
+   *
+   * @returns {Promise<number>}
+   */
+  async function tick() {
+    if (stopped) return 0;
+    if (inFlight) {
+      debug('tick skipped: previous tick still in flight');
+      return 0;
+    }
+    inFlight = true;
+
+    let processed = 0;
+    try {
+      let worlds;
+      try {
+        worlds = await listActiveWorlds();
+      } catch (err) {
+        log(`listActiveWorlds failed: ${err?.message ?? err}`);
+        return 0;
+      }
+
+      for (const worldId of worlds) {
+        if (stopped) break;
+        if (typeof worldId !== 'string') continue;
+
+        try {
+          await probeWorld(worldId);
+          processed += 1;
+        } catch (err) {
+          // Per-world fail-soft: one bad world doesn't crash the loop.
+          debug(`probe ${worldId} failed: ${err?.message ?? err}`);
+        }
+      }
+    } finally {
+      inFlight = false;
+    }
+
+    return processed;
+  }
+
+  // Kick off an initial tick on next event-loop turn so callers can
+  // attach test spies before any probe work happens.
+  setImmediate(() => {
+    if (stopped) return;
+    void tick().catch((err) => {
+      log(`initial tick crashed: ${err?.message ?? err}`);
+    });
+  });
+
+  intervalHandle = setTimer(() => {
+    void tick().catch((err) => {
+      log(`tick crashed: ${err?.message ?? err}`);
+    });
+  }, intervalMs);
+  // Don't pin the event loop on shutdown.
+  if (intervalHandle && typeof intervalHandle.unref === 'function') {
+    intervalHandle.unref();
+  }
+
+  log(`started: interval=${intervalMs}ms`);
+
+  return {
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      if (intervalHandle !== null) {
+        try { clearTimer(intervalHandle); } catch { /* ignore */ }
+        intervalHandle = null;
+      }
+    },
+
+    tickNow: tick,
+
+    /**
+     * Return the latest in-memory verdict entry for a world.
+     * Returns null if no tick has fired for this world yet.
+     *
+     * @param {string} worldId
+     * @returns {WorldWatchdogState|null}
+     */
+    getVerdict(worldId) {
+      return worldState.get(worldId) ?? null;
+    },
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pleri/olam-cli",
-  "version": "0.1.195",
+  "version": "0.1.198",
   "type": "module",
   "bin": {
     "olam": "./bin/olam.cjs"