@pleri/olam-cli 0.1.196 → 0.1.199

package/host-cp/src/world-watchdog.mjs

@@ -0,0 +1,313 @@
+/**
+ * world-watchdog.mjs — periodic watchdog that probes each active world's
+ * `claude` PID for the three wedge signals (wchan + CLOSE_WAIT + CPU) and
+ * emits `world.watchdog.tick` events on the host-stream broadcaster.
+ *
+ * Design:
+ *   - Mirrors `world-activity-tracker.mjs` shape exactly: `startWorldWatchdog(deps)`
+ *     returns `{ stop, tickNow }`.
+ *   - Per-world 2-tick confirm: a `'wedged'` classification is only emitted
+ *     after TWO consecutive ticks with the wedge signature. A single-tick
+ *     wedge emits `'suspect'`. A healthy tick resets the streak.
+ *   - Per-world fail-soft: a probe error for one world never skips other worlds.
+ *   - `OLAM_WORLD_WATCHDOG_DISABLED=1` → `start()` is a no-op (returns stub).
+ *   - Cadence: `OLAM_WORLD_WATCHDOG_TICK_MS` env or `intervalMs` dep (default 30_000).
+ *
+ * v1 stub: `getClaudePidForWorld(worldId)` returns null for all worlds in
+ * Phase A. When null, the tick still fires but all probe signals are null,
+ * producing `verdict: 'unknown'`. Real PID lookup (docker inspect →
+ * /proc/<hostPid>/status NSpid field) is wired in a follow-up.
+ * This is documented here and in docs/architecture/world-watchdog.md.
+ *
+ * Wire-in: `server.mjs` constructs once after broadcaster is ready and calls
+ * `.stop()` from the SIGTERM/SIGINT handler. Gated on `!SERVE_ONLY`.
+ *
+ * @see docs/architecture/world-watchdog.md
+ * @see packages/host-cp/src/world-watchdog-probes.mjs
+ * @see packages/host-cp/src/world-activity-tracker.mjs  (shape reference)
+ */
+
+import {
+  readWchan,
+  readCloseWaitSockets,
+  readCpuPercent,
+  classify,
+} from './world-watchdog-probes.mjs';
+// Recovery hook (B5). Optional dep — when absent (recovery is null/undefined),
+// the watchdog behaves exactly as Phase A: detection-only, no kill, no replay.
+// Wire via startWorldWatchdog({ recovery: createRecovery({...}) }) in server.mjs.
+
+const DEFAULT_TICK_MS = 30_000;
+// CPU measurement window: shorter than the tick cadence so we don't overlap.
+const CPU_WINDOW_MS = 500;
+
+/**
+ * @typedef {object} WorldWatchdogDeps
+ * @property {object}  [broadcaster]          Object with `.broadcast(type, payload)`.
+ *   Optional — when absent events are skipped but state tracking still works.
+ * @property {number}  [intervalMs]           Tick cadence in ms. Defaults to
+ *   `OLAM_WORLD_WATCHDOG_TICK_MS` env or 30_000.
+ * @property {() => Promise<string[]>} [listActiveWorlds]
+ *   Returns an array of active world IDs to probe each tick.
+ *   Defaults to returning [].
+ * @property {(worldId: string) => Promise<number|null>} [getClaudePidForWorld]
+ *   Returns the host-side PID of the claude process for a world, or null.
+ *   v1 default: always returns null (all worlds → verdict 'unknown').
+ * @property {{ procRoot?: string }} [probes]
+ *   Injectable probe options (procRoot for tests).
+ * @property {{ onWedgedVerdict(opts: { worldId: string, pid: number|null }): Promise<void> }} [recovery]
+ *   Optional recovery handle (from world-watchdog-recovery.mjs). When present,
+ *   called once on verdict-transition to 'wedged' (suspect → wedged), NOT on
+ *   steady-state re-wedge. When absent, detection-only (Phase A behaviour).
+ * @property {(msg: string) => void} [log]    Defaults to `console.log`.
+ * @property {(msg: string) => void} [debug]  Defaults to no-op.
+ * @property {(cb: () => void, ms: number) => any} [setTimer]
+ *   Injectable `setInterval` for tests.
+ * @property {(handle: any) => void} [clearTimer]
+ *   Injectable `clearInterval` for tests.
+ * @property {() => Date} [now]               Clock injection for tests.
+ */
+
+/**
+ * @typedef {object} WorldWatchdogHandle
+ * @property {() => void} stop
+ * @property {() => Promise<number>} tickNow  Run one tick immediately (returns
+ *   the count of worlds processed). Exposed for tests.
+ * @property {(worldId: string) => object|null} getVerdict
+ *   Returns the latest in-memory verdict entry for a world, or null if no tick
+ *   has fired yet. Used by the HTTP endpoint (A5).
+ */
+
+/**
+ * Per-world state tracked between ticks for the 2-tick confirm.
+ *
+ * @typedef {object} WorldWatchdogState
+ * @property {'healthy'|'suspect'|'wedged'|'unknown'} lastClassification
+ *   The raw classification from the previous tick (before 2-tick confirm).
+ * @property {'healthy'|'suspect'|'wedged'|'unknown'} lastVerdict
+ *   The emitted verdict (post-confirm).
+ * @property {string} lastTickAt   ISO-8601 timestamp of last tick.
+ * @property {object|null} lastSignals  The signals from the last tick.
+ * @property {number|null} lastPid  The PID probed last tick.
+ */
+
+/**
+ * Start the world watchdog. Returns a `{ stop, tickNow, getVerdict }` handle.
+ *
+ * Honoring `OLAM_WORLD_WATCHDOG_DISABLED=1`: if the env var is set, returns
+ * a no-op stub immediately without starting the interval or making any probe
+ * calls.
+ *
+ * @param {WorldWatchdogDeps} [deps]
+ * @returns {WorldWatchdogHandle}
+ */
+export function startWorldWatchdog(deps = {}) {
+  // Honour kill switch — return a no-op stub.
+  if (process.env.OLAM_WORLD_WATCHDOG_DISABLED === '1') {
+    return {
+      stop() {},
+      tickNow: async () => 0,
+      getVerdict: () => null,
+    };
+  }
+
+  const log = deps.log ?? ((m) => console.log(`[world-watchdog] ${m}`));
+  const debug = deps.debug ?? (() => {});
+  const setTimer = deps.setTimer ?? ((cb, ms) => setInterval(cb, ms));
+  const clearTimer = deps.clearTimer ?? ((h) => clearInterval(h));
+  const now = deps.now ?? (() => new Date());
+
+  const intervalMs =
+    deps.intervalMs ??
+    parseInt(process.env.OLAM_WORLD_WATCHDOG_TICK_MS ?? `${DEFAULT_TICK_MS}`, 10);
+
+  const broadcaster = deps.broadcaster ?? null;
+  const listActiveWorlds = deps.listActiveWorlds ?? (async () => []);
+  const getClaudePidForWorld = deps.getClaudePidForWorld ?? (async (_id) => null);
+  const probeOpts = deps.probes ?? {};
+  // Recovery hook — null when not configured (Phase A / default-off behaviour).
+  const recovery = deps.recovery ?? null;
+
+  // Per-world state map: worldId → WorldWatchdogState.
+  /** @type {Map<string, WorldWatchdogState>} */
+  const worldState = new Map();
+
+  let stopped = false;
+  let inFlight = false;
+  let intervalHandle = null;
+
+  /**
+   * Probe a single world and update its state. Returns the verdict emitted.
+   *
+   * @param {string} worldId
+   * @returns {Promise<'healthy'|'suspect'|'wedged'|'unknown'>}
+   */
+  async function probeWorld(worldId) {
+    const pid = await getClaudePidForWorld(worldId);
+
+    let wchan = null;
+    let closeWaitSockets = [];
+    let cpuPercent = null;
+
+    if (pid !== null) {
+      // All probes are fail-soft — they return null/[] on I/O error.
+      [wchan, closeWaitSockets, cpuPercent] = await Promise.all([
+        readWchan(pid, probeOpts),
+        readCloseWaitSockets(pid, probeOpts),
+        readCpuPercent(pid, CPU_WINDOW_MS, probeOpts),
+      ]);
+    }
+
+    const closeWaitCount = closeWaitSockets.length;
+    const signals = pid !== null
+      ? { wchan, closeWaitCount, cpuPercent }
+      : null;
+
+    // Classify raw signals.
+    const rawClassification = pid !== null
+      ? classify({ wchan, closeWaitCount, cpuPercent })
+      : 'unknown';
+
+    // 2-tick confirm: only emit 'wedged' if BOTH this tick AND the previous tick
+    // classified as 'wedged'. Otherwise emit the raw classification.
+    const prev = worldState.get(worldId);
+    let verdict;
+    if (rawClassification === 'wedged' && prev?.lastClassification === 'wedged') {
+      verdict = 'wedged';
+    } else if (rawClassification === 'wedged') {
+      // First 'wedged' tick — emit 'suspect' (2-tick confirm pending).
+      verdict = 'suspect';
+    } else {
+      verdict = rawClassification;
+    }
+
+    const tickAt = now().toISOString();
+
+    // Update per-world state.
+    worldState.set(worldId, {
+      lastClassification: rawClassification,
+      lastVerdict: verdict,
+      lastTickAt: tickAt,
+      lastSignals: signals,
+      lastPid: pid,
+    });
+
+    // Recovery hook — fire ONCE on verdict-transition to 'wedged' (not on
+    // steady-state re-wedge). Guard: prev?.lastVerdict !== 'wedged' ensures
+    // only the suspect→wedged transition triggers, not wedged→wedged.
+    if (
+      verdict === 'wedged' &&
+      recovery !== null &&
+      prev?.lastVerdict !== 'wedged'
+    ) {
+      // Fire-and-forget; fail-soft so a recovery error never skips other worlds.
+      void recovery.onWedgedVerdict({ worldId, pid }).catch((err) => {
+        log(`recovery.onWedgedVerdict ${worldId} failed: ${err?.message ?? err}`);
+      });
+    }
+
+    // Emit broadcaster event.
+    if (broadcaster && typeof broadcaster.broadcast === 'function') {
+      try {
+        broadcaster.broadcast('world.watchdog.tick', {
+          worldId,
+          verdict,
+          signals,
+          pid,
+          lastTickAt: tickAt,
+        });
+      } catch (err) {
+        log(`broadcast ${worldId} failed: ${err?.message ?? err}`);
+      }
+    }
+
+    return verdict;
+  }
+
+  /**
+   * One tick: get active worlds, probe each, return count processed.
+   *
+   * @returns {Promise<number>}
+   */
+  async function tick() {
+    if (stopped) return 0;
+    if (inFlight) {
+      debug('tick skipped: previous tick still in flight');
+      return 0;
+    }
+    inFlight = true;
+
+    let processed = 0;
+    try {
+      let worlds;
+      try {
+        worlds = await listActiveWorlds();
+      } catch (err) {
+        log(`listActiveWorlds failed: ${err?.message ?? err}`);
+        return 0;
+      }
+
+      for (const worldId of worlds) {
+        if (stopped) break;
+        if (typeof worldId !== 'string') continue;
+
+        try {
+          await probeWorld(worldId);
+          processed += 1;
+        } catch (err) {
+          // Per-world fail-soft: one bad world doesn't crash the loop.
+          debug(`probe ${worldId} failed: ${err?.message ?? err}`);
+        }
+      }
+    } finally {
+      inFlight = false;
+    }
+
+    return processed;
+  }
+
+  // Kick off an initial tick on next event-loop turn so callers can
+  // attach test spies before any probe work happens.
+  setImmediate(() => {
+    if (stopped) return;
+    void tick().catch((err) => {
+      log(`initial tick crashed: ${err?.message ?? err}`);
+    });
+  });
+
+  intervalHandle = setTimer(() => {
+    void tick().catch((err) => {
+      log(`tick crashed: ${err?.message ?? err}`);
+    });
+  }, intervalMs);
+  // Don't pin the event loop on shutdown.
+  if (intervalHandle && typeof intervalHandle.unref === 'function') {
+    intervalHandle.unref();
+  }
+
+  log(`started: interval=${intervalMs}ms`);
+
+  return {
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      if (intervalHandle !== null) {
+        try { clearTimer(intervalHandle); } catch { /* ignore */ }
+        intervalHandle = null;
+      }
+    },
+
+    tickNow: tick,
+
+    /**
+     * Return the latest in-memory verdict entry for a world.
+     * Returns null if no tick has fired for this world yet.
+     *
+     * @param {string} worldId
+     * @returns {WorldWatchdogState|null}
+     */
+    getVerdict(worldId) {
+      return worldState.get(worldId) ?? null;
+    },
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pleri/olam-cli",
-  "version": "0.1.196",
+  "version": "0.1.199",
   "type": "module",
   "bin": {
     "olam": "./bin/olam.cjs"