@pleri/olam-cli 0.1.196 → 0.1.199

package/host-cp/src/dispatch-persister.mjs

@@ -0,0 +1,157 @@
+/**
+ * dispatch-persister.mjs — persist the last dispatch for each world.
+ *
+ * The world watchdog's recovery hook reads this to replay the last
+ * unanswered prompt when it auto-recovers a wedged claude process.
+ *
+ * Contract:
+ *   persist({ worldId, messageId, prompt, source, statePath?, now? })
+ *     Atomically writes ~/.olam/worlds/<worldId>/state/last-dispatch.json.
+ *     Overwrites any previous file — only the LATEST dispatch matters for
+ *     replay. Atomic write (tmp + fs.rename) prevents partial-write residue
+ *     from corrupting recovery reads.
+ *
+ *   read({ worldId, statePath? })
+ *     Returns { messageId, prompt, dispatchedAt, source } or null.
+ *     null on ENOENT (no dispatch persisted yet) — never throws.
+ *     null on JSON parse error (logs + skips) — never throws on corrupt file.
+ *
+ * Multiple worlds are independent: world A and world B have separate files.
+ * Multiple concurrent persist() calls for the SAME world are safe — each
+ * write is a rename of a tmp file so the worst case is one write winning.
+ *
+ * @see docs/architecture/world-watchdog.md
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+
+// Default base path under which per-world state directories live.
+const DEFAULT_STATE_BASE = path.join(os.homedir(), '.olam', 'worlds');
+
+/**
+ * Derive the path to last-dispatch.json for a world.
+ *
+ * @param {string} worldId
+ * @param {string} [stateBase]  Override the base directory (for tests).
+ * @returns {string}
+ */
+export function lastDispatchPath(worldId, stateBase = DEFAULT_STATE_BASE) {
+  return path.join(stateBase, worldId, 'state', 'last-dispatch.json');
+}
+
+/**
+ * Persist the last dispatch for a world.
+ *
+ * @param {{
+ *   worldId: string,
+ *   messageId: string,
+ *   prompt: string,
+ *   source: string,
+ *   statePath?: string,
+ *   now?: () => number,
+ * }} opts
+ * @returns {Promise<void>}
+ */
+export async function persist({
+  worldId,
+  messageId,
+  prompt,
+  source,
+  statePath,
+  now = () => Date.now(),
+}) {
+  const filePath = statePath ?? lastDispatchPath(worldId);
+  const dir = path.dirname(filePath);
+  const tmpPath = `${filePath}.tmp`;
+
+  const record = {
+    messageId,
+    prompt,
+    dispatchedAt: new Date(now()).toISOString(),
+    source,
+  };
+
+  // Ensure the directory exists.
+  await fs.mkdir(dir, { recursive: true });
+
+  // Atomic write: write to .tmp then rename over the target.
+  await fs.writeFile(tmpPath, JSON.stringify(record, null, 2) + '\n', 'utf8');
+  await fs.rename(tmpPath, filePath);
+}
+
+/**
+ * Fire-and-forget persist wrapper used at the dispatch call-sites.
+ *
+ * Centralises the void/.catch boilerplate so the two enrichment sites
+ * (pr-nanny + /api/cloud-dispatch) can't drift on future changes.
+ * Logs failures via the supplied logSource tag; never throws.
+ *
+ * @param {{
+ *   worldId: string,
+ *   messageId: string,
+ *   prompt: string,
+ *   source: string,
+ *   logSource?: string,
+ *   statePath?: string,
+ *   now?: () => number,
+ * }} opts
+ * @returns {void}
+ */
+export function safePersistLastDispatch(opts) {
+  const { logSource = opts.source, ...persistOpts } = opts;
+  void persist(persistOpts).catch((err) => {
+    console.warn(
+      `[${logSource}] persistLastDispatch failed (non-fatal): ${err?.message ?? err}`,
+    );
+  });
+}
+
+/**
+ * Read the last persisted dispatch for a world.
+ *
+ * @param {{
+ *   worldId: string,
+ *   statePath?: string,
+ * }} opts
+ * @returns {Promise<{ messageId: string, prompt: string, dispatchedAt: string, source: string } | null>}
+ */
+export async function read({ worldId, statePath }) {
+  const filePath = statePath ?? lastDispatchPath(worldId);
+
+  let raw;
+  try {
+    raw = await fs.readFile(filePath, 'utf8');
+  } catch (err) {
+    if (err?.code === 'ENOENT') return null;
+    // Other I/O errors (e.g. permissions) — log + return null (fail-soft).
+    console.error(`[dispatch-persister] readFile ${filePath}: ${err?.message ?? err}`);
+    return null;
+  }
+
+  try {
+    const parsed = JSON.parse(raw);
+    // Basic shape validation — don't throw on corrupt file.
+    if (
+      typeof parsed !== 'object' ||
+      parsed === null ||
+      typeof parsed.messageId !== 'string' ||
+      typeof parsed.prompt !== 'string' ||
+      typeof parsed.dispatchedAt !== 'string' ||
+      typeof parsed.source !== 'string'
+    ) {
+      console.error(`[dispatch-persister] ${filePath}: unexpected shape, skipping`);
+      return null;
+    }
+    return {
+      messageId: parsed.messageId,
+      prompt: parsed.prompt,
+      dispatchedAt: parsed.dispatchedAt,
+      source: parsed.source,
+    };
+  } catch (err) {
+    console.error(`[dispatch-persister] ${filePath}: JSON parse error: ${err?.message ?? err}`);
+    return null;
+  }
+}

package/host-cp/src/pr-nanny.mjs

@@ -24,6 +24,7 @@
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
 import { pickNextTier } from './dispatch/tier-escalator.mjs';
+import { safePersistLastDispatch } from './dispatch-persister.mjs';
 
 const execFileAsync = promisify(execFile);
 
@@ -251,6 +252,12 @@ export function createPrNanny({
 
     // Dispatch fix
     try {
+      safePersistLastDispatch({
+        worldId,
+        messageId: `nanny-${worldId}-${Date.now()}`,
+        prompt,
+        source: 'pr-nanny',
+      });
       await dispatchToWorld(worldId, prompt, { tier: tierForThisDispatch });
       const now = new Date().toISOString();
       prStateStore.set(worldId, {

package/host-cp/src/server.mjs

@@ -84,6 +84,11 @@ import {
   defaultListContainerNames,
 } from './boot-reconciler.mjs';
 import { startWorldActivityTracker } from './world-activity-tracker.mjs';
+import { startWorldWatchdog } from './world-watchdog.mjs';
+import { createRecovery } from './world-watchdog-recovery.mjs';
+import { createLeakyBucket } from './lib/leaky-bucket.mjs';
+import { read as readLastDispatch, safePersistLastDispatch } from './dispatch-persister.mjs';
+import { findClaudePid } from './world-watchdog-pid-lookup.mjs';
 import { authSecretHint } from './auth-secret-hint.mjs';
 import * as tunnelManager from './world-tunnel-manager.mjs';
 import * as bridgeManager from './port-bridge-manager.mjs';
@@ -96,6 +101,7 @@ import {
 import { instrumentHandler, renderMetrics } from './metrics.mjs';
 import { handleDispatchFromEmail } from './lib/email-dispatch.mjs';
 import { handleDispatchFromLinear } from './lib/linear-dispatch.mjs';
+// (safePersistLastDispatch imported above alongside readLastDispatch)
 import { emitTierSuggestion } from '../dispatch/auto-tier-scheduler.mjs';
 import { isServeOnly, isOrchestrationRoute, ORCHESTRATION_UNAVAILABLE } from './serve-only-config.mjs';
 
@@ -1874,6 +1880,41 @@ const server = http.createServer(instrumentHandler('host-cp', async (req, res) =
   // the isOrchestrationRoute guard — it covers /api/world/, /api/worlds/<id>,
   // and /v1/worlds/ for all methods, so no per-route guard is needed here.)
 
+  // GET /api/world/<id>/socket-health — world watchdog verdict (A5).
+  // Returns the latest in-memory probe result from the world watchdog.
+  // Read-only; never mutates world state.
+  //   200: { worldId, verdict, signals, pid, lastTickAt } — known world + tick fired
+  //   200 verdict='unknown': known world but no tick has fired yet
+  //   404: unknown_world
+  // serve-only: returns 503 orchestration_unavailable (isOrchestrationRoute covers
+  //   /api/world/* prefix, so this route is already blocked upstream before reaching here).
+  const socketHealthMatch = /^\/api\/world\/([^/?#]+)\/socket-health\/?$/.exec(url.pathname);
+  if (socketHealthMatch && req.method === 'GET') {
+    const worldId = decodeURIComponent(socketHealthMatch[1]);
+    if (!(worldId in WORLDS)) {
+      return jsonReply(res, 404, { error: 'unknown_world' });
+    }
+    // worldWatchdog is null in serve-only mode (but the serve-only gate above
+    // would have returned 503 before we get here; belt-and-suspenders).
+    const entry = worldWatchdog ? worldWatchdog.getVerdict(worldId) : null;
+    if (!entry) {
+      return jsonReply(res, 200, {
+        worldId,
+        verdict: 'unknown',
+        signals: null,
+        pid: null,
+        lastTickAt: null,
+      });
+    }
+    return jsonReply(res, 200, {
+      worldId,
+      verdict: entry.lastVerdict,
+      signals: entry.lastSignals,
+      pid: entry.lastPid,
+      lastTickAt: entry.lastTickAt,
+    });
+  }
+
   // GET /api/world/<id>/progress — phase ladder progress for inbox row.
   const progressMatch = /^\/api\/world\/([^/?#]+)\/progress\/?$/.exec(url.pathname);
   if (progressMatch && req.method === 'GET') {
@@ -1892,8 +1933,16 @@ const server = http.createServer(instrumentHandler('host-cp', async (req, res) =
       prStateStore,
       getGhToken: resolveGhToken,
     });
-    progressCache.set(worldId, { fetchedAt: Date.now(), data });
-    return jsonReply(res, 200, data);
+    // C1 — attach socketHealth if watchdog has fired for this world.
+    const verdictEntry = worldWatchdog ? worldWatchdog.getVerdict(worldId) : null;
+    const enriched = verdictEntry
+      ? {
+          ...data,
+          socketHealth: buildSocketHealthPayload(worldId, verdictEntry),
+        }
+      : data;
+    progressCache.set(worldId, { fetchedAt: Date.now(), data: enriched });
+    return jsonReply(res, 200, enriched);
   }
 
   // /api/world/<id>/* → proxy to per-world CP with X-Olam-Secret injected.
@@ -2686,6 +2735,15 @@ const server = http.createServer(instrumentHandler('host-cp', async (req, res) =
       // env-var enrichments are applied.
       const enriched = enrichedObj ? JSON.stringify(enrichedObj) : JSON.stringify(parsed);
 
+      if (parsed.world_id && parsed.prompt) {
+        safePersistLastDispatch({
+          worldId: parsed.world_id,
+          messageId: parsed.session_id ?? `cloud-dispatch-${Date.now()}`,
+          prompt: parsed.prompt,
+          source: 'cloud-dispatch',
+        });
+      }
+
       // Phase H h2: attach CF Access service-token headers when configured
       // (machine-to-machine auth). Additive alongside Basic auth. CF Access
       // headers are validated at the EDGE of origins behind a CF Access app
@@ -3674,6 +3732,119 @@ const worldActivityTracker = SERVE_ONLY
       broadcaster: hostStream,
     });
 
+// World watchdog — periodic probe of each active world's claude PID for the
+// three wedge signals (wchan + CLOSE_WAIT + CPU). Emits `world.watchdog.tick`
+// events on the host-stream broadcaster.
+//
+// Phase B: recovery is wired when compute.autoRecover !== false.
+// Default is false (detection-only, byte-identical to Phase A behaviour).
+//
+// SERVE-ONLY: no docker / worlds.db on a managed cluster; null sentinel keeps
+// the shutdown handler's `worldWatchdog?.stop()` a no-op.
+//
+// getClaudePidForWorld is a v1 stub returning null for all worlds. All worlds
+// therefore emit verdict='unknown' until real PID lookup is wired in a follow-up.
+
+// ── Recovery setup (Phase B) ─────────────────────────────────────────────────
+// Load autoRecover from env OLAM_AUTO_RECOVER (or false by default — D4).
+// Falls back to false if config unavailable or field absent.
+
+// The compute.autoRecover field lives in .olam/config.yaml (per workspace),
+// not in ~/.olam/config.json (global). Host-cp does not load workspace YAML at
+// startup. Read from env OLAM_AUTO_RECOVER; default false (D4 — OFF by default).
+const _watchdogAutoRecoverMode = (() => {
+  const envVal = process.env.OLAM_AUTO_RECOVER;
+  if (envVal === 'true') return true;
+  if (envVal === 'dry-run') return 'dry-run';
+  return false;
+})();
+
+const _watchdogLeakyBucket = createLeakyBucket({ capacity: 3, windowMs: 3_600_000 });
+
+const _watchdogRecovery = SERVE_ONLY
+  ? null
+  : createRecovery({
+      autoRecoverMode: _watchdogAutoRecoverMode,
+      leakyBucket: _watchdogLeakyBucket,
+      broadcaster: hostStream,
+      persister: { read: readLastDispatch },
+      // TODO: wire real replay once operator has run the B3 idempotence probe
+      // and confirmed dispatch is idempotent for all substrates in use.
+      // See docs/architecture/world-watchdog.md Recovery > Idempotence probe.
+      // For now: log + emit replay_stub breadcrumb so the stub path is visible.
+      replay: async ({ worldId, prompt }) => {
+        console.warn(
+          `[world-watchdog-recovery] replay stub: worldId=${worldId} prompt="${prompt.slice(0, 80)}..." — real replay deferred pending B3 sign-off`,
+        );
+        // breadcrumb already emitted by createRecovery before calling replay
+      },
+    });
+
+const worldWatchdog = SERVE_ONLY
+  ? null
+  : startWorldWatchdog({
+      broadcaster: hostStream,
+      recovery: _watchdogRecovery,
+      listActiveWorlds: async () => {
+        // Reuse the same worlds.db query as worldActivityTracker: return all
+        // non-destroyed/failed world IDs for probing.
+        let Database;
+        try {
+          const { createRequire } = await import('node:module');
+          const req = createRequire(import.meta.url);
+          Database = req('better-sqlite3');
+        } catch {
+          return [];
+        }
+        let db;
+        try {
+          db = new Database(WORLDS_DB_PATH, { fileMustExist: true });
+        } catch {
+          return [];
+        }
+        try {
+          const rows = db
+            .prepare("SELECT id FROM worlds WHERE status NOT IN ('destroyed', 'failed')")
+            .all();
+          return rows.map((r) => r.id).filter((id) => typeof id === 'string');
+        } catch {
+          return [];
+        } finally {
+          try { db.close(); } catch { /* ignore */ }
+        }
+      },
+      getClaudePidForWorld: async (worldId) =>
+        findClaudePid({ containerId: `olam-${worldId}-devbox` }),
+    });
+
+/**
+ * C1 — Serialize a WorldWatchdogState entry into the `socketHealth` sub-object
+ * shape shared by the /progress payload and the SPA TypeScript types.
+ *
+ * @param {string} worldId  Used to peek the per-world leaky-bucket count.
+ * @param {import('./world-watchdog.mjs').WorldWatchdogState} entry
+ * @returns {object}
+ */
+function buildSocketHealthPayload(worldId, entry) {
+  const payload = {
+    verdict: entry.lastVerdict,
+    signals: entry.lastSignals,
+    pid: entry.lastPid,
+    lastTickAt: entry.lastTickAt,
+  };
+  // Attach recovery sub-object only when OLAM_AUTO_RECOVER is non-false.
+  if (_watchdogRecovery) {
+    payload.recovery = {
+      mode: _watchdogAutoRecoverMode,
+      restartsInWindow: _watchdogLeakyBucket
+        ? _watchdogLeakyBucket.peek(worldId).totalInWindow
+        : 0,
+      lastRestartAt: null, // tracking per-world last-restart-at is a future enhancement
+    };
+  }
+  return payload;
+}
+
 // ── Phase 1a / B1 (PR3): engine-select + await-before-listen ─────
 //
 // Decision 15: the async KubernetesEngine factory MUST be fully awaited
@@ -3774,12 +3945,13 @@ for (const sig of ['SIGTERM', 'SIGINT']) {
     console.log(`received ${sig}, shutting down`);
     stopEvents();
     prPoller.stop();
-    // worldsDbReconciler + worldActivityTracker are null in SERVE-ONLY mode.
+    // worldsDbReconciler + worldActivityTracker + worldWatchdog are null in SERVE-ONLY mode.
     worldsDbReconciler?.stop();
     stopWorldsSnapshotLoop();
     stopTunnelsSnapshotLoop();
     stopListeningSnapshotLoop();
     worldActivityTracker?.stop();
+    worldWatchdog?.stop();
     if (serversSnapshotTimer) { clearTimeout(serversSnapshotTimer); serversSnapshotTimer = null; }
     hostStream.close();
     if (ndjsonSpanSink) ndjsonSpanSink.close().catch(() => {});

package/host-cp/src/world-watchdog-pid-lookup.mjs

@@ -0,0 +1,119 @@
+/**
+ * world-watchdog-pid-lookup.mjs — host-visible PID lookup for the world watchdog.
+ *
+ * Uses `docker top <containerId>` to enumerate processes inside a world's
+ * container and returns the host-visible PID of the claude process.
+ *
+ * `docker top` output format (Linux Docker / Colima):
+ *   UID   PID   PPID  C  STIME  TTY  TIME     CMD
+ *   root  1234  1     0  10:00  ?    00:00:00  node /usr/local/bin/claude ...
+ *
+ * The PID column (index 1 in default ps output) is already the host-visible
+ * PID. On Mac/Colima the container runs inside a Linux VM so `docker top`
+ * returns PIDs within the VM's PID namespace — these are NOT the macOS host
+ * PIDs, but they ARE the PIDs visible from within the Linux layer (where
+ * /proc reads happen). This is the same namespace the watchdog probes use
+ * when reading /proc/<pid>/wchan etc., so the PIDs are correct for probe use.
+ *
+ * Inject `docker` for tests (avoids spawning real docker processes).
+ *
+ * @see docs/architecture/world-watchdog.md
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Default docker executor — shells out to the real `docker` CLI.
+ *
+ * @param {string} containerId
+ * @returns {Promise<string>}  stdout from `docker top <containerId>`
+ */
+async function defaultDockerTop(containerId) {
+  const { stdout } = await execFileAsync('docker', ['top', containerId], {
+    timeout: 5_000,
+  });
+  return stdout;
+}
+
+/**
+ * Parse the stdout from `docker top` and extract host-visible PIDs whose
+ * CMD column matches a claude process.
+ *
+ * docker top default output columns (ps -ef format):
+ *   UID PID PPID C STIME TTY TIME CMD
+ * Indices: 0=UID, 1=PID, 2=PPID, ..., 7+=CMD (rest of line after 7 columns).
+ *
+ * @param {string} stdout  Raw output from `docker top <id>`
+ * @returns {number[]}     Host-visible PIDs of matching claude processes, sorted ascending.
+ */
+export function parseDockerTopOutput(stdout) {
+  const lines = stdout.split('\n').filter((l) => l.trim().length > 0);
+  if (lines.length < 2) return []; // header only or empty
+
+  // Skip the header line (first line contains column names).
+  const dataLines = lines.slice(1);
+
+  const pids = [];
+  for (const line of dataLines) {
+    // Split on any whitespace — `docker top` columns are space-separated.
+    // CMD may contain spaces; split into at most 8 parts (last = full CMD string).
+    const parts = line.trim().split(/\s+/);
+    if (parts.length < 8) continue;
+
+    const pid = parseInt(parts[1], 10);
+    if (!Number.isFinite(pid) || pid <= 0) continue;
+
+    // parts[7] onward is the CMD. Rejoin the remainder.
+    const cmd = parts.slice(7).join(' ');
+
+    // Match: `claude` as standalone binary, or `node` process running claude.
+    if (/(?:^|\/)claude(\s|$)/.test(cmd) || /node[^\s]*\s+.*[/\\]claude(?:\s|$)/.test(cmd)) {
+      pids.push(pid);
+    }
+  }
+
+  return pids.sort((a, b) => a - b);
+}
+
+/**
+ * Find the host-visible PID of the claude process running inside a container.
+ *
+ * Returns the lowest matching PID (parent process heuristic — the supervisor
+ * claude process has a lower PID than any child workers it spawns).
+ *
+ * Fail-soft:
+ *   - docker unreachable / container not found → null + log
+ *   - no claude process in the container → null (silent)
+ *   - multiple claude processes → return the lowest PID
+ *
+ * @param {{
+ *   containerId: string,
+ *   dockerTop?: (containerId: string) => Promise<string>,
+ *   log?: (msg: string) => void,
+ * }} opts
+ * @returns {Promise<number | null>}
+ */
+export async function findClaudePid({
+  containerId,
+  dockerTop = defaultDockerTop,
+  log = (m) => console.log(`[world-watchdog-pid-lookup] ${m}`),
+}) {
+  if (!containerId) return null;
+
+  let stdout;
+  try {
+    stdout = await dockerTop(containerId);
+  } catch (err) {
+    log(`docker top ${containerId} failed: ${err?.message ?? err}`);
+    return null;
+  }
+
+  const pids = parseDockerTopOutput(stdout);
+  if (pids.length === 0) return null;
+
+  // Lowest PID = the parent/supervisor process.
+  return pids[0];
+}