@pleri/olam-cli 0.1.173 → 0.1.175

package/host-cp/src/boot-reconciler.mjs

@@ -0,0 +1,238 @@
+/**
+ * Boot-time reconciler — sync worlds.db with live docker state.
+ *
+ * Problem (issue #963): after Colima / userspace restart, host-cp can
+ * start with worlds.db rows that no longer reflect docker reality. The
+ * existing `worlds-db-source.mjs` reconciler runs DB→registry (reads
+ * 'running' rows and adds them to in-memory WORLDS). It does NOT heal
+ * the inverse case: a container is alive on docker but worlds.db has
+ * no row (Hazel coral-sky-2478 scenario), or worlds.db says a world is
+ * running but the container is gone (orphaned row).
+ *
+ * This module fills both gaps with a one-shot pass at boot:
+ *
+ *   1. List `olam-*-devbox` containers via the docker API.
+ *   2. For each container, derive the worldId (strip prefix + suffix).
+ *   3. Cross-check against worlds.db rows:
+ *        - container alive, row exists                 → no-op
+ *        - container alive, row missing                → INSERT (status=reconciled)
+ *        - row says running/active, container missing  → UPDATE status=orphaned
+ *
+ * Fail-soft: if the docker daemon is unreachable OR better-sqlite3 is
+ * not available, the function logs a warning and returns without
+ * throwing. Server boot continues.
+ *
+ * Idempotent: a second invocation against the same docker + DB state
+ * produces no further changes (existing rows are skipped at step 3a,
+ * already-orphaned rows are skipped at step 3c).
+ *
+ * Coordination with issue #962: the dedup logic in `olam create` handles
+ * per-call deduplication; this reconciler handles boot-time cleanup.
+ * They don't conflict — both operate on the worlds.db source-of-truth.
+ */
+
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+const CONTAINER_NAME_PATTERN = /^\/?(olam-(.+)-devbox)$/;
+
+/**
+ * @typedef {object} ReconcileDeps
+ * @property {string} dbPath                                  Path to worlds.db
+ * @property {() => Promise<string[] | null>} listContainerNames  Returns null when docker is unreachable
+ * @property {(msg: string) => void} [log]                    Defaults to console.log
+ * @property {() => string} [now]                             ISO timestamp generator (overridable for tests)
+ * @property {(path: string) => unknown | null} [openDb]      Overridable DB opener (tests inject fakes)
+ */
+
+/**
+ * @typedef {object} ReconcileSummary
+ * @property {number} inserted   Number of new rows inserted (reconciled containers)
+ * @property {number} orphaned   Number of rows transitioned to status='orphaned'
+ * @property {number} skipped    Containers/rows where no change was needed
+ * @property {boolean} dockerUnreachable
+ * @property {boolean} dbUnavailable
+ */
+
+/**
+ * Extract a worldId from a docker container name.
+ * Accepts either `olam-foo-bar-1234-devbox` or `/olam-foo-bar-1234-devbox`
+ * (the docker API prefixes container names with a slash).
+ *
+ * @param {string} name
+ * @returns {string | null}
+ */
+export function extractWorldIdFromContainerName(name) {
+  if (typeof name !== 'string') return null;
+  const match = CONTAINER_NAME_PATTERN.exec(name);
+  if (!match) return null;
+  const worldId = match[2];
+  if (!worldId || worldId.length === 0) return null;
+  return worldId;
+}
+
+/**
+ * Default docker container lister. Hits the Docker Engine API.
+ * Returns null on any failure (fail-soft).
+ *
+ * @param {string} dockerApiBase   e.g. 'http://docker-socket-proxy:2375'
+ * @param {(msg: string) => void} log
+ * @returns {Promise<string[] | null>}
+ */
+export async function defaultListContainerNames(dockerApiBase, log) {
+  if (!dockerApiBase || dockerApiBase === 'http://localhost:2375') {
+    // 'docker-cli' sentinel; no API available in this deployment mode.
+    log('[boot-reconciler] docker API unavailable (bare-node mode); skipping');
+    return null;
+  }
+  try {
+    const filters = encodeURIComponent(JSON.stringify({ name: ['olam-'] }));
+    const url = `${dockerApiBase}/containers/json?filters=${filters}`;
+    const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
+    if (!res.ok) {
+      log(`[boot-reconciler] docker /containers/json returned ${res.status}; skipping`);
+      return null;
+    }
+    const data = await res.json();
+    if (!Array.isArray(data)) return [];
+    const names = [];
+    for (const container of data) {
+      const list = container?.Names;
+      if (!Array.isArray(list)) continue;
+      for (const n of list) {
+        if (typeof n === 'string') names.push(n);
+      }
+    }
+    return names;
+  } catch (err) {
+    log(`[boot-reconciler] docker query failed: ${err.message}; skipping`);
+    return null;
+  }
+}
+
+/**
+ * Default DB opener. Loads better-sqlite3 dynamically so a missing
+ * native build degrades gracefully instead of crashing host-cp boot.
+ *
+ * @param {string} dbPath
+ * @param {(msg: string) => void} log
+ * @returns {unknown | null}
+ */
+export function defaultOpenDb(dbPath, log) {
+  try {
+    const Database = require('better-sqlite3');
+    return new Database(dbPath, { fileMustExist: true });
+  } catch (err) {
+    if (err && err.code === 'MODULE_NOT_FOUND') {
+      log('[boot-reconciler] better-sqlite3 not available; skipping');
+    } else if (err && err.code === 'SQLITE_CANTOPEN') {
+      log(`[boot-reconciler] ${dbPath} not found; nothing to reconcile`);
+    } else {
+      log(`[boot-reconciler] failed to open ${dbPath}: ${err.message}`);
+    }
+    return null;
+  }
+}
+
+/**
+ * Run a single boot-time reconciliation pass. Pure and dep-injected
+ * for testability.
+ *
+ * @param {ReconcileDeps} deps
+ * @returns {Promise<ReconcileSummary>}
+ */
+export async function reconcileWorldsWithDocker(deps) {
+  const log = deps.log ?? console.log;
+  const now = deps.now ?? (() => new Date().toISOString());
+  const openDb = deps.openDb ?? ((p) => defaultOpenDb(p, log));
+
+  const summary = {
+    inserted: 0,
+    orphaned: 0,
+    skipped: 0,
+    dockerUnreachable: false,
+    dbUnavailable: false,
+  };
+
+  const containerNames = await deps.listContainerNames();
+  if (containerNames === null) {
+    summary.dockerUnreachable = true;
+    return summary;
+  }
+
+  const liveWorldIds = new Set();
+  for (const name of containerNames) {
+    const worldId = extractWorldIdFromContainerName(name);
+    if (worldId) liveWorldIds.add(worldId);
+  }
+
+  const db = openDb(deps.dbPath);
+  if (!db) {
+    summary.dbUnavailable = true;
+    return summary;
+  }
+
+  try {
+    /** @type {Array<{ id: string, status: string }>} */
+    let rows;
+    try {
+      rows = db.prepare('SELECT id, status FROM worlds').all();
+    } catch (err) {
+      log(`[boot-reconciler] query failed: ${err.message}; skipping`);
+      summary.dbUnavailable = true;
+      return summary;
+    }
+
+    const dbWorlds = new Map(rows.map((r) => [r.id, r.status]));
+
+    // Pass 1: containers alive but missing from DB → insert.
+    const insertStmt = db.prepare(
+      `INSERT INTO worlds
+       (id, name, status, repos, branch, port_offset, workspace_path,
+        compute_provider, total_cost_usd, thought_count, created_at, updated_at)
+       VALUES (?, ?, 'reconciled', '[]', 'main', 0, ?, 'docker', 0, 0, ?, ?)`,
+    );
+    for (const worldId of liveWorldIds) {
+      if (dbWorlds.has(worldId)) {
+        summary.skipped += 1;
+        continue;
+      }
+      const ts = now();
+      const workspacePath = `~/.olam/worlds/${worldId}`;
+      try {
+        insertStmt.run(worldId, worldId, workspacePath, ts, ts);
+        summary.inserted += 1;
+        log(`[boot-reconciler] inserted reconciled row for ${worldId} (container alive, no DB row)`);
+      } catch (err) {
+        log(`[boot-reconciler] failed to insert ${worldId}: ${err.message}`);
+      }
+    }
+
+    // Pass 2: DB says alive but container missing → mark orphaned.
+    const orphanStmt = db.prepare(
+      `UPDATE worlds SET status = 'orphaned', updated_at = ? WHERE id = ?`,
+    );
+    const aliveStatuses = new Set(['running', 'active', 'creating']);
+    for (const [worldId, status] of dbWorlds) {
+      if (liveWorldIds.has(worldId)) continue;
+      if (!aliveStatuses.has(status)) continue;
+      try {
+        orphanStmt.run(now(), worldId);
+        summary.orphaned += 1;
+        log(`[boot-reconciler] marked ${worldId} as orphaned (was '${status}', container missing)`);
+      } catch (err) {
+        log(`[boot-reconciler] failed to mark ${worldId} orphaned: ${err.message}`);
+      }
+    }
+
+    log(
+      `[boot-reconciler] complete: inserted=${summary.inserted} orphaned=${summary.orphaned} ` +
+        `skipped=${summary.skipped} live-containers=${liveWorldIds.size}`,
+    );
+  } finally {
+    try { db.close?.(); } catch { /* ignore */ }
+  }
+
+  return summary;
+}

package/host-cp/src/port-bridge-manager.mjs

@@ -10,6 +10,7 @@ import path from 'node:path';
 
 const DOCKER_HOST = process.env.DOCKER_HOST ?? 'docker-cli';
 const SOCAT_IMAGE = 'alpine/socat';
+const SOCAT_IMAGE_TAGGED = 'alpine/socat:latest';
 const HOST_PORT_MIN = 25000;
 const HOST_PORT_MAX = 25999;
 const INFRA_PORTS = new Set([8080, 7681, 7682]);
@@ -83,11 +84,73 @@ async function dockerApiBase() {
 }
 
 /**
- * Create and start a socat bridge container. Returns containerId.
+ * Detect whether a docker error message indicates the image is missing
+ * (and therefore a `docker pull` retry would help). Docker uses a handful
+ * of phrasings across CLI + HTTP API surfaces.
+ */
+function isImageMissingError(message) {
+  if (!message) return false;
+  return /Unable to find image|pull access denied|manifest unknown|No such image|not found in (the )?(repository|registry)/i.test(
+    message,
+  );
+}
+
+/**
+ * Pull alpine/socat:latest via docker CLI. Used by the bare-node bridge
+ * create path's fallback retry. 60s budget — image is ~5MB; real pull
+ * is typically <2s.
+ *
+ * @returns {{ok: boolean, stderr: string}}
+ */
+function pullSocatViaCli() {
+  const r = spawnSync('docker', ['pull', SOCAT_IMAGE_TAGGED], {
+    encoding: 'utf-8',
+    timeout: 60_000,
+  });
+  return {
+    ok: r.status === 0,
+    stderr: (r.stderr ?? '').trim() || (r.stdout ?? '').trim(),
+  };
+}
+
+/**
+ * Pull alpine/socat:latest via Docker HTTP API. Used by the container-mode
+ * bridge create path's fallback retry. Streams the pull progress body so
+ * Docker actually performs the pull (it's a streaming endpoint).
+ *
+ * @param {string} apiBase — Docker HTTP API base URL
+ * @returns {Promise<{ok: boolean, stderr: string}>}
+ */
+async function pullSocatViaHttpApi(apiBase) {
+  try {
+    const resp = await fetch(
+      `${apiBase}/images/create?fromImage=${encodeURIComponent(SOCAT_IMAGE)}&tag=latest`,
+      { method: 'POST', signal: AbortSignal.timeout(60_000) },
+    );
+    if (!resp.ok) {
+      const body = await resp.text().catch(() => '');
+      return { ok: false, stderr: `pull failed: ${resp.status} ${body}` };
+    }
+    // Drain the streaming progress body — Docker only completes the pull
+    // when the response is consumed.
+    await resp.text();
+    return { ok: true, stderr: '' };
+  } catch (err) {
+    return { ok: false, stderr: err?.message ?? String(err) };
+  }
+}
+
+/**
+ * Create and start a socat bridge container.
+ *
+ * Returns `{ containerId, pulledImage }` — `pulledImage: true` indicates the
+ * function had to fall back to `docker pull alpine/socat:latest` (issue #964
+ * — preflight in `olam services up` should normally have already pulled it).
+ *
  * @param {string} worldId
  * @param {number} containerPort
  * @param {number} hostPort
- * @returns {Promise<string>} containerId
+ * @returns {Promise<{containerId: string, pulledImage: boolean}>}
  */
 async function createBridgeContainer(worldId, containerPort, hostPort) {
   const name = bridgeContainerName(worldId, containerPort);
@@ -111,11 +174,28 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
       'TCP-LISTEN:' + containerPort + ',fork,reuseaddr',
       'TCP:' + devboxName + ':' + containerPort,
     ];
-    const result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    let result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    let pulledImage = false;
+
+    // Issue #964 fallback: if docker run failed because the image is missing,
+    // pull it and retry once. This covers hosts where `olam services up`
+    // didn't run the preflight (e.g. fresh Hazel install, docker restart
+    // pruned the image, etc.).
+    if (result.status !== 0 && isImageMissingError(result.stderr ?? '')) {
+      const pull = pullSocatViaCli();
+      if (!pull.ok) {
+        throw new Error(
+          `alpine/socat image missing and pull failed: ${pull.stderr || 'unknown error'}`,
+        );
+      }
+      pulledImage = true;
+      result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    }
+
     if (result.status !== 0) {
       throw new Error(result.stderr?.trim() || 'docker run failed');
     }
-    return result.stdout.trim(); // container ID
+    return { containerId: result.stdout.trim(), pulledImage };
   }
 
   // container mode: Docker HTTP API
@@ -135,7 +215,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     },
   };
 
-  const createResp = await fetch(
+  const doCreate = () => fetch(
     `${apiBase}/containers/create?name=${encodeURIComponent(name)}`,
     {
       method: 'POST',
@@ -145,6 +225,28 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     },
   );
 
+  let createResp = await doCreate();
+  let pulledImage = false;
+
+  // Issue #964 fallback for HTTP API path. Docker returns 404 with a body
+  // like {"message":"No such image: alpine/socat:latest"} when the image
+  // is missing.
+  if (!createResp.ok && createResp.status === 404) {
+    const body = await createResp.text().catch(() => '');
+    if (isImageMissingError(body)) {
+      const pull = await pullSocatViaHttpApi(apiBase);
+      if (!pull.ok) {
+        throw new Error(
+          `alpine/socat image missing and pull failed: ${pull.stderr || 'unknown error'}`,
+        );
+      }
+      pulledImage = true;
+      createResp = await doCreate();
+    } else {
+      throw new Error(`container create failed: 404 ${body}`);
+    }
+  }
+
   if (!createResp.ok) {
     const body = await createResp.text().catch(() => '');
     // If container already exists (409), try to get its ID
@@ -155,7 +257,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
       );
       if (inspectResp.ok) {
         const info = await inspectResp.json();
-        return info.Id;
+        return { containerId: info.Id, pulledImage };
       }
     }
     throw new Error(`container create failed: ${createResp.status} ${body}`);
@@ -171,7 +273,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     throw new Error(`container start failed: ${startResp.status}`);
   }
 
-  return containerId;
+  return { containerId, pulledImage };
 }
 
 async function removeBridgeContainer(containerName, containerId) {
@@ -196,7 +298,7 @@ async function removeBridgeContainer(containerName, containerId) {
  *
  * @param {string} worldId
  * @param {number} containerPort
- * @returns {Promise<{hostPort: number, containerPort: number, url: string, containerId: string}>}
+ * @returns {Promise<{hostPort: number, containerPort: number, url: string, containerId: string, pulledImage?: boolean}>}
  */
 export async function exposePort(worldId, containerPort) {
   if (INFRA_PORTS.has(containerPort)) {
@@ -220,18 +322,22 @@ export async function exposePort(worldId, containerPort) {
   }
 
   const containerName = bridgeContainerName(worldId, containerPort);
-  const containerId = await createBridgeContainer(worldId, containerPort, hostPort);
+  const { containerId, pulledImage } = await createBridgeContainer(worldId, containerPort, hostPort);
 
   const entry = { worldId, containerPort, hostPort, containerId, containerName };
   registry.set(key, entry);
   saveState();
 
-  return {
+  const result = {
     hostPort,
     containerPort,
     url: `http://${HOST_IP}:${hostPort}`,
     containerId,
   };
+  // Only attach pulledImage when true so existing callers/tests don't see
+  // an unexpected key when the preflight succeeded.
+  if (pulledImage) result.pulledImage = true;
+  return result;
 }
 
 /**

package/host-cp/src/server.mjs

@@ -76,6 +76,11 @@ import { readSecret as readPlanChatSecret, SECRET_PATH as PLAN_CHAT_SECRET_PATH
 import { createPrMergePoller } from './pr-merge-poller.mjs';
 import { parse as parseYaml } from 'yaml';
 import { startWorldsDbReconciler } from './worlds-db-source.mjs';
+import {
+  reconcileWorldsWithDocker,
+  defaultListContainerNames,
+} from './boot-reconciler.mjs';
+import { startWorldActivityTracker } from './world-activity-tracker.mjs';
 import { authSecretHint } from './auth-secret-hint.mjs';
 import * as tunnelManager from './world-tunnel-manager.mjs';
 import * as bridgeManager from './port-bridge-manager.mjs';
@@ -3270,6 +3275,16 @@ startWorldsSnapshotLoop();
 startTunnelsSnapshotLoop();
 startListeningSnapshotLoop();
 
+// Closes #965: live thought_count + total_cost_usd updates from each
+// active world's Claude session JSONL. Periodic (60s default) so Rico's
+// scheduling loop can read fresh values from the `worlds` table and
+// SPAs can subscribe to the `world.activity.tick` event. Fail-soft per
+// world: missing/malformed JSONL never crashes the loop.
+const worldActivityTracker = startWorldActivityTracker({
+  dbPath: WORLDS_DB_PATH,
+  broadcaster: hostStream,
+});
+
 // ── Phase 1a / B1 (PR3): engine-select + await-before-listen ─────
 //
 // Decision 15: the async KubernetesEngine factory MUST be fully awaited
@@ -3296,6 +3311,22 @@ const hostCpEngine = await (async () => {
   return createDockerEngine({ dockerHost: DOCKER_HOST });
 })();
 
+// ── Boot-time worlds.db ↔ docker reconciler (issue #963) ─────────────
+//
+// One-shot pass: if a container is alive but worlds.db has no row, insert
+// a status='reconciled' row so host-cp can see it. If worlds.db says a
+// world is running/active but the container is gone, mark it 'orphaned'.
+// Fail-soft: docker unreachable or DB unavailable → log + continue boot.
+// Runs BEFORE server.listen() so the first request sees reconciled state.
+try {
+  await reconcileWorldsWithDocker({
+    dbPath: WORLDS_DB_PATH,
+    listContainerNames: () => defaultListContainerNames(DOCKER_API_BASE, console.log),
+  });
+} catch (err) {
+  console.error(`[boot-reconciler] unexpected error (continuing boot): ${err.message}`);
+}
+
 server.listen(PORT, '0.0.0.0', () => {
   console.log(`olam-host-cp B3 listening on :${PORT}`);
   console.log(`  DOCKER_HOST=${DOCKER_HOST}`);
@@ -3336,6 +3367,7 @@ for (const sig of ['SIGTERM', 'SIGINT']) {
     stopWorldsSnapshotLoop();
     stopTunnelsSnapshotLoop();
     stopListeningSnapshotLoop();
+    worldActivityTracker.stop();
     if (serversSnapshotTimer) { clearTimeout(serversSnapshotTimer); serversSnapshotTimer = null; }
     hostStream.close();
     if (ndjsonSpanSink) ndjsonSpanSink.close().catch(() => {});