@pleri/olam-cli 0.1.170 → 0.1.174

package/host-cp/src/port-bridge-manager.mjs

@@ -10,6 +10,7 @@ import path from 'node:path';
 
 const DOCKER_HOST = process.env.DOCKER_HOST ?? 'docker-cli';
 const SOCAT_IMAGE = 'alpine/socat';
+const SOCAT_IMAGE_TAGGED = 'alpine/socat:latest';
 const HOST_PORT_MIN = 25000;
 const HOST_PORT_MAX = 25999;
 const INFRA_PORTS = new Set([8080, 7681, 7682]);
@@ -83,11 +84,73 @@ async function dockerApiBase() {
 }
 
 /**
- * Create and start a socat bridge container. Returns containerId.
+ * Detect whether a docker error message indicates the image is missing
+ * (and therefore a `docker pull` retry would help). Docker uses a handful
+ * of phrasings across CLI + HTTP API surfaces.
+ */
+function isImageMissingError(message) {
+  if (!message) return false;
+  return /Unable to find image|pull access denied|manifest unknown|No such image|not found in (the )?(repository|registry)/i.test(
+    message,
+  );
+}
+
+/**
+ * Pull alpine/socat:latest via docker CLI. Used by the bare-node bridge
+ * create path's fallback retry. 60s budget — image is ~5MB; real pull
+ * is typically <2s.
+ *
+ * @returns {{ok: boolean, stderr: string}}
+ */
+function pullSocatViaCli() {
+  const r = spawnSync('docker', ['pull', SOCAT_IMAGE_TAGGED], {
+    encoding: 'utf-8',
+    timeout: 60_000,
+  });
+  return {
+    ok: r.status === 0,
+    stderr: (r.stderr ?? '').trim() || (r.stdout ?? '').trim(),
+  };
+}
+
+/**
+ * Pull alpine/socat:latest via Docker HTTP API. Used by the container-mode
+ * bridge create path's fallback retry. Streams the pull progress body so
+ * Docker actually performs the pull (it's a streaming endpoint).
+ *
+ * @param {string} apiBase — Docker HTTP API base URL
+ * @returns {Promise<{ok: boolean, stderr: string}>}
+ */
+async function pullSocatViaHttpApi(apiBase) {
+  try {
+    const resp = await fetch(
+      `${apiBase}/images/create?fromImage=${encodeURIComponent(SOCAT_IMAGE)}&tag=latest`,
+      { method: 'POST', signal: AbortSignal.timeout(60_000) },
+    );
+    if (!resp.ok) {
+      const body = await resp.text().catch(() => '');
+      return { ok: false, stderr: `pull failed: ${resp.status} ${body}` };
+    }
+    // Drain the streaming progress body — Docker only completes the pull
+    // when the response is consumed.
+    await resp.text();
+    return { ok: true, stderr: '' };
+  } catch (err) {
+    return { ok: false, stderr: err?.message ?? String(err) };
+  }
+}
+
+/**
+ * Create and start a socat bridge container.
+ *
+ * Returns `{ containerId, pulledImage }` — `pulledImage: true` indicates the
+ * function had to fall back to `docker pull alpine/socat:latest` (issue #964
+ * — preflight in `olam services up` should normally have already pulled it).
+ *
  * @param {string} worldId
  * @param {number} containerPort
  * @param {number} hostPort
- * @returns {Promise<string>} containerId
+ * @returns {Promise<{containerId: string, pulledImage: boolean}>}
  */
 async function createBridgeContainer(worldId, containerPort, hostPort) {
   const name = bridgeContainerName(worldId, containerPort);
@@ -111,11 +174,28 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
       'TCP-LISTEN:' + containerPort + ',fork,reuseaddr',
       'TCP:' + devboxName + ':' + containerPort,
     ];
-    const result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    let result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    let pulledImage = false;
+
+    // Issue #964 fallback: if docker run failed because the image is missing,
+    // pull it and retry once. This covers hosts where `olam services up`
+    // didn't run the preflight (e.g. fresh Hazel install, docker restart
+    // pruned the image, etc.).
+    if (result.status !== 0 && isImageMissingError(result.stderr ?? '')) {
+      const pull = pullSocatViaCli();
+      if (!pull.ok) {
+        throw new Error(
+          `alpine/socat image missing and pull failed: ${pull.stderr || 'unknown error'}`,
+        );
+      }
+      pulledImage = true;
+      result = spawnSync('docker', args, { encoding: 'utf-8', timeout: 10000 });
+    }
+
     if (result.status !== 0) {
       throw new Error(result.stderr?.trim() || 'docker run failed');
     }
-    return result.stdout.trim(); // container ID
+    return { containerId: result.stdout.trim(), pulledImage };
   }
 
   // container mode: Docker HTTP API
@@ -135,7 +215,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     },
   };
 
-  const createResp = await fetch(
+  const doCreate = () => fetch(
     `${apiBase}/containers/create?name=${encodeURIComponent(name)}`,
     {
       method: 'POST',
@@ -145,6 +225,28 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     },
   );
 
+  let createResp = await doCreate();
+  let pulledImage = false;
+
+  // Issue #964 fallback for HTTP API path. Docker returns 404 with a body
+  // like {"message":"No such image: alpine/socat:latest"} when the image
+  // is missing.
+  if (!createResp.ok && createResp.status === 404) {
+    const body = await createResp.text().catch(() => '');
+    if (isImageMissingError(body)) {
+      const pull = await pullSocatViaHttpApi(apiBase);
+      if (!pull.ok) {
+        throw new Error(
+          `alpine/socat image missing and pull failed: ${pull.stderr || 'unknown error'}`,
+        );
+      }
+      pulledImage = true;
+      createResp = await doCreate();
+    } else {
+      throw new Error(`container create failed: 404 ${body}`);
+    }
+  }
+
   if (!createResp.ok) {
     const body = await createResp.text().catch(() => '');
     // If container already exists (409), try to get its ID
@@ -155,7 +257,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
       );
       if (inspectResp.ok) {
         const info = await inspectResp.json();
-        return info.Id;
+        return { containerId: info.Id, pulledImage };
       }
     }
     throw new Error(`container create failed: ${createResp.status} ${body}`);
@@ -171,7 +273,7 @@ async function createBridgeContainer(worldId, containerPort, hostPort) {
     throw new Error(`container start failed: ${startResp.status}`);
   }
 
-  return containerId;
+  return { containerId, pulledImage };
 }
 
 async function removeBridgeContainer(containerName, containerId) {
@@ -196,7 +298,7 @@ async function removeBridgeContainer(containerName, containerId) {
  *
  * @param {string} worldId
  * @param {number} containerPort
- * @returns {Promise<{hostPort: number, containerPort: number, url: string, containerId: string}>}
+ * @returns {Promise<{hostPort: number, containerPort: number, url: string, containerId: string, pulledImage?: boolean}>}
  */
 export async function exposePort(worldId, containerPort) {
   if (INFRA_PORTS.has(containerPort)) {
@@ -220,18 +322,22 @@ export async function exposePort(worldId, containerPort) {
   }
 
   const containerName = bridgeContainerName(worldId, containerPort);
-  const containerId = await createBridgeContainer(worldId, containerPort, hostPort);
+  const { containerId, pulledImage } = await createBridgeContainer(worldId, containerPort, hostPort);
 
   const entry = { worldId, containerPort, hostPort, containerId, containerName };
   registry.set(key, entry);
   saveState();
 
-  return {
+  const result = {
     hostPort,
     containerPort,
     url: `http://${HOST_IP}:${hostPort}`,
     containerId,
   };
+  // Only attach pulledImage when true so existing callers/tests don't see
+  // an unexpected key when the preflight succeeded.
+  if (pulledImage) result.pulledImage = true;
+  return result;
 }
 
 /**

package/host-cp/src/server.mjs

@@ -41,7 +41,11 @@ import {
   WorldStartupFailureKind,
 } from '../lifecycle/index.mjs';
 import { createHostStream, newStreamId } from './host-stream.mjs';
-import { createNdjsonSpanSink } from '../observability/ndjson-span-sink.mjs';
+import {
+  createNdjsonSpanSink,
+  attachBetaResponseEvents,
+} from '../observability/ndjson-span-sink.mjs';
+import { betaResponseEmitter } from '@olam/auth-client';
 import { attemptRecovery, findScenarioForKind } from '../recovery/index.mjs';
 import { detectHaltChunk } from './halt-detect.mjs';
 import { spawnUpgraderContainer } from './upgrade-spawner.mjs';
@@ -72,6 +76,11 @@ import { readSecret as readPlanChatSecret, SECRET_PATH as PLAN_CHAT_SECRET_PATH
 import { createPrMergePoller } from './pr-merge-poller.mjs';
 import { parse as parseYaml } from 'yaml';
 import { startWorldsDbReconciler } from './worlds-db-source.mjs';
+import {
+  reconcileWorldsWithDocker,
+  defaultListContainerNames,
+} from './boot-reconciler.mjs';
+import { startWorldActivityTracker } from './world-activity-tracker.mjs';
 import { authSecretHint } from './auth-secret-hint.mjs';
 import * as tunnelManager from './world-tunnel-manager.mjs';
 import * as bridgeManager from './port-bridge-manager.mjs';
@@ -83,6 +92,7 @@ import {
 } from './routes/process-port.mjs';
 import { instrumentHandler, renderMetrics } from './metrics.mjs';
 import { handleDispatchFromEmail } from './lib/email-dispatch.mjs';
+import { emitTierSuggestion } from '../dispatch/auto-tier-scheduler.mjs';
 
 // ── Deployment-mode detection ─────────────────────────────────────
 //
@@ -490,6 +500,20 @@ const ndjsonSpanSink = await createNdjsonSpanSink({ hostStream }).catch((err) =>
   return null;
 });
 
+// Wire @olam/auth-client `beta-response` events (Anthropic SDK 0.96+ beta
+// flags — thinking-token-count, cache-diagnostics, future passthrough) into
+// the NDJSON trace as `withCredential.beta-response` spans. Opt-in via the
+// caller's `withCredential('claude', fn, { betas: [...] })` options; when
+// no caller opts in, the emitter never fires and this subscription is a
+// no-op. See docs/decisions/047-anthropic-sdk-beta-flags.md.
+if (ndjsonSpanSink) {
+  try {
+    attachBetaResponseEvents({ sink: ndjsonSpanSink, emitter: betaResponseEmitter });
+  } catch (err) {
+    console.warn(`[trace] beta-response wire unavailable: ${err?.message ?? err}`);
+  }
+}
+
 // A4: coalesce docker-event bursts into a single servers.snapshot. World
 // boot fires `create` + `start` + healthcheck transitions in <100ms; we
 // don't want a broadcast storm. Window matches plan-source.md P3 target.
@@ -922,6 +946,58 @@ const server = http.createServer(instrumentHandler('host-cp', async (req, res) =
     if (handled) return;
   }
 
+  // /api/telemetry/planning-sessions — B9: aggregate planning_sessions by
+  // session_source for the canonical-surface bet's adoption signal. Per
+  // plan-chat-spa-canonical-surface plan § Operator workflow seam falsification
+  // trigger: if plan-chat-spa weekly-active sessions < 60% of control-plane/app
+  // by 2026-Q3, freeze plan-chat-spa feature work. This endpoint is the
+  // data source for that measurement.
+  //
+  // Query param: ?since=YYYY-MM-DD (required; rejects with 400 otherwise).
+  // Response: { plan_chat_spa: N, control_plane_app: M, unknown: K, ratio: pct }
+  // where ratio = plan_chat_spa / (plan_chat_spa + control_plane_app) * 100,
+  // null if denominator is 0.
+  if (url.pathname === '/api/telemetry/planning-sessions' && req.method === 'GET') {
+    const since = url.searchParams.get('since');
+    if (!since || !/^\d{4}-\d{2}-\d{2}$/.test(since)) {
+      res.writeHead(400, { 'Content-Type': 'application/json' });
+      return res.end(JSON.stringify({
+        error: 'bad_request',
+        message: 'Missing or malformed `since` query param. Expected YYYY-MM-DD.',
+      }));
+    }
+    // B9 ships the endpoint CONTRACT + the session_source schema column.
+    // The query implementation goes through plan-chat-service.mjs (which
+    // owns the pg pool); this host-cp handler currently emits a 503 with
+    // a structured "not_implemented" marker so callers can verify the
+    // endpoint shape + auth + query-param parsing without the data path.
+    //
+    // Phase G of this epic adds the plan-chat-service handler that this
+    // endpoint will proxy to. Until then operators can run the SQL
+    // directly:
+    //   SELECT COALESCE(session_source, 'unknown'), COUNT(*)
+    //   FROM planning_sessions
+    //   WHERE created_at >= $since
+    //   GROUP BY 1;
+    //
+    // Notify-C: ship contract + schema; defer data path to Phase G.
+    res.writeHead(503, { 'Content-Type': 'application/json' });
+    return res.end(JSON.stringify({
+      error: 'not_implemented',
+      message: 'B9 ships the endpoint contract + session_source schema column. ' +
+        'Aggregation handler scaffolded in plan-chat-service.mjs lands in Phase G.',
+      since,
+      contractShape: {
+        plan_chat_spa: 0,
+        control_plane_app: 0,
+        unknown: 0,
+        ratio: null,
+        since: '<YYYY-MM-DD>',
+        asOf: '<ISO 8601>',
+      },
+    }));
+  }
+
   // /api/version/status: returns the current version snapshot (baked SHA
   // vs operator's local HEAD). No auth required beyond the existing gate
   // (already applied above). Phase 1 only — detection, no auto-upgrade.
@@ -2224,6 +2300,23 @@ const server = http.createServer(instrumentHandler('host-cp', async (req, res) =
         return jsonReply(res, 400, { error: 'invalid_json', message: err.message });
       }
       try {
+        // Auto-tier-scheduler v1 (ADR 042): emit an informational
+        // `dispatch.tier-suggestion` event BEFORE handing off to the
+        // dispatch handler. Pure-informational — never changes which
+        // provider actually runs. The dispatch payload's optional
+        // `tierSpec` ({ kind?, expectedDurationMs?, explicitTier? })
+        // carries the shape; absent it, the heuristic falls through to
+        // its default (`cloudflare-sandbox`).
+        if (dispatch && typeof dispatch.worldId === 'string') {
+          try {
+            emitTierSuggestion({
+              worldId: dispatch.worldId,
+              dispatchSpec: dispatch.tierSpec ?? {},
+              currentTier: null,
+              hostStream,
+            });
+          } catch { /* never let a hint surface break dispatch */ }
+        }
         const result = await handleDispatchFromEmail({
           dispatch,
           worlds: WORLDS,
@@ -3182,6 +3275,16 @@ startWorldsSnapshotLoop();
 startTunnelsSnapshotLoop();
 startListeningSnapshotLoop();
 
+// Closes #965: live thought_count + total_cost_usd updates from each
+// active world's Claude session JSONL. Periodic (60s default) so Rico's
+// scheduling loop can read fresh values from the `worlds` table and
+// SPAs can subscribe to the `world.activity.tick` event. Fail-soft per
+// world: missing/malformed JSONL never crashes the loop.
+const worldActivityTracker = startWorldActivityTracker({
+  dbPath: WORLDS_DB_PATH,
+  broadcaster: hostStream,
+});
+
 // ── Phase 1a / B1 (PR3): engine-select + await-before-listen ─────
 //
 // Decision 15: the async KubernetesEngine factory MUST be fully awaited
@@ -3208,6 +3311,22 @@ const hostCpEngine = await (async () => {
   return createDockerEngine({ dockerHost: DOCKER_HOST });
 })();
 
+// ── Boot-time worlds.db ↔ docker reconciler (issue #963) ─────────────
+//
+// One-shot pass: if a container is alive but worlds.db has no row, insert
+// a status='reconciled' row so host-cp can see it. If worlds.db says a
+// world is running/active but the container is gone, mark it 'orphaned'.
+// Fail-soft: docker unreachable or DB unavailable → log + continue boot.
+// Runs BEFORE server.listen() so the first request sees reconciled state.
+try {
+  await reconcileWorldsWithDocker({
+    dbPath: WORLDS_DB_PATH,
+    listContainerNames: () => defaultListContainerNames(DOCKER_API_BASE, console.log),
+  });
+} catch (err) {
+  console.error(`[boot-reconciler] unexpected error (continuing boot): ${err.message}`);
+}
+
 server.listen(PORT, '0.0.0.0', () => {
   console.log(`olam-host-cp B3 listening on :${PORT}`);
   console.log(`  DOCKER_HOST=${DOCKER_HOST}`);
@@ -3248,6 +3367,7 @@ for (const sig of ['SIGTERM', 'SIGINT']) {
     stopWorldsSnapshotLoop();
     stopTunnelsSnapshotLoop();
     stopListeningSnapshotLoop();
+    worldActivityTracker.stop();
     if (serversSnapshotTimer) { clearTimeout(serversSnapshotTimer); serversSnapshotTimer = null; }
     hostStream.close();
     if (ndjsonSpanSink) ndjsonSpanSink.close().catch(() => {});