@pleri/olam-cli 0.1.103 → 0.1.105

package/host-cp/src/server.mjs

@@ -34,6 +34,7 @@ import { computeProgress } from './world-progress.mjs';
 import { createPrCache } from './pr-cache.mjs';
 import { fetchContainerSecret } from './container-secret-fetcher.mjs';
 import { subscribeDockerEvents } from './docker-events.mjs';
+import { createHostStream, newStreamId } from './host-stream.mjs';
 import { spawnUpgraderContainer } from './upgrade-spawner.mjs';
 import { parseProxyPath, perWorldBase, proxyToWorld } from './proxy.mjs';
 import { StartupToken } from './auth.mjs';
@@ -422,11 +423,219 @@ let prListCacheEntry = null; // /api/prs — 60s TTL global PR list for Cmd+K
 const sseGate = new SseGate({ maxConcurrent: SSE_CAP });
 
 // Subscribe to docker events on boot. Unsubscribed at shutdown.
+//
+// Two consumers now: the legacy secret-cache invalidator (Phase F-2-B B3)
+// AND the host-stream broadcaster (sse-consolidation Phase A4). The
+// broadcaster pushes a debounced `servers.snapshot` whenever an
+// `olam-*-devbox` container starts/stops so the SPA can drop its
+// poll-every-2s `useListeningServers` loop.
+const hostStream = createHostStream({ log: (m) => console.log(`[host-stream] ${m}`) });
+
+// A4: coalesce docker-event bursts into a single servers.snapshot. World
+// boot fires `create` + `start` + healthcheck transitions in <100ms; we
+// don't want a broadcast storm. Window matches plan-source.md P3 target.
+const SERVERS_SNAPSHOT_DEBOUNCE_MS = 100;
+let serversSnapshotTimer = null;
+function scheduleServersSnapshot() {
+  if (serversSnapshotTimer) return;
+  serversSnapshotTimer = setTimeout(() => {
+    serversSnapshotTimer = null;
+    const ids = Object.keys(WORLDS).sort();
+    hostStream.broadcast('servers.snapshot', {
+      servers: ids.map((id) => ({ id, port: WORLDS[id] })),
+      snapshotAt: new Date().toISOString(),
+    });
+  }, SERVERS_SNAPSHOT_DEBOUNCE_MS);
+}
+
 const stopEvents = subscribeDockerEvents({
   dockerHost: DOCKER_HOST,
-  onWorldRestart: (worldId) => cache.invalidate(worldId),
+  onWorldRestart: (worldId) => {
+    cache.invalidate(worldId);
+    // Only push the snapshot for olam-* world containers; the
+    // onWorldRestart handler is already filtered to `olam-<id>-devbox`
+    // by handleEvent in docker-events.mjs, so any worldId reaching
+    // this callback is by construction an olam world.
+    scheduleServersSnapshot();
+  },
 });
 
+// Initial servers.snapshot so subscribers connecting before any docker
+// event have a current snapshot to replay from.
+scheduleServersSnapshot();
+
+// A5: 1-Hz worlds.db hash-diff loop. The worlds-db reconciler above
+// reacts to fs.watch + every 30s; that's enough to keep the WORLDS
+// registry consistent, but the SPA wants sub-second freshness on
+// create/destroy. Hash the composed worlds list each tick, broadcast
+// only on change. Pure-server-side polling — never touches the network.
+const WORLDS_SNAPSHOT_TICK_MS = 1000;
+let lastWorldsHash = '';
+let worldsSnapshotTimer = null;
+let worldsSnapshotInFlight = false;
+
+async function tickWorldsSnapshot() {
+  if (worldsSnapshotInFlight) return;
+  worldsSnapshotInFlight = true;
+  try {
+    const worlds = await composeWorldsSources(worldsSources);
+    // Stable JSON for hashing — JSON.stringify is order-sensitive, so
+    // sort the array by id before serialising. Sources already return
+    // stable shape per world.
+    const sorted = [...worlds].sort((a, b) =>
+      String(a?.id ?? '').localeCompare(String(b?.id ?? '')),
+    );
+    const json = JSON.stringify(sorted);
+    const { createHash } = await import('node:crypto');
+    const hash = createHash('sha1').update(json).digest('hex');
+    if (hash === lastWorldsHash) return;
+    lastWorldsHash = hash;
+    hostStream.broadcast('world.snapshot', {
+      worlds: sorted,
+      snapshotAt: new Date().toISOString(),
+    });
+  } catch (err) {
+    // Don't kill the loop on transient compose errors. Worst case the
+    // next tick succeeds and the diff is detected then.
+    console.warn(`[host-stream] worlds.snapshot tick failed: ${err?.message ?? err}`);
+  } finally {
+    worldsSnapshotInFlight = false;
+  }
+}
+
+function startWorldsSnapshotLoop() {
+  // Initial broadcast on host-cp start so the first subscriber gets
+  // current state before the next tick fires.
+  void tickWorldsSnapshot();
+  worldsSnapshotTimer = setInterval(() => { void tickWorldsSnapshot(); }, WORLDS_SNAPSHOT_TICK_MS);
+}
+
+function stopWorldsSnapshotLoop() {
+  if (worldsSnapshotTimer) {
+    clearInterval(worldsSnapshotTimer);
+    worldsSnapshotTimer = null;
+  }
+}
+
+// ── Phase B-bonus: tunnels.snapshot + listening.snapshot ─────────────
+//
+// Phase A wired `world.snapshot` (worlds.db) and `servers.snapshot`
+// (docker events for the host worlds-port map). The remaining Phase-B
+// hooks need additional broadcasters so they can drop their poll loops:
+//
+//   - usePublishedTunnels  → `tunnels.snapshot` (per-row, was 4 req/min × N rows on /api/worlds/<id>/tunnels)
+//   - useListeningServers  → `listening.snapshot` (per-world, scoped via filter)
+//
+// Both broadcast a SINGLE aggregate event with all worlds' data; hooks
+// filter by worldId client-side. host-stream caches by event type so a
+// reconnecting tab gets the full set on connect (replay).
+//
+// Cadence: 2s for tunnels (state changes on startTunnel/stopTunnel are
+// fast, polling catches missed updates), 5s for listening servers
+// (matches existing per-world poll cadence). The poller is server-side
+// so the network never carries N polls per refresh window.
+//
+// Note: `tunnels.snapshot` is hash-debounced so idle windows produce
+// zero broadcasts. Same pattern as world.snapshot.
+
+const TUNNELS_SNAPSHOT_TICK_MS = 2_000;
+const LISTENING_SNAPSHOT_TICK_MS = 5_000;
+let tunnelsSnapshotTimer = null;
+let listeningSnapshotTimer = null;
+let lastTunnelsHash = '';
+let lastListeningHash = '';
+
+async function tickTunnelsSnapshot() {
+  try {
+    const byWorld = tunnelManager.getAllTunnels();
+    // Stable JSON for hashing — sort worldIds + service names so
+    // identical state produces identical hash.
+    const ids = Object.keys(byWorld).sort();
+    const stable = ids.map((id) => ({
+      worldId: id,
+      tunnels: [...byWorld[id]].sort((a, b) => a.name.localeCompare(b.name)),
+    }));
+    const json = JSON.stringify(stable);
+    const { createHash } = await import('node:crypto');
+    const hash = createHash('sha1').update(json).digest('hex');
+    if (hash === lastTunnelsHash) return;
+    lastTunnelsHash = hash;
+    hostStream.broadcast('tunnels.snapshot', {
+      worlds: stable,
+      snapshotAt: new Date().toISOString(),
+    });
+  } catch (err) {
+    console.warn(`[host-stream] tunnels.snapshot tick failed: ${err?.message ?? err}`);
+  }
+}
+
+async function tickListeningSnapshot() {
+  try {
+    // Lazy import to keep the boot path light; the poller module also
+    // owns the docker exec details.
+    const { getListeningServers } = await import('./listening-server-poller.mjs');
+    const bridgeManager = await import('./port-bridge-manager.mjs');
+    const ids = Object.keys(WORLDS).sort();
+    if (ids.length === 0) return;
+    // Per-world fetch in parallel; failures yield empty array for that world.
+    const perWorld = await Promise.all(
+      ids.map(async (id) => {
+        try {
+          const snapshot = await getListeningServers(id);
+          const bridges = bridgeManager.getWorldBridges(id);
+          const bridgeByPort = new Map(bridges.map((b) => [b.containerPort, b]));
+          const servers = (snapshot?.servers ?? []).map((s) => ({
+            ...s,
+            bridge: bridgeByPort.get(s.port) ?? null,
+          }));
+          return { worldId: id, servers };
+        } catch {
+          return { worldId: id, servers: [] };
+        }
+      }),
+    );
+    const stable = perWorld.map((w) => ({
+      worldId: w.worldId,
+      servers: [...w.servers].sort((a, b) => a.port - b.port),
+    }));
+    const json = JSON.stringify(stable);
+    const { createHash } = await import('node:crypto');
+    const hash = createHash('sha1').update(json).digest('hex');
+    if (hash === lastListeningHash) return;
+    lastListeningHash = hash;
+    hostStream.broadcast('listening.snapshot', {
+      worlds: stable,
+      snapshotAt: new Date().toISOString(),
+    });
+  } catch (err) {
+    console.warn(`[host-stream] listening.snapshot tick failed: ${err?.message ?? err}`);
+  }
+}
+
+function startTunnelsSnapshotLoop() {
+  void tickTunnelsSnapshot();
+  tunnelsSnapshotTimer = setInterval(() => { void tickTunnelsSnapshot(); }, TUNNELS_SNAPSHOT_TICK_MS);
+}
+
+function stopTunnelsSnapshotLoop() {
+  if (tunnelsSnapshotTimer) {
+    clearInterval(tunnelsSnapshotTimer);
+    tunnelsSnapshotTimer = null;
+  }
+}
+
+function startListeningSnapshotLoop() {
+  void tickListeningSnapshot();
+  listeningSnapshotTimer = setInterval(() => { void tickListeningSnapshot(); }, LISTENING_SNAPSHOT_TICK_MS);
+}
+
+function stopListeningSnapshotLoop() {
+  if (listeningSnapshotTimer) {
+    clearInterval(listeningSnapshotTimer);
+    listeningSnapshotTimer = null;
+  }
+}
+
 /**
  * Resolve worldId → secret. Cache hit returns immediately; miss fetches
  * from the docker-socket-proxy + caches.
@@ -1117,6 +1326,35 @@ const server = http.createServer(async (req, res) => {
     return;
   }
 
+  // sse-consolidation Phase A2: multiplexed host-stream endpoint.
+  //
+  // One long-lived SSE per SPA tab replaces 20+ setInterval polls. Event
+  // types include `world.snapshot`, `servers.snapshot`, etc. — wiring
+  // for each lives in the broadcaster + its subscribers. This handler
+  // just opens the channel, emits `ready`, replays cached snapshots,
+  // and registers for future broadcasts. Auth is the global gate above
+  // (cookie + Bearer matching auth.isAuthorized).
+  if (url.pathname === '/api/host-stream' && req.method === 'GET') {
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream; charset=utf-8',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    });
+    res.write(':\n\n'); // initial heartbeat — keeps proxies from buffering
+
+    const streamId = newStreamId();
+    res.write(`event: ready\ndata: ${JSON.stringify({
+      streamId,
+      serverTime: new Date().toISOString(),
+    })}\n\n`);
+
+    const cleanup = hostStream.addSink(res);
+    req.on('close', cleanup);
+    req.on('error', cleanup);
+    return;
+  }
+
   // ── Per-world credential telemetry routes ─────────────────────────────
   //
   // These are called by the in-world HTTPS proxy when it intercepts
@@ -2382,6 +2620,14 @@ tunnelManager.probeAllOnStartup().catch((err) => {
   console.error(`tunnel startup probe failed: ${err.message}`);
 });
 
+// Start the 1-Hz worlds.db hash-diff loop after the server boots so
+// the initial broadcast happens once the route is reachable.
+startWorldsSnapshotLoop();
+// Phase B-bonus: start tunnel + listening snapshot loops. Both
+// hash-debounce so idle windows produce zero broadcasts.
+startTunnelsSnapshotLoop();
+startListeningSnapshotLoop();
+
 server.listen(PORT, '0.0.0.0', () => {
   console.log(`olam-host-cp B3 listening on :${PORT}`);
   console.log(`  DOCKER_HOST=${DOCKER_HOST}`);
@@ -2411,6 +2657,11 @@ for (const sig of ['SIGTERM', 'SIGINT']) {
     stopEvents();
     prPoller.stop();
     worldsDbReconciler.stop();
+    stopWorldsSnapshotLoop();
+    stopTunnelsSnapshotLoop();
+    stopListeningSnapshotLoop();
+    if (serversSnapshotTimer) { clearTimeout(serversSnapshotTimer); serversSnapshotTimer = null; }
+    hostStream.close();
     clearInterval(versionPollTimer);
     cache.clear();
     server.close(() => process.exit(0));

package/host-cp/src/world-tunnel-manager.mjs

@@ -201,6 +201,29 @@ export function stopTunnel(worldId, serviceName) {
   saveState();
 }
 
+/**
+ * Return tunnel state for ALL worlds, keyed by worldId. Used by the
+ * host-stream broadcaster (sse-consolidation Phase B-bonus) to push a
+ * `tunnels.snapshot` whenever the registry changes — replaces the
+ * SPA's per-row `usePublishedTunnels` poll loop.
+ *
+ * @returns {{ [worldId: string]: Array<{name: string, port: number, url: string|null, status: string}> }}
+ */
+export function getAllTunnels() {
+  /** @type {Record<string, Array<{name: string, port: number, url: string|null, status: string}>>} */
+  const byWorld = {};
+  for (const entry of registry.values()) {
+    if (!byWorld[entry.worldId]) byWorld[entry.worldId] = [];
+    byWorld[entry.worldId].push({
+      name: entry.serviceName,
+      port: entry.port,
+      url: entry.url,
+      status: entry.status,
+    });
+  }
+  return byWorld;
+}
+
 /**
  * Return the current tunnel state for all services in a world.
  * @param {string} worldId

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pleri/olam-cli",
-  "version": "0.1.103",
+  "version": "0.1.105",
   "type": "module",
   "bin": {
     "olam": "./bin/olam.cjs"