@pleri/olam-cli 0.1.170 → 0.1.174

package/host-cp/src/world-activity-tracker.mjs

@@ -0,0 +1,392 @@
+/**
+ * WorldActivityTracker — periodic scanner that turns each active world's
+ * Claude session JSONL into `thought_count` + `total_cost_usd` updates on
+ * the `worlds` table (~/.olam/worlds.db), plus a `world.activity.tick`
+ * event on the host-stream broadcaster.
+ *
+ * Closes #965. Pre-fix, `olam_status <world>` always reported
+ * `Cost $0.0000 / Thoughts 0` because nothing wrote those columns after
+ * world creation. Rico (the orchestrator) reads those fields to decide
+ * whether a world is progressing or stalled, so as far as it was
+ * concerned every world was frozen.
+ *
+ * Design notes:
+ *   - **JSONL path is operator-configurable.** Default contract per #965
+ *     is `~/.olam/worlds/<id>/state/claude-main.jsonl`; override the
+ *     template via `OLAM_WORLD_JSONL_PATH_TEMPLATE`. On this host the
+ *     producer for the default path is not yet shipped (Claude Code
+ *     writes to `~/.claude/projects/<sanitized>/<uuid>.jsonl` by
+ *     default), so values stay at 0 until either the producer lands or
+ *     the env override repoints the scanner.
+ *   - **Dedupe by `message.id`.** Claude SDK JSONL emits multiple lines
+ *     per assistant API turn (one per content block), each carrying the
+ *     SAME `message.id` + the SAME `usage` block. Naive sum-by-line
+ *     double-counts. We dedupe by `message.id` for usage totals and
+ *     count unique-message-id as `thoughtCount`.
+ *   - **Idempotent.** Re-scanning the same JSONL produces the same
+ *     numbers; safe to run at any cadence.
+ *   - **Fail-soft per world.** A bad JSONL line, missing file, or
+ *     unreadable handle never crashes the loop — the failing world is
+ *     skipped with a debug log and the next world proceeds.
+ *
+ * Cadence: `OLAM_WORLD_ACTIVITY_TICK_MS` (default 60_000).
+ *
+ * Wire-in: `server.mjs` constructs once with `{ db, broadcaster }` after
+ * both are ready and calls `.stop()` from the SIGTERM/SIGINT handler.
+ *
+ * @see ../host-stream.mjs   broadcaster API
+ * @see ../worlds-db-source.mjs  read-only DB open pattern (model for
+ *      `tryOpenDb` here, though tracker WRITES not reads).
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import readline from 'node:readline';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// TODO(rates): source live model rates from auth-service or a config
+// file. For now we anchor on Claude Opus per-million baseline ($3 input
+// / $15 output) — the issue surface is "value advances post-creation",
+// not "is dollar-accurate to 4 decimals". When per-model rates land,
+// pluck the model id from the assistant message and dispatch.
+const INPUT_USD_PER_M_TOKENS = 3.0;
+const OUTPUT_USD_PER_M_TOKENS = 15.0;
+
+const DEFAULT_TICK_MS = 60_000;
+
+/**
+ * Resolve a per-world JSONL path from an operator-supplied template
+ * string. The template supports a single `{worldId}` placeholder, and a
+ * leading `~/` is expanded to `os.homedir()`.
+ *
+ * @param {string} template
+ * @param {string} worldId
+ * @returns {string}
+ */
+export function resolveJsonlPath(template, worldId) {
+  const swapped = template.replace(/\{worldId\}/g, worldId);
+  if (swapped.startsWith('~/')) {
+    return path.join(os.homedir(), swapped.slice(2));
+  }
+  return swapped;
+}
+
+/**
+ * Scan a single JSONL file and return aggregate counts.
+ *
+ * @param {string} jsonlPath
+ * @returns {Promise<{thoughtCount:number, inputTokens:number, outputTokens:number, costUsd:number, lastActivityAt:string|null}>}
+ */
+export async function scanWorldJsonl(jsonlPath) {
+  const seenMessageIds = new Set();
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let lastTimestamp = null;
+
+  let stream;
+  try {
+    stream = fs.createReadStream(jsonlPath, { encoding: 'utf8' });
+  } catch {
+    // ENOENT or permission error — return zeros.
+    return zeroStats();
+  }
+
+  // createReadStream defers ENOENT to the 'error' event; convert to a
+  // rejected promise so the caller's try/catch sees it uniformly.
+  const errorPromise = new Promise((_, reject) => {
+    stream.on('error', reject);
+  });
+
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  const linesPromise = (async () => {
+    for await (const line of rl) {
+      if (!line) continue;
+      let row;
+      try {
+        row = JSON.parse(line);
+      } catch {
+        // Skip malformed lines silently — the JSONL has been observed
+        // to contain partial writes during active sessions.
+        continue;
+      }
+      if (!row || row.type !== 'assistant') continue;
+      const msg = row.message;
+      if (!msg || typeof msg !== 'object') continue;
+
+      const messageId = typeof msg.id === 'string' ? msg.id : null;
+      if (messageId === null) continue;
+      if (seenMessageIds.has(messageId)) continue;
+      seenMessageIds.add(messageId);
+
+      const usage = msg.usage;
+      if (usage && typeof usage === 'object') {
+        if (Number.isFinite(usage.input_tokens)) {
+          inputTokens += Number(usage.input_tokens);
+        }
+        if (Number.isFinite(usage.output_tokens)) {
+          outputTokens += Number(usage.output_tokens);
+        }
+      }
+
+      if (typeof row.timestamp === 'string') {
+        // Lexicographic comparison is correct on ISO-8601 with consistent zone.
+        if (lastTimestamp === null || row.timestamp > lastTimestamp) {
+          lastTimestamp = row.timestamp;
+        }
+      }
+    }
+  })();
+
+  try {
+    await Promise.race([linesPromise, errorPromise]);
+  } catch {
+    return zeroStats();
+  } finally {
+    try { stream.destroy(); } catch { /* ignore */ }
+  }
+
+  const costUsd =
+    (inputTokens / 1_000_000) * INPUT_USD_PER_M_TOKENS +
+    (outputTokens / 1_000_000) * OUTPUT_USD_PER_M_TOKENS;
+
+  return {
+    thoughtCount: seenMessageIds.size,
+    inputTokens,
+    outputTokens,
+    costUsd,
+    lastActivityAt: lastTimestamp,
+  };
+}
+
+function zeroStats() {
+  return {
+    thoughtCount: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    costUsd: 0,
+    lastActivityAt: null,
+  };
+}
+
+/**
+ * @typedef {object} WorldActivityTrackerDeps
+ * @property {string}  [dbPath]            Path to worlds.db; defaults to
+ *   `OLAM_WORLDS_DB` env var or `~/.olam/worlds.db`.
+ * @property {object}  [broadcaster]       Object with `.broadcast(type, payload)`
+ *   (e.g. the return of `createHostStream`). Optional — when absent
+ *   events are skipped but DB writes still happen.
+ * @property {number}  [intervalMs]        Tick cadence. Defaults to
+ *   `OLAM_WORLD_ACTIVITY_TICK_MS` env or 60000.
+ * @property {string}  [jsonlPathTemplate] JSONL path template.
+ *   `{worldId}` is replaced per world. Defaults to
+ *   `OLAM_WORLD_JSONL_PATH_TEMPLATE` env or
+ *   `~/.olam/worlds/{worldId}/state/claude-main.jsonl`.
+ * @property {(msg: string) => void} [log] Defaults to `console.log`.
+ * @property {(msg: string) => void} [debug] Optional verbose log; defaults
+ *   to no-op (debug-level skips on missing JSONL would be noisy).
+ * @property {(cb: () => void, ms: number) => any} [setTimer] Injectable
+ *   `setInterval` for tests.
+ * @property {(handle: any) => void} [clearTimer] Injectable
+ *   `clearInterval` for tests.
+ * @property {() => Date} [now] Clock injection for tests.
+ */
+
+/**
+ * @typedef {object} WorldActivityTrackerHandle
+ * @property {() => void} stop
+ * @property {() => Promise<number>} tickNow Run one tick synchronously
+ *   (returns the count of worlds processed). Exposed for tests.
+ */
+
+/**
+ * Start the world activity tracker. Returns a `{ stop, tickNow }`
+ * handle. Safe to call before the worlds.db file exists — the tracker
+ * skip-with-log until the file appears.
+ *
+ * @param {WorldActivityTrackerDeps} [deps]
+ * @returns {WorldActivityTrackerHandle}
+ */
+export function startWorldActivityTracker(deps = {}) {
+  const log = deps.log ?? ((m) => console.log(`[world-activity] ${m}`));
+  const debug = deps.debug ?? (() => {});
+  const setTimer = deps.setTimer ?? ((cb, ms) => setInterval(cb, ms));
+  const clearTimer = deps.clearTimer ?? ((h) => clearInterval(h));
+  const now = deps.now ?? (() => new Date());
+
+  const intervalMs =
+    deps.intervalMs ??
+    parseInt(process.env.OLAM_WORLD_ACTIVITY_TICK_MS ?? `${DEFAULT_TICK_MS}`, 10);
+
+  const dbPath =
+    deps.dbPath ??
+    process.env.OLAM_WORLDS_DB ??
+    path.join(os.homedir(), '.olam', 'worlds.db');
+
+  const jsonlPathTemplate =
+    deps.jsonlPathTemplate ??
+    process.env.OLAM_WORLD_JSONL_PATH_TEMPLATE ??
+    '~/.olam/worlds/{worldId}/state/claude-main.jsonl';
+
+  const broadcaster = deps.broadcaster ?? null;
+
+  let stopped = false;
+  let inFlight = false;
+  let intervalHandle = null;
+
+  /**
+   * One tick: open DB, read active worlds, scan each JSONL, write back,
+   * emit event. Returns the count of worlds processed.
+   *
+   * @returns {Promise<number>}
+   */
+  async function tick() {
+    if (stopped) return 0;
+    if (inFlight) {
+      // Skip overlap — slow filesystem must not pile up ticks.
+      debug('tick skipped: previous tick still in flight');
+      return 0;
+    }
+    inFlight = true;
+
+    let db = null;
+    let processed = 0;
+    try {
+      let Database;
+      try {
+        Database = require('better-sqlite3');
+      } catch (err) {
+        // better-sqlite3 unavailable (e.g. container without native
+        // build) — degrade silently.
+        log(`better-sqlite3 unavailable; skipping tick: ${err.message}`);
+        return 0;
+      }
+
+      try {
+        db = new Database(dbPath, { fileMustExist: true });
+      } catch (err) {
+        // SQLITE_CANTOPEN (file absent) is the expected first-boot
+        // case; everything else is worth surfacing.
+        if (err.code !== 'SQLITE_CANTOPEN') {
+          log(`open ${dbPath} failed: ${err.message}`);
+        } else {
+          debug(`${dbPath} not present yet; skipping tick`);
+        }
+        return 0;
+      }
+
+      let activeWorlds;
+      try {
+        activeWorlds = db
+          .prepare(
+            "SELECT id FROM worlds WHERE status NOT IN ('destroyed', 'failed')",
+          )
+          .all();
+      } catch (err) {
+        log(`query active worlds failed: ${err.message}`);
+        return 0;
+      }
+
+      const updateStmt = db.prepare(
+        `UPDATE worlds
+            SET thought_count = ?,
+                total_cost_usd = ?,
+                updated_at = ?
+          WHERE id = ?`,
+      );
+
+      for (const row of activeWorlds) {
+        if (stopped) break;
+        const worldId = row.id;
+        if (typeof worldId !== 'string') continue;
+        const jsonlPath = resolveJsonlPath(jsonlPathTemplate, worldId);
+
+        let stats;
+        try {
+          stats = await scanWorldJsonl(jsonlPath);
+        } catch (err) {
+          // Defence in depth — scanWorldJsonl is already fail-soft, but
+          // this catches anything unforeseen at the call seam.
+          debug(`scan ${worldId} failed: ${err.message}`);
+          continue;
+        }
+
+        const updatedAt = now().toISOString();
+        try {
+          updateStmt.run(
+            stats.thoughtCount,
+            stats.costUsd,
+            updatedAt,
+            worldId,
+          );
+        } catch (err) {
+          log(`update ${worldId} failed: ${err.message}`);
+          continue;
+        }
+
+        if (broadcaster && typeof broadcaster.broadcast === 'function') {
+          try {
+            broadcaster.broadcast('world.activity.tick', {
+              worldId,
+              thoughtCount: stats.thoughtCount,
+              costUsd: stats.costUsd,
+              inputTokens: stats.inputTokens,
+              outputTokens: stats.outputTokens,
+              lastActivityAt: stats.lastActivityAt,
+              updatedAt,
+            });
+          } catch (err) {
+            log(`broadcast ${worldId} failed: ${err.message}`);
+          }
+        }
+
+        processed += 1;
+      }
+    } finally {
+      if (db) {
+        try { db.close(); } catch { /* ignore */ }
+      }
+      inFlight = false;
+    }
+
+    return processed;
+  }
+
+  // Kick off an initial tick on next event-loop turn so callers can
+  // attach test spies before any DB work happens.
+  setImmediate(() => {
+    if (stopped) return;
+    void tick().catch((err) => {
+      log(`initial tick crashed: ${err?.message ?? err}`);
+    });
+  });
+
+  intervalHandle = setTimer(() => {
+    void tick().catch((err) => {
+      log(`tick crashed: ${err?.message ?? err}`);
+    });
+  }, intervalMs);
+  // Don't pin the event loop on shutdown.
+  if (intervalHandle && typeof intervalHandle.unref === 'function') {
+    intervalHandle.unref();
+  }
+
+  log(
+    `started: db=${dbPath} template=${jsonlPathTemplate} interval=${intervalMs}ms`,
+  );
+
+  return {
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      if (intervalHandle !== null) {
+        try { clearTimer(intervalHandle); } catch { /* ignore */ }
+        intervalHandle = null;
+      }
+    },
+    tickNow: tick,
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pleri/olam-cli",
-  "version": "0.1.170",
+  "version": "0.1.174",
   "type": "module",
   "bin": {
     "olam": "./bin/olam.cjs"