mobygate 0.8.3 → 0.9.2

package/lib/captures-index.js

@@ -0,0 +1,524 @@
+/**
+ * Captures index — SQLite-backed search over the capture summary fields.
+ *
+ * Why: ~/.mobygate/captures/ accumulates hundreds of (json, summary.txt)
+ * pairs. Filesystem ls/grep gets unwieldy fast. This module mirrors a
+ * small set of structured fields from each capture into SQLite so the
+ * `mobygate captures query` CLI can filter by model, session, status,
+ * stop_reason, duration, recency, etc.
+ *
+ * Design constraints:
+ *   - Best-effort: if better-sqlite3 fails to load (native build broke,
+ *     unsupported platform), every export becomes a no-op and the proxy
+ *     keeps running. Indexing is purely a convenience layer.
+ *   - Synchronous via better-sqlite3. Each capture write is two ~100µs
+ *     calls — well below request-handling overhead.
+ *   - Single writer (the proxy) and occasional reader (the CLI). WAL
+ *     mode lets the CLI read while the proxy writes.
+ *   - Schema is forward-compatible: new columns added via ALTER TABLE
+ *     (see ensureSchema). Old captures keep working.
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+const MOBYGATE_DIR = join(homedir(), '.mobygate');
+const CAPTURE_DIR = join(MOBYGATE_DIR, 'captures');
+export const INDEX_DB_PATH = join(MOBYGATE_DIR, 'captures.sqlite');
+
+// Lazily loaded better-sqlite3 — gracefully degrades if it fails to load.
+let DatabaseCtor = null;
+let loadAttempted = false;
+let db = null;
+let loadError = null;
+
+async function loadDatabase() {
+  if (loadAttempted) return DatabaseCtor;
+  loadAttempted = true;
+  try {
+    const mod = await import('better-sqlite3');
+    DatabaseCtor = mod.default || mod;
+  } catch (e) {
+    loadError = e;
+    DatabaseCtor = null;
+  }
+  return DatabaseCtor;
+}
+
+async function getDB() {
+  if (db) return db;
+  const Ctor = await loadDatabase();
+  if (!Ctor) return null;
+
+  if (!existsSync(MOBYGATE_DIR)) mkdirSync(MOBYGATE_DIR, { recursive: true });
+
+  try {
+    db = new Ctor(INDEX_DB_PATH);
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+    ensureSchema(db);
+  } catch (e) {
+    loadError = e;
+    db = null;
+  }
+  return db;
+}
+
+function ensureSchema(d) {
+  d.exec(`
+    CREATE TABLE IF NOT EXISTS captures (
+      request_id          TEXT PRIMARY KEY,
+      ts                  TEXT NOT NULL,
+      path                TEXT NOT NULL,
+      session_key         TEXT,
+      session_source      TEXT,
+      model               TEXT,
+      resolved_model      TEXT,
+      stream              INTEGER,
+      max_tokens          INTEGER,
+      temperature         REAL,
+      system_bytes        INTEGER,
+      message_count       INTEGER,
+      message_bytes       INTEGER,
+      image_count         INTEGER,
+      tool_use_count      INTEGER,
+      tool_result_count   INTEGER,
+      tools_declared      INTEGER,
+      tool_schema_bytes   INTEGER,
+      total_input_bytes   INTEGER,
+      total_input_tokens  INTEGER,
+      first_user_text     TEXT,
+      last_user_text      TEXT,
+      status              TEXT,
+      duration_ms         INTEGER,
+      stop_reason         TEXT,
+      input_tokens        INTEGER,
+      cache_read_tokens   INTEGER,
+      cache_create_tokens INTEGER,
+      output_tokens       INTEGER,
+      cache_hit_pct       REAL,
+      json_path           TEXT,
+      summary_path        TEXT,
+      indexed_at          TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_captures_ts ON captures(ts DESC);
+    CREATE INDEX IF NOT EXISTS idx_captures_session ON captures(session_key, ts DESC);
+    CREATE INDEX IF NOT EXISTS idx_captures_status ON captures(status, ts DESC);
+    CREATE INDEX IF NOT EXISTS idx_captures_model ON captures(model);
+    CREATE INDEX IF NOT EXISTS idx_captures_stop ON captures(stop_reason);
+  `);
+}
+
+// ---------------------------------------------------------------------------
+// Field extraction — pull index columns out of an Anthropic-shape body.
+// Mirrors the analyzeAnthropic() logic in request-capture.js but produces
+// structured fields instead of formatted text.
+// ---------------------------------------------------------------------------
+
+function bytesOfContent(content) {
+  if (typeof content === 'string') return content.length;
+  if (!Array.isArray(content)) return 0;
+  return content.reduce((acc, b) => {
+    if (!b) return acc;
+    if (b.type === 'text') return acc + (b.text?.length || 0);
+    if (b.type === 'tool_result') {
+      const inner = typeof b.content === 'string' ? b.content : JSON.stringify(b.content || '');
+      return acc + inner.length;
+    }
+    if (b.type === 'tool_use') return acc + JSON.stringify(b.input || {}).length;
+    if (b.type === 'image') return acc + 100; // placeholder for image marker
+    return acc + JSON.stringify(b).length;
+  }, 0);
+}
+
+function extractTextFromContent(content) {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter((b) => b?.type === 'text')
+    .map((b) => b.text || '')
+    .join(' ')
+    .slice(0, 500);
+}
+
+export function extractIndexFields(body) {
+  const f = {
+    model: body?.model || null,
+    stream: body?.stream ? 1 : 0,
+    max_tokens: body?.max_tokens ?? body?.max_completion_tokens ?? null,
+    temperature: typeof body?.temperature === 'number' ? body.temperature : null,
+  };
+
+  // System bytes
+  const sys = body?.system;
+  if (typeof sys === 'string') f.system_bytes = sys.length;
+  else if (Array.isArray(sys)) f.system_bytes = sys.reduce((a, b) => a + (b?.text?.length || 0), 0);
+  else f.system_bytes = 0;
+
+  // Messages
+  const msgs = Array.isArray(body?.messages) ? body.messages : [];
+  f.message_count = msgs.length;
+  f.message_bytes = msgs.reduce((a, m) => a + bytesOfContent(m.content), 0);
+
+  let imageCount = 0;
+  let toolUseCount = 0;
+  let toolResultCount = 0;
+  for (const m of msgs) {
+    if (Array.isArray(m.content)) {
+      for (const b of m.content) {
+        if (b?.type === 'image') imageCount += 1;
+        if (b?.type === 'tool_use') toolUseCount += 1;
+        if (b?.type === 'tool_result') toolResultCount += 1;
+      }
+    }
+  }
+  f.image_count = imageCount;
+  f.tool_use_count = toolUseCount;
+  f.tool_result_count = toolResultCount;
+
+  // First + last user message previews — useful for "find that one request"
+  const userMsgs = msgs.filter((m) => m.role === 'user');
+  f.first_user_text = userMsgs.length ? extractTextFromContent(userMsgs[0].content) : null;
+  f.last_user_text = userMsgs.length ? extractTextFromContent(userMsgs[userMsgs.length - 1].content) : null;
+
+  // Tools
+  if (Array.isArray(body?.tools)) {
+    f.tools_declared = body.tools.length;
+    f.tool_schema_bytes = JSON.stringify(body.tools).length;
+  } else {
+    f.tools_declared = 0;
+    f.tool_schema_bytes = 0;
+  }
+
+  f.total_input_bytes = (f.system_bytes || 0) + (f.message_bytes || 0) + (f.tool_schema_bytes || 0);
+  f.total_input_tokens = Math.ceil(f.total_input_bytes / 4); // rough estimate, matches summary
+
+  return f;
+}
+
+// ---------------------------------------------------------------------------
+// Write paths — called from request-capture.js
+// ---------------------------------------------------------------------------
+
+const insertStmt = (d) => d.prepare(`
+  INSERT OR REPLACE INTO captures (
+    request_id, ts, path, session_key, session_source,
+    model, stream, max_tokens, temperature,
+    system_bytes, message_count, message_bytes,
+    image_count, tool_use_count, tool_result_count,
+    tools_declared, tool_schema_bytes,
+    total_input_bytes, total_input_tokens,
+    first_user_text, last_user_text,
+    json_path, summary_path
+  ) VALUES (
+    @request_id, @ts, @path, @session_key, @session_source,
+    @model, @stream, @max_tokens, @temperature,
+    @system_bytes, @message_count, @message_bytes,
+    @image_count, @tool_use_count, @tool_result_count,
+    @tools_declared, @tool_schema_bytes,
+    @total_input_bytes, @total_input_tokens,
+    @first_user_text, @last_user_text,
+    @json_path, @summary_path
+  )
+`);
+
+export async function indexCapture({ requestId, ts, path, body, sessionKey, sessionSource, jsonPath, summaryPath }) {
+  const d = await getDB();
+  if (!d) return;
+  try {
+    const fields = extractIndexFields(body || {});
+    insertStmt(d).run({
+      request_id: requestId,
+      ts: ts || new Date().toISOString(),
+      path: path || '',
+      session_key: sessionKey || null,
+      session_source: sessionSource || null,
+      json_path: jsonPath || null,
+      summary_path: summaryPath || null,
+      ...fields,
+    });
+  } catch (e) {
+    // Best-effort — never break the request path.
+    if (process.env.MOBY_DEBUG_INDEX) console.warn(`[captures-index] insert failed: ${e.message}`);
+  }
+}
+
+const updateResponseStmt = (d) => d.prepare(`
+  UPDATE captures
+     SET status = @status,
+         duration_ms = @duration_ms,
+         stop_reason = @stop_reason,
+         resolved_model = COALESCE(@resolved_model, resolved_model),
+         input_tokens = @input_tokens,
+         cache_read_tokens = @cache_read_tokens,
+         cache_create_tokens = @cache_create_tokens,
+         output_tokens = @output_tokens,
+         cache_hit_pct = @cache_hit_pct
+   WHERE request_id = @request_id
+`);
+
+export async function updateCaptureResponse({ requestId, usage, durationMs, status, stopReason, model }) {
+  const d = await getDB();
+  if (!d) return;
+  try {
+    const u = usage || {};
+    const totalInput = (u.input_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+    const cacheHitPct = totalInput > 0
+      ? +(((u.cache_read_input_tokens || 0) / totalInput) * 100).toFixed(2)
+      : 0;
+    updateResponseStmt(d).run({
+      request_id: requestId,
+      status: status || null,
+      duration_ms: durationMs ?? null,
+      stop_reason: stopReason || null,
+      resolved_model: model || null,
+      input_tokens: u.input_tokens ?? 0,
+      cache_read_tokens: u.cache_read_input_tokens ?? 0,
+      cache_create_tokens: u.cache_creation_input_tokens ?? 0,
+      output_tokens: u.output_tokens ?? 0,
+      cache_hit_pct: cacheHitPct,
+    });
+  } catch (e) {
+    if (process.env.MOBY_DEBUG_INDEX) console.warn(`[captures-index] update failed: ${e.message}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Read paths — called from the CLI
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a SELECT with optional filters. Returns rows ordered by ts DESC.
+ * Filters:
+ *   - text:        LIKE %text% on first_user_text / last_user_text / session_key
+ *   - sinceMs:     ts >= now() - sinceMs
+ *   - model:       model LIKE %model% (matches resolved_model too)
+ *   - sessionKey:  exact match on session_key
+ *   - status:      exact match on status
+ *   - stopReason:  exact match on stop_reason
+ *   - minDurationMs / maxDurationMs
+ *   - hasTools:    tools_declared > 0
+ *   - limit:       default 20
+ */
+export async function queryCaptures(filters = {}) {
+  const d = await getDB();
+  if (!d) return { rows: [], error: loadError?.message || 'index unavailable' };
+
+  const where = [];
+  const params = {};
+
+  if (filters.text) {
+    where.push(`(first_user_text LIKE @text OR last_user_text LIKE @text OR session_key LIKE @text)`);
+    params.text = `%${filters.text}%`;
+  }
+  if (filters.sinceMs) {
+    const cutoff = new Date(Date.now() - filters.sinceMs).toISOString();
+    where.push(`ts >= @cutoff`);
+    params.cutoff = cutoff;
+  }
+  if (filters.model) {
+    where.push(`(model LIKE @model OR resolved_model LIKE @model)`);
+    params.model = `%${filters.model}%`;
+  }
+  if (filters.sessionKey) {
+    where.push(`session_key = @sessionKey`);
+    params.sessionKey = filters.sessionKey;
+  }
+  if (filters.status) {
+    where.push(`status = @status`);
+    params.status = filters.status;
+  }
+  if (filters.stopReason) {
+    where.push(`stop_reason = @stopReason`);
+    params.stopReason = filters.stopReason;
+  }
+  if (filters.minDurationMs != null) {
+    where.push(`duration_ms >= @minDur`);
+    params.minDur = filters.minDurationMs;
+  }
+  if (filters.maxDurationMs != null) {
+    where.push(`duration_ms <= @maxDur`);
+    params.maxDur = filters.maxDurationMs;
+  }
+  if (filters.hasTools) {
+    where.push(`tools_declared > 0`);
+  }
+
+  const limit = Math.max(1, Math.min(filters.limit || 20, 1000));
+  const sql = `
+    SELECT request_id, ts, path, session_key, model, resolved_model,
+           status, duration_ms, stop_reason,
+           total_input_tokens, output_tokens, cache_hit_pct,
+           tools_declared, message_count,
+           first_user_text, last_user_text,
+           summary_path, json_path
+    FROM captures
+    ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
+    ORDER BY ts DESC
+    LIMIT ${limit}
+  `;
+  try {
+    const rows = d.prepare(sql).all(params);
+    return { rows };
+  } catch (e) {
+    return { rows: [], error: e.message };
+  }
+}
+
+export async function getCapture(requestId) {
+  const d = await getDB();
+  if (!d) return null;
+  try {
+    return d.prepare(`SELECT * FROM captures WHERE request_id = ?`).get(requestId) || null;
+  } catch {
+    return null;
+  }
+}
+
+export async function captureStats() {
+  const d = await getDB();
+  if (!d) return { error: loadError?.message || 'index unavailable' };
+  try {
+    const total = d.prepare(`SELECT COUNT(*) as n FROM captures`).get().n;
+    const byStatus = d.prepare(`SELECT status, COUNT(*) as n FROM captures GROUP BY status ORDER BY n DESC`).all();
+    const byModel = d.prepare(`SELECT COALESCE(resolved_model, model) as m, COUNT(*) as n FROM captures GROUP BY m ORDER BY n DESC LIMIT 10`).all();
+    const byStop = d.prepare(`SELECT stop_reason, COUNT(*) as n FROM captures WHERE stop_reason IS NOT NULL GROUP BY stop_reason ORDER BY n DESC`).all();
+    const tokens = d.prepare(`
+      SELECT SUM(input_tokens) as in_t, SUM(output_tokens) as out_t,
+             SUM(cache_read_tokens) as cr_t, SUM(cache_create_tokens) as cc_t,
+             AVG(cache_hit_pct) as avg_hit, AVG(duration_ms) as avg_ms
+      FROM captures WHERE status IS NOT NULL
+    `).get();
+    return { total, byStatus, byModel, byStop, tokens };
+  } catch (e) {
+    return { error: e.message };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Backfill — rebuild index from existing capture files
+// ---------------------------------------------------------------------------
+
+/**
+ * Walk ~/.mobygate/captures/, parse each .json + .summary.txt pair, and
+ * upsert into the index. Used for first-time setup or recovery from a
+ * lost DB. Idempotent — REPLACE on request_id keeps it safe to re-run.
+ *
+ * Returns { scanned, indexed, errors }.
+ */
+export async function rebuildFromFilesystem(onProgress) {
+  const d = await getDB();
+  if (!d) return { error: loadError?.message || 'index unavailable' };
+
+  const result = { scanned: 0, indexed: 0, errors: 0 };
+  if (!existsSync(CAPTURE_DIR)) return result;
+
+  const files = readdirSync(CAPTURE_DIR).filter((f) => f.endsWith('.json'));
+  result.scanned = files.length;
+
+  // Prepare statements once. better-sqlite3 transactions must be synchronous,
+  // so we sidestep the async indexCapture/updateCaptureResponse wrappers and
+  // call the prepared statements directly here.
+  const ins = insertStmt(d);
+  const upd = updateResponseStmt(d);
+
+  const tx = d.transaction((jsonFiles) => {
+    for (const fname of jsonFiles) {
+      try {
+        const jsonPath = join(CAPTURE_DIR, fname);
+        const summaryPath = jsonPath.replace(/\.json$/, '.summary.txt');
+        const stat = statSync(jsonPath);
+        const body = JSON.parse(readFileSync(jsonPath, 'utf8'));
+
+        // Parse the filename for ts + path + requestId.
+        // Format: YYYY-MM-DD_HH-MM-SS_<route-slug>_<requestId>.json
+        const m = fname.match(/^(\d{4}-\d{2}-\d{2})_(\d{2}-\d{2}-\d{2})_(.+?)_([0-9a-f]+)\.json$/);
+        const ts = m
+          ? `${m[1]}T${m[2].replace(/-/g, ':')}.000Z`
+          : stat.mtime.toISOString();
+        const requestId = m ? m[4] : fname.replace(/\.json$/, '');
+        const routePath = m ? '/' + m[3].replace(/-/g, '/') : '';
+
+        // Pull session key + status + usage from the summary file (best-effort).
+        let sessionKey = null;
+        let sessionSource = null;
+        let status = null;
+        let stopReason = null;
+        let durationMs = null;
+        let resolvedModel = null;
+        const usage = {};
+        if (existsSync(summaryPath)) {
+          const txt = readFileSync(summaryPath, 'utf8');
+          sessionKey = matchOne(txt, /session_key:\s+(\S+)/) || null;
+          if (sessionKey === '(none)') sessionKey = null;
+          sessionSource = matchOne(txt, /session_source:\s+(\S+)/) || null;
+          if (sessionSource === '(unknown)') sessionSource = null;
+          status = matchOne(txt, /^status:\s+(\S+)/m) || null;
+          stopReason = matchOne(txt, /^stop_reason:\s+(\S+)/m) || null;
+          durationMs = parseIntOrNull(matchOne(txt, /^duration:\s+(\d+)\s+ms/m));
+          resolvedModel = matchOne(txt, /^model:\s+(\S+)/m) || null;
+          usage.input_tokens = parseIntOrNull(matchOne(txt, /input_tokens \(uncached\):\s+(\d+)/));
+          usage.cache_read_input_tokens = parseIntOrNull(matchOne(txt, /cache_read_input_tokens:\s+(\d+)/));
+          usage.cache_creation_input_tokens = parseIntOrNull(matchOne(txt, /cache_creation_input_tokens:\s+(\d+)/));
+          usage.output_tokens = parseIntOrNull(matchOne(txt, /^\s*output_tokens:\s+(\d+)/m));
+        }
+
+        const fields = extractIndexFields(body || {});
+        ins.run({
+          request_id: requestId,
+          ts,
+          path: routePath,
+          session_key: sessionKey,
+          session_source: sessionSource,
+          json_path: jsonPath,
+          summary_path: existsSync(summaryPath) ? summaryPath : null,
+          ...fields,
+        });
+        if (status || stopReason || durationMs != null) {
+          const totalInput = (usage.input_tokens || 0) + (usage.cache_read_input_tokens || 0) + (usage.cache_creation_input_tokens || 0);
+          const cacheHitPct = totalInput > 0
+            ? +(((usage.cache_read_input_tokens || 0) / totalInput) * 100).toFixed(2)
+            : 0;
+          upd.run({
+            request_id: requestId,
+            status,
+            duration_ms: durationMs,
+            stop_reason: stopReason,
+            resolved_model: resolvedModel,
+            input_tokens: usage.input_tokens ?? 0,
+            cache_read_tokens: usage.cache_read_input_tokens ?? 0,
+            cache_create_tokens: usage.cache_creation_input_tokens ?? 0,
+            output_tokens: usage.output_tokens ?? 0,
+            cache_hit_pct: cacheHitPct,
+          });
+        }
+        result.indexed += 1;
+        if (onProgress && result.indexed % 25 === 0) onProgress(result);
+      } catch (e) {
+        result.errors += 1;
+        if (process.env.MOBY_DEBUG_INDEX) console.warn(`[captures-index] backfill ${fname}: ${e.message}`);
+      }
+    }
+  });
+  tx(files);
+  return result;
+}
+
+function matchOne(text, re) {
+  const m = text.match(re);
+  return m ? m[1] : null;
+}
+function parseIntOrNull(s) {
+  if (s == null) return null;
+  const n = parseInt(s, 10);
+  return Number.isFinite(n) ? n : null;
+}
+
+export function isIndexAvailable() {
+  return loadError == null;
+}
+export function getLoadError() {
+  return loadError;
+}