agentboss 0.1.0

package/server/etl/opencode.js

@@ -0,0 +1,606 @@
+/**
+ * OpenCode ETL collector for Agent Boss
+ *
+ * Reads session, message, part, and tool-call data from the OpenCode SQLite
+ * database (opencode.db — can be 2+ GB) and writes unified rows into boss.db.
+ *
+ * IMPORTANT: The source database is opened with the native `sqlite3` npm
+ * package (async, streaming, read-only) so that the entire file is never
+ * loaded into memory.  The destination boss.db uses sql.js (in-memory) and
+ * is accessed via helpers in server/db/queries.js.
+ *
+ * @author Felix
+ */
+
+const sqlite3 = require('sqlite3');
+const { saveDb } = require('../db/connection');
+const {
+  isJudgePrompt,
+  isInternalAbossTitle,
+  isPlaceholderOpencodeTitle,
+} = require('./judge-filter');
+const {
+  upsertSession,
+  bulkInsertMessages,
+  bulkInsertParts,
+  bulkInsertToolCalls,
+  getEtlState,
+  updateEtlState,
+} = require('../db/queries');
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const SOURCE = 'opencode';
+const BATCH_SIZE = 50;
+const SQLITE_BUSY = 5; // SQLite error code for SQLITE_BUSY
+const MAX_RETRIES = 3;
+const RETRY_DELAY_MS = 1000;
+
+// ---------------------------------------------------------------------------
+// sqlite3 Promise wrappers
+// ---------------------------------------------------------------------------
+
+/**
+ * Open a sqlite3 database in read-only mode and return the instance.
+ * @param {string} dbPath
+ * @returns {Promise<sqlite3.Database>}
+ */
+function openReadOnly(dbPath) {
+  return new Promise((resolve, reject) => {
+    const db = new sqlite3.Database(dbPath, sqlite3.OPEN_READONLY, (err) => {
+      if (err) return reject(err);
+      resolve(db);
+    });
+  });
+}
+
+/**
+ * Close a sqlite3 database.
+ * @param {sqlite3.Database} db
+ * @returns {Promise<void>}
+ */
+function closeDb(db) {
+  return new Promise((resolve, reject) => {
+    db.close((err) => {
+      if (err) return reject(err);
+      resolve();
+    });
+  });
+}
+
+/**
+ * Run db.all() as a Promise.
+ * @param {sqlite3.Database} db
+ * @param {string} sql
+ * @param {Array} params
+ * @returns {Promise<Object[]>}
+ */
+function dbAll(db, sql, params = []) {
+  return new Promise((resolve, reject) => {
+    db.all(sql, params, (err, rows) => {
+      if (err) return reject(err);
+      resolve(rows || []);
+    });
+  });
+}
+
+/**
+ * Run db.get() as a Promise (single row).
+ * @param {sqlite3.Database} db
+ * @param {string} sql
+ * @param {Array} params
+ * @returns {Promise<Object|undefined>}
+ */
+function dbGet(db, sql, params = []) {
+  return new Promise((resolve, reject) => {
+    db.get(sql, params, (err, row) => {
+      if (err) return reject(err);
+      resolve(row);
+    });
+  });
+}
+
+/**
+ * Retry a function on SQLITE_BUSY errors with exponential back-off.
+ * @param {() => Promise<T>} fn
+ * @param {number} retries
+ * @returns {Promise<T>}
+ * @template T
+ */
+async function withRetry(fn, retries = MAX_RETRIES) {
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      const isBusy = err && (err.errno === SQLITE_BUSY || /SQLITE_BUSY/.test(err.message));
+      if (isBusy && attempt < retries) {
+        const delay = RETRY_DELAY_MS * Math.pow(2, attempt);
+        await new Promise((r) => setTimeout(r, delay));
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Date / time helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert epoch milliseconds to ISO 8601 string (UTC).
+ * @param {number|null} ms
+ * @returns {string|null}
+ */
+function msToIso(ms) {
+  if (ms == null || ms <= 0) return null;
+  return new Date(ms).toISOString();
+}
+
+/**
+ * Convert epoch milliseconds to YYYY-MM-DD string (UTC).
+ * @param {number|null} ms
+ * @returns {string|null}
+ */
+function msToDate(ms) {
+  if (ms == null || ms <= 0) return null;
+  return new Date(ms).toISOString().slice(0, 10);
+}
+
+// ---------------------------------------------------------------------------
+// Safe JSON parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a JSON string, returning a default value on failure.
+ * @param {string|null} raw
+ * @param {*} fallback
+ * @returns {*}
+ */
+function safeParse(raw, fallback = null) {
+  if (raw == null || raw === '') return fallback;
+  try {
+    return JSON.parse(raw);
+  } catch (_) {
+    return fallback;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Mapping functions
+// ---------------------------------------------------------------------------
+
+/**
+ * Map an OpenCode session row + aggregate counts/totals to a unified session
+ * object.
+ *
+ * NOTE: The OpenCode `session` table does NOT contain token / cost / model
+ * columns — those live inside each `message.data` JSON blob.  The caller
+ * therefore aggregates them across all messages and passes them in via
+ * `agg`.
+ *
+ * @param {Object} row       Raw session row from opencode.db
+ * @param {number} msgCount  Number of messages in this session
+ * @param {number} errCount  Number of error parts
+ * @param {number} toolCount Number of tool-call parts
+ * @param {Object} agg       Aggregated values from messages: {
+ *   tokens_input, tokens_output, tokens_reasoning,
+ *   tokens_cache_read, tokens_cache_write, cost_usd, model_id
+ * }
+ * @returns {Object}
+ */
+function mapSession(row, msgCount, errCount, toolCount, agg = {}) {
+  return {
+    id: row.id,
+    source: SOURCE,
+    date: msToDate(row.time_created),
+    started_at: msToIso(row.time_created),
+    ended_at: msToIso(row.time_updated),
+    duration_minutes:
+      row.time_updated && row.time_created
+        ? Math.round((row.time_updated - row.time_created) / 60000)
+        : 0,
+    active_minutes: null, // calculated separately by active-time calculator
+    message_count: msgCount,
+    tokens_input: agg.tokens_input || 0,
+    tokens_output: agg.tokens_output || 0,
+    tokens_reasoning: agg.tokens_reasoning || 0,
+    tokens_cache_read: agg.tokens_cache_read || 0,
+    tokens_cache_write: agg.tokens_cache_write || 0,
+    cost_usd: agg.cost_usd || 0,
+    project: row.directory || null,
+    title: row.title || null,
+    model: agg.model_id || null,
+    error_count: errCount,
+    tool_call_count: toolCount,
+    summary_additions: row.summary_additions || 0,
+    summary_deletions: row.summary_deletions || 0,
+    summary_files: row.summary_files || 0,
+    reverted: row.revert != null ? 1 : 0,
+    time_compacting: row.time_compacting
+      ? Math.round(row.time_compacting / 1000)
+      : 0,
+    // Subagent linkage from the source `session` table.  parent_id is
+    // non-null only for sessions spawned by a parent through the `task`
+    // tool; agent describes the flavour (typically 'build' or 'explore').
+    // Used by the UI to filter subagents out of "会话列表" views while
+    // keeping their cost / token totals in aggregate stats.
+    parent_session_id: row.parent_id || null,
+    agent_type:        row.agent || null,
+  };
+}
+
+/**
+ * Map an OpenCode message row to a unified message object.
+ * @param {Object} row  Raw message row from opencode.db
+ * @returns {Object}
+ */
+function mapMessage(row) {
+  const data = safeParse(row.data, {});
+  const tokens = data.tokens || {};
+  const cache = tokens.cache || {};
+
+  return {
+    id: row.id,
+    session_id: row.session_id,
+    source: SOURCE,
+    role: data.role || null,
+    timestamp: msToIso(row.time_created),
+    tokens_input: tokens.input || 0,
+    tokens_output: tokens.output || 0,
+    tokens_reasoning: tokens.reasoning || 0,
+    tokens_cache_read: cache.read || 0,
+    tokens_cache_write: cache.write || 0,
+    cost_usd: data.cost || 0,
+    content_length: 0, // estimated from part text lengths elsewhere
+    is_error: 0,       // derived from parts
+    model_id: (data.modelID || (data.providerID ? `${data.providerID}/${data.modelID || ''}` : null)) || null,
+  };
+}
+
+/**
+ * Map an OpenCode part row to a unified part object.
+ * @param {Object} row  Raw part row from opencode.db
+ * @returns {Object}
+ */
+function mapPart(row) {
+  const data = safeParse(row.data, {});
+
+  return {
+    id: row.id,
+    message_id: row.message_id,
+    session_id: row.session_id,
+    source: SOURCE,
+    type: data.type || null,
+    timestamp: msToIso(row.time_created),
+  };
+}
+
+/**
+ * Extract a target file path from tool-call part data when applicable.
+ * Common tool names that carry file paths: read, write, edit, glob, grep.
+ * @param {Object} data  Parsed part data JSON
+ * @returns {string|null}
+ */
+function extractTargetFile(data) {
+  if (!data || !data.state || !data.state.metadata) return null;
+  const meta = data.state.metadata;
+
+  // Various known metadata shapes
+  if (typeof meta.filePath === 'string') return meta.filePath;
+  if (typeof meta.path === 'string') return meta.path;
+  if (typeof meta.file === 'string') return meta.file;
+
+  return null;
+}
+
+/**
+ * Map an OpenCode part row (where data.type === 'tool') to a unified tool
+ * call object.
+ * @param {Object} row  Raw part row from opencode.db
+ * @param {Object} data Parsed part data JSON
+ * @returns {Object}
+ */
+function mapToolCall(row, data) {
+  const state = data.state || {};
+
+  return {
+    id: row.id,
+    part_id: row.id,
+    session_id: row.session_id,
+    source: SOURCE,
+    tool_name: data.tool || null,
+    timestamp: msToIso(row.time_created),
+    status: state.status || null,
+    error_message: state.error || null,
+    target_file: extractTargetFile(data),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Core ETL logic
+// ---------------------------------------------------------------------------
+
+/**
+ * Process a single session: read its messages and parts from opencode.db,
+ * map them, and write to boss.db.
+ *
+ * @param {sqlite3.Database} srcDb   opencode.db (native sqlite3, read-only)
+ * @param {Object}           bossDb  boss.db (sql.js, in-memory)
+ * @param {Object}           session Raw session row
+ * @returns {Promise<{messages: number, parts: number, toolCalls: number, errors: number}>}
+ */
+async function processSession(srcDb, bossDb, session) {
+  // Short-circuit: sessions created by our own internal `opencode run`
+  // invocations (analysis / polish jobs) carry a stable title prefix.
+  // Filter them out before touching messages/parts.
+  if (isInternalAbossTitle(session.title)) {
+    return { messages: 0, parts: 0, toolCalls: 0, skipped: true };
+  }
+
+  // Short-circuit: OpenCode's placeholder title for sessions that were
+  // started programmatically without any meaningful prompt (typical of
+  // aboss-triggered judge / polish calls, but also of any third-party
+  // `opencode run`).  These add zero analytical value — drop them.
+  if (isPlaceholderOpencodeTitle(session.title)) {
+    return { messages: 0, parts: 0, toolCalls: 0, skipped: true };
+  }
+
+  // --- Messages ---
+  const rawMessages = await withRetry(() =>
+    dbAll(srcDb, 'SELECT * FROM message WHERE session_id = ?', [session.id])
+  );
+  const messages = rawMessages.map(mapMessage);
+
+  // --- Parts ---
+  const rawParts = await withRetry(() =>
+    dbAll(srcDb, 'SELECT * FROM part WHERE session_id = ?', [session.id])
+  );
+  const parts = [];
+  const toolCalls = [];
+  const textByMsg = Object.create(null); // message_id -> concatenated text
+  let errorCount = 0;
+
+  for (const row of rawParts) {
+    parts.push(mapPart(row));
+
+    const data = safeParse(row.data, {});
+
+    // Collect text content for v2 capability-model signal extraction.
+    // OpenCode stores user/assistant text in parts with type='text';
+    // some flavours also use 'reasoning'.  We keep both, capped at 4 KB
+    // per message to avoid bloating boss.db.
+    if (data.type === 'text' || data.type === 'reasoning') {
+      const txt = typeof data.text === 'string' ? data.text : '';
+      if (txt) {
+        const prev = textByMsg[row.message_id] || '';
+        const next = prev ? prev + '\n' + txt : txt;
+        textByMsg[row.message_id] = next.length > 4096 ? next.slice(0, 4096) : next;
+      }
+    }
+
+    if (data.type === 'tool') {
+      toolCalls.push(mapToolCall(row, data));
+
+      const state = data.state || {};
+      if (state.status === 'error' || state.error) {
+        errorCount++;
+      }
+    }
+  }
+
+  // Attach concatenated text to each message
+  for (const m of messages) {
+    m.text = textByMsg[m.id] || null;
+  }
+
+  // Skip sessions created by our own LLM judge — `opencode run` logs every
+  // judge call as a session here; importing them back would create a
+  // feedback loop (each analysis pass spawns more sessions to analyze).
+  if (messages.some((m) => m.role === 'user' && isJudgePrompt(m.text))) {
+    return { messages: 0, parts: 0, toolCalls: 0, skipped: true };
+  }
+
+  // --- Aggregate token / cost / model across messages ---
+  // OpenCode stores these per-message inside data JSON, not on the session
+  // row.  We sum them here so the unified_session row carries totals.
+  const agg = {
+    tokens_input: 0,
+    tokens_output: 0,
+    tokens_reasoning: 0,
+    tokens_cache_read: 0,
+    tokens_cache_write: 0,
+    cost_usd: 0,
+    model_id: null,
+  };
+  for (const m of messages) {
+    agg.tokens_input       += m.tokens_input       || 0;
+    agg.tokens_output      += m.tokens_output      || 0;
+    agg.tokens_reasoning   += m.tokens_reasoning   || 0;
+    agg.tokens_cache_read  += m.tokens_cache_read  || 0;
+    agg.tokens_cache_write += m.tokens_cache_write || 0;
+    agg.cost_usd           += m.cost_usd           || 0;
+    if (!agg.model_id && m.model_id) agg.model_id = m.model_id;
+  }
+
+  // --- Session (needs aggregate counts) ---
+  const sessionObj = mapSession(
+    session,
+    messages.length,
+    errorCount,
+    toolCalls.length,
+    agg
+  );
+
+  // --- Write to boss.db ---
+  upsertSession(bossDb, sessionObj);
+  bulkInsertMessages(bossDb, messages);
+  bulkInsertParts(bossDb, parts);
+  bulkInsertToolCalls(bossDb, toolCalls);
+
+  return {
+    messages: messages.length,
+    parts: parts.length,
+    toolCalls: toolCalls.length,
+    errors: errorCount,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Collect data from the OpenCode database and write to boss.db.
+ *
+ * Performs incremental sync based on etl_state.last_session_time.  Sessions
+ * are processed in batches of {@link BATCH_SIZE} for memory efficiency.
+ * Each session is individually wrapped in try/catch so that one bad row
+ * never stops the whole ETL run.
+ *
+ * @param {Object}  bossDb       sql.js database instance (boss.db)
+ * @param {string}  opencodePath Path to opencode.db
+ * @param {Object}  [options]    Options
+ * @param {(msg: string) => void} [options.onProgress] Progress callback
+ * @returns {Promise<{sessionCount: number, messageCount: number, partCount: number, toolCallCount: number, errorSessionCount: number}>}
+ */
+async function collectOpenCode(bossDb, opencodePath, options = {}) {
+  const log = options.onProgress || (() => {});
+
+  // -- 1. Read ETL watermark --------------------------------------------------
+  const etlState = getEtlState(bossDb, SOURCE);
+  const lastSessionTime = etlState ? (etlState.last_session_time || 0) : 0;
+  log(`ETL watermark: last_session_time = ${lastSessionTime}`);
+
+  // -- 2. Open source database ------------------------------------------------
+  let srcDb;
+  try {
+    srcDb = await withRetry(() => openReadOnly(opencodePath));
+  } catch (err) {
+    log(`Failed to open opencode.db: ${err.message}`);
+    throw err;
+  }
+
+  // Configure for read-heavy workload
+  try {
+    await new Promise((resolve) => srcDb.run('PRAGMA journal_mode = WAL', resolve));
+  } catch (_) {
+    // Best-effort; some builds may not support WAL in read-only mode
+  }
+
+  const totals = {
+    sessionCount: 0,
+    messageCount: 0,
+    partCount: 0,
+    toolCallCount: 0,
+    errorSessionCount: 0,
+    judgeSkippedCount: 0,
+  };
+
+  try {
+    // -- 3. Count sessions to process -----------------------------------------
+    const countRow = await withRetry(() =>
+      dbGet(srcDb, 'SELECT COUNT(*) AS cnt FROM session WHERE time_created > ?', [lastSessionTime])
+    );
+    const totalSessions = countRow ? countRow.cnt : 0;
+    log(`Found ${totalSessions} new session(s) to process`);
+
+    if (totalSessions === 0) {
+      updateEtlState(bossDb, SOURCE, {
+        last_sync_at: new Date().toISOString(),
+        last_session_id: etlState ? etlState.last_session_id : null,
+        last_session_time: lastSessionTime || null,
+        status: 'idle',
+      });
+      return totals;
+    }
+
+    // -- 4. Process in batches ------------------------------------------------
+    let offset = 0;
+    let latestTime = lastSessionTime;
+    let latestSessionId = etlState ? etlState.last_session_id : null;
+
+    while (offset < totalSessions) {
+      const sessions = await withRetry(() =>
+        dbAll(
+          srcDb,
+          'SELECT * FROM session WHERE time_created > ? ORDER BY time_created ASC LIMIT ? OFFSET ?',
+          [lastSessionTime, BATCH_SIZE, offset]
+        )
+      );
+
+      if (sessions.length === 0) break;
+
+      for (const session of sessions) {
+        try {
+          const result = await processSession(srcDb, bossDb, session);
+          if (result.skipped) {
+            totals.judgeSkippedCount++;
+          } else {
+            totals.sessionCount++;
+            totals.messageCount += result.messages;
+            totals.partCount += result.parts;
+            totals.toolCallCount += result.toolCalls;
+          }
+
+          // Track watermark (skipped sessions advance it too, so they are
+          // never re-scanned)
+          if (session.time_created > latestTime) {
+            latestTime = session.time_created;
+            latestSessionId = session.id;
+          }
+        } catch (err) {
+          totals.errorSessionCount++;
+          log(`Error processing session ${session.id}: ${err.message}`);
+        }
+      }
+
+      offset += sessions.length;
+
+      // -- 5. Update watermark after each batch & persist boss.db -------------
+      updateEtlState(bossDb, SOURCE, {
+        last_sync_at: new Date().toISOString(),
+        last_session_id: latestSessionId,
+        last_session_time: latestTime,
+        status: 'running',
+      });
+      saveDb();
+
+      log(`Processed ${Math.min(offset, totalSessions)}/${totalSessions} sessions`);
+    }
+
+    // -- 6. Final watermark update --------------------------------------------
+    updateEtlState(bossDb, SOURCE, {
+      last_sync_at: new Date().toISOString(),
+      last_session_id: latestSessionId,
+      last_session_time: latestTime,
+      status: 'idle',
+    });
+    saveDb();
+
+    log(
+      `ETL complete: ${totals.sessionCount} sessions, ` +
+      `${totals.messageCount} messages, ${totals.partCount} parts, ` +
+      `${totals.toolCallCount} tool calls` +
+      (totals.errorSessionCount ? `, ${totals.errorSessionCount} failed` : '') +
+      (totals.judgeSkippedCount ? `, ${totals.judgeSkippedCount} judge artifact(s) skipped` : '')
+    );
+  } finally {
+    // Always close the source database
+    try {
+      await closeDb(srcDb);
+    } catch (_) {
+      // best-effort
+    }
+  }
+
+  return totals;
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = { collectOpenCode };