agentel 0.2.8 → 0.3.1

package/src/session-store.js

@@ -0,0 +1,405 @@
+"use strict";
+
+// SQLite-backed session metadata index.
+//
+// Why this exists:
+//   listSessions used to walk every *.metadata.json under the archive on each
+//   read (~85ms warm for 3,900 sessions on the dev archive). A JSON
+//   session-list index later replaced the walk, but parsing the 22MB file
+//   still cost ~125ms on every cold list-endpoint hit. SQLite gives us
+//   indexed ORDER BY / WHERE / GROUP BY with sub-millisecond reads and
+//   incremental upserts on import.
+//
+// Source of truth: still the *.metadata.json files on disk. This DB is a
+// derived cache, rebuildable any time. If it goes missing/corrupt the
+// archive.js fallback rebuilds it by walking the filesystem.
+//
+// Schema is a hybrid: hot columns are extracted so SQL can index/sort/group;
+// everything else is stored as an opaque JSON blob in session_json. This
+// avoids maintaining 30+ ALTER-able columns as session metadata evolves.
+
+const fs = require("fs");
+const path = require("path");
+const { ensureDir } = require("./paths");
+
+const SESSION_STORE_VERSION = 2;
+
+let _Database = null;
+let _driverLoaded = false;
+let _driverLoadError = null;
+let _driverBroken = false;
+
+const _connState = {
+  path: "",
+  ino: 0,
+  db: null,
+  prepared: null,
+  broken: false
+};
+
+function loadDriver() {
+  if (_driverLoaded) return _driverBroken ? null : _Database;
+  _driverLoaded = true;
+  try {
+    _Database = require("better-sqlite3");
+  } catch (error) {
+    _Database = null;
+    _driverLoadError = error;
+  }
+  return _Database;
+}
+
+function markDriverBroken() {
+  _driverBroken = true;
+  _Database = null;
+  closeConnection();
+}
+
+function closeConnection() {
+  if (_connState.db) {
+    try { _connState.db.close(); } catch { /* ignore */ }
+  }
+  _connState.db = null;
+  _connState.prepared = null;
+  _connState.path = "";
+  _connState.ino = 0;
+  _connState.broken = false;
+}
+
+/**
+ * Path to the sessions SQLite file inside an archive root. Kept as its own
+ * file so an FTS rebuild (different version, different cadence) does not
+ * disturb the session index.
+ */
+function sessionStorePath(archiveRoot) {
+  return path.join(archiveRoot, "indexes", "sessions", "sessions.sqlite");
+}
+
+const SCHEMA_SQL = [
+  "CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);",
+  "CREATE TABLE IF NOT EXISTS sessions (",
+  "  session_id TEXT PRIMARY KEY,",
+  "  metadata_path TEXT NOT NULL,",
+  "  mtime_ms INTEGER NOT NULL,",
+  "  size INTEGER NOT NULL,",
+  "  started_at TEXT,",
+  "  ended_at TEXT,",
+  "  provider TEXT,",
+  "  source_type TEXT,",
+  "  repo_canonical TEXT,",
+  "  scope_canonical TEXT,",
+  "  storage_scope TEXT,",
+  "  conversation_kind TEXT,",
+  "  is_subagent INTEGER NOT NULL DEFAULT 0,",
+  // source_path is the original importer-side path the session came from
+  // (e.g. a Cursor agent-transcripts folder). Used by the Cursor/Devin
+  // multi-session-store provider flow to supersede prior snapshots written
+  // under the same source.
+  "  source_path TEXT,",
+  "  session_json TEXT NOT NULL",
+  ");",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_started_at_desc ON sessions(started_at DESC);",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_provider ON sessions(provider);",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_repo_canonical ON sessions(repo_canonical);",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_is_subagent ON sessions(is_subagent);",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_metadata_path ON sessions(metadata_path);",
+  "CREATE INDEX IF NOT EXISTS idx_sessions_provider_source_path ON sessions(provider, source_path);"
+].join("\n");
+
+/**
+ * Open or reuse a connection. Returns null if the driver is unavailable
+ * (the caller then falls back to the JSON / filesystem path). A wedged
+ * driver (ABI mismatch surfacing at first `new Database()`) latches so we
+ * do not re-attempt on every call.
+ */
+function openConnection(archiveRoot) {
+  if (_driverBroken) return null;
+  const Driver = loadDriver();
+  if (!Driver) return null;
+  const filePath = sessionStorePath(archiveRoot);
+  let stat = null;
+  try { stat = fs.statSync(filePath); } catch { stat = null; }
+  if (_connState.db && _connState.path === filePath && stat && _connState.ino === stat.ino) {
+    return _connState;
+  }
+  closeConnection();
+  ensureDir(path.dirname(filePath));
+  let db;
+  try {
+    db = new Driver(filePath);
+  } catch (error) {
+    if (isAbiMismatch(error)) {
+      // Wedged native binding — latch so we don't retry this process. The
+      // archive.js fallback path takes over.
+      markDriverBroken();
+      return null;
+    }
+    // Any other failure (truncated file, partial write from an OS crash,
+    // unreadable bytes, schema mismatch SQLite can't parse) means the file
+    // on disk is unusable. Since this DB is a derived cache rebuildable
+    // from *.metadata.json files, unlink so the next caller gets a fresh
+    // db instead of pinning the broken state for the rest of the process.
+    try { fs.unlinkSync(filePath); } catch { /* ignore */ }
+    try { fs.unlinkSync(`${filePath}-wal`); } catch { /* ignore */ }
+    try { fs.unlinkSync(`${filePath}-shm`); } catch { /* ignore */ }
+    return null;
+  }
+  try {
+    db.pragma("journal_mode = WAL");
+    db.pragma("synchronous = NORMAL");
+    db.pragma("temp_store = MEMORY");
+    // Derived cache: bumping SESSION_STORE_VERSION wipes the sessions table
+    // so we can change the schema without writing ALTERs. The on-disk
+    // metadata.json files are the source of truth; the next list call
+    // repopulates from a filesystem walk.
+    let storedVersion = "";
+    try {
+      const row = db.prepare("SELECT value FROM meta WHERE key = 'version'").get();
+      storedVersion = String(row?.value || "");
+    } catch {
+      // meta table may not exist on a fresh db — fine.
+    }
+    if (storedVersion && storedVersion !== String(SESSION_STORE_VERSION)) {
+      try { db.exec("DROP TABLE IF EXISTS sessions"); } catch { /* ignore */ }
+    }
+    db.exec(SCHEMA_SQL);
+    upsertMeta(db, "version", String(SESSION_STORE_VERSION));
+    upsertMeta(db, "archive_root", archiveRoot);
+  } catch (error) {
+    // Schema setup failed — the file is likely partially populated or has
+    // an unreadable on-disk schema header. Unlink so the next caller gets
+    // a clean DB instead of pinning this broken state for the process.
+    try { db.close(); } catch { /* ignore */ }
+    try { fs.unlinkSync(filePath); } catch { /* ignore */ }
+    try { fs.unlinkSync(`${filePath}-wal`); } catch { /* ignore */ }
+    try { fs.unlinkSync(`${filePath}-shm`); } catch { /* ignore */ }
+    return null;
+  }
+  let nextStat = null;
+  try { nextStat = fs.statSync(filePath); } catch { nextStat = null; }
+  _connState.path = filePath;
+  _connState.ino = nextStat?.ino || 0;
+  _connState.db = db;
+  _connState.prepared = new Map();
+  _connState.broken = false;
+  return _connState;
+}
+
+function isAbiMismatch(error) {
+  const message = String(error?.message || "");
+  return /NODE_MODULE_VERSION|was compiled against a different Node\.js version/i.test(message);
+}
+
+function prepareCached(state, sql) {
+  let stmt = state.prepared.get(sql);
+  if (!stmt) {
+    stmt = state.db.prepare(sql);
+    state.prepared.set(sql, stmt);
+  }
+  return stmt;
+}
+
+function upsertMeta(db, key, value) {
+  db.prepare("INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
+}
+
+function entryToRowParams(entry) {
+  const session = entry.session || {};
+  const isSubagent = /_subagent$/.test(String(session.conversationKind || "")) ? 1 : 0;
+  return {
+    session_id: session.sessionId || "",
+    metadata_path: entry.metadataPath || session.metadataPath || "",
+    mtime_ms: Math.trunc(Number(entry.mtimeMs) || 0),
+    size: Math.trunc(Number(entry.size) || 0),
+    started_at: session.startedAt || "",
+    ended_at: session.endedAt || "",
+    provider: session.provider || "",
+    source_type: session.sourceType || "",
+    repo_canonical: session.repoCanonical || "",
+    scope_canonical: session.scopeCanonical || "",
+    storage_scope: session.storageScope || "",
+    conversation_kind: session.conversationKind || "",
+    is_subagent: isSubagent,
+    source_path: session.sourcePath || "",
+    session_json: JSON.stringify(session)
+  };
+}
+
+function rowToEntry(row) {
+  if (!row || !row.session_json) return null;
+  let session = null;
+  try { session = JSON.parse(row.session_json); } catch { return null; }
+  if (!session || typeof session !== "object") return null;
+  return {
+    metadataPath: row.metadata_path || "",
+    mtimeMs: Number(row.mtime_ms) || 0,
+    size: Number(row.size) || 0,
+    session
+  };
+}
+
+const ROW_INSERT_SQL =
+  "INSERT INTO sessions(" +
+  "  session_id, metadata_path, mtime_ms, size, started_at, ended_at, provider, source_type, " +
+  "  repo_canonical, scope_canonical, storage_scope, conversation_kind, is_subagent, source_path, session_json" +
+  ") VALUES(" +
+  "  @session_id, @metadata_path, @mtime_ms, @size, @started_at, @ended_at, @provider, @source_type, " +
+  "  @repo_canonical, @scope_canonical, @storage_scope, @conversation_kind, @is_subagent, @source_path, @session_json" +
+  ") ON CONFLICT(session_id) DO UPDATE SET " +
+  "  metadata_path = excluded.metadata_path, mtime_ms = excluded.mtime_ms, size = excluded.size, " +
+  "  started_at = excluded.started_at, ended_at = excluded.ended_at, provider = excluded.provider, " +
+  "  source_type = excluded.source_type, repo_canonical = excluded.repo_canonical, " +
+  "  scope_canonical = excluded.scope_canonical, storage_scope = excluded.storage_scope, " +
+  "  conversation_kind = excluded.conversation_kind, is_subagent = excluded.is_subagent, " +
+  "  source_path = excluded.source_path, session_json = excluded.session_json";
+
+const ROW_SELECT_ALL_SQL =
+  "SELECT session_id, metadata_path, mtime_ms, size, session_json FROM sessions";
+
+const ROW_DELETE_SQL = "DELETE FROM sessions WHERE session_id = ?";
+const ROW_DELETE_BY_PATH_SQL = "DELETE FROM sessions WHERE metadata_path = ?";
+// Used by the Cursor/Devin provider flow to evict prior snapshots written
+// for the same (provider, source_path) before upserting the fresh one.
+const ROW_DELETE_BY_SOURCE_PATH_SQL =
+  "DELETE FROM sessions WHERE provider = ? AND source_path = ? AND source_path != '' AND session_id != ?";
+
+/**
+ * Bulk-replace the entire sessions table from a rebuilt entry list. Used by
+ * the filesystem-walk fallback after it scans the archive. Done in a single
+ * transaction so concurrent readers either see the old or new set.
+ */
+function replaceAllSessions(entries, archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return false;
+  const insertStmt = prepareCached(state, ROW_INSERT_SQL);
+  try {
+    const replace = state.db.transaction(function (rows) {
+      state.db.exec("DELETE FROM sessions");
+      // Track actually-inserted rows rather than rows.length so the count
+      // meta reflects what's queryable in the sessions table. Entries that
+      // lack session_id or metadata_path are skipped above and must not
+      // count toward the stored total.
+      let inserted = 0;
+      for (const entry of rows) {
+        if (!entry || !entry.session) continue;
+        const params = entryToRowParams(entry);
+        if (!params.session_id || !params.metadata_path) continue;
+        insertStmt.run(params);
+        inserted += 1;
+      }
+      upsertMeta(state.db, "count", String(inserted));
+      upsertMeta(state.db, "updated_at", new Date().toISOString());
+    });
+    replace(entries);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Single-row upsert used from writeSession after a session is committed to
+ * disk. Cheap path: one INSERT ... ON CONFLICT, no walk.
+ */
+function upsertSession(entry, archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return false;
+  if (!entry || !entry.session) return false;
+  const params = entryToRowParams(entry);
+  if (!params.session_id || !params.metadata_path) return false;
+  try {
+    prepareCached(state, ROW_INSERT_SQL).run(params);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Delete by session id or by metadata path. Either is sufficient since both
+ * are indexed; both are accepted because callers may know only one.
+ */
+function deleteSession({ sessionId, metadataPath }, archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return false;
+  try {
+    if (sessionId) prepareCached(state, ROW_DELETE_SQL).run(sessionId);
+    if (metadataPath) prepareCached(state, ROW_DELETE_BY_PATH_SQL).run(metadataPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Cursor/Devin replace-old-snapshot path: evict every prior session row that
+ * shares (provider, source_path) with the incoming session, except the
+ * incoming session itself. Returns the count of deleted rows for observability.
+ */
+function deleteSameSourceSessions({ provider, sourcePath, keepSessionId }, archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return 0;
+  if (!provider || !sourcePath) return 0;
+  try {
+    const result = prepareCached(state, ROW_DELETE_BY_SOURCE_PATH_SQL).run(provider, sourcePath, keepSessionId || "");
+    return Number(result?.changes || 0);
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Read all session entries from the store. Returns null when the store is
+ * unavailable or empty so the caller can fall back to the JSON file / a
+ * filesystem walk and then populate the store.
+ */
+function readAllSessions(archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return null;
+  let rows;
+  try {
+    rows = prepareCached(state, ROW_SELECT_ALL_SQL).all();
+  } catch {
+    return null;
+  }
+  if (!rows.length) return null;
+  const entries = rows.map(rowToEntry).filter(Boolean);
+  return entries.length ? entries : null;
+}
+
+function countSessions(archiveRoot) {
+  const state = openConnection(archiveRoot);
+  if (!state) return null;
+  try {
+    const row = prepareCached(state, "SELECT COUNT(*) AS c FROM sessions").get();
+    return row?.c || 0;
+  } catch {
+    return null;
+  }
+}
+
+function storeAvailable(archiveRoot) {
+  return Boolean(openConnection(archiveRoot));
+}
+
+function resetStateForTests() {
+  closeConnection();
+  _driverBroken = false;
+  _driverLoaded = false;
+  _Database = null;
+  _driverLoadError = null;
+}
+
+module.exports = {
+  SESSION_STORE_VERSION,
+  sessionStorePath,
+  replaceAllSessions,
+  upsertSession,
+  deleteSession,
+  deleteSameSourceSessions,
+  readAllSessions,
+  countSessions,
+  storeAvailable,
+  closeConnection,
+  resetStateForTests
+};