agentel 0.2.8 → 0.3.0

package/src/search.js

@@ -4,13 +4,13 @@ const fs = require("fs");
 const os = require("os");
 const path = require("path");
 const { spawnSync } = require("child_process");
-const { ensureConversationMarkdown, listSessions, readEvents, readTranscript, sessionHistoryTime } = require("./archive");
+const { ensureConversationMarkdown, listSessions, listSessionsSnapshot, readEvents, readTranscript, sessionHistoryTime } = require("./archive");
 const { EVENT_KINDS, renderEventText } = require("./canonical-events");
 const { loadConfig } = require("./config");
 const { paths, ensureDir, readJson, writeJson } = require("./paths");
 const { canonicalRepo } = require("./repo");
 
-const INDEX_VERSION = 3;
+const INDEX_VERSION = 8;
 const INDEX_STALE_CHECK_TTL_MS = 5000;
 const SQLITE_BUILD_BATCH_SIZE = 500;
 const RIPGREP_SEARCH_TIMEOUT_MS = 8000;
@@ -18,6 +18,18 @@ const RIPGREP_BATCH_FILE_COUNT = 200;
 const MARKDOWN_MATCHES_PER_FILE = 3;
 const FTS_SEARCH_BATCH_SIZE = 250;
 const FTS_MAX_SCAN_ROWS = 5000;
+// Most queries find `limit` distinct sessions within the first few hundred
+// ranked rows — broader terms used to hit FTS_MAX_SCAN_ROWS unconditionally,
+// which forced FTS5 to read ~5000 rows from the 469MB index even when the
+// answer was in the first 200. Try a cheap primary scan first and only
+// re-query at the wide cap if dedupe leaves fewer than `limit` sessions.
+const FTS_PRIMARY_SCAN_MULTIPLIER = 100;
+// Tool *output* (Bash stdout, file dumps, test logs) is high-volume and
+// low-signal: the useful part (error strings, the head of a diff, the start
+// of a stack trace) is front-loaded. We still index tool.completed events so
+// those head matches are findable, but cap the indexed/excerpt text so a
+// 50k-line log doesn't add dozens of chunk docs and inflate every BM25 scan.
+const TOOL_COMPLETED_INDEX_MAX_CHARS = 600;
 const _indexCache = {
   path: "",
   mtimeMs: 0,
@@ -46,14 +58,29 @@ const _ftsReadConn = {
 let _betterSqlite3 = null;
 let _betterSqlite3Loaded = false;
 let _betterSqlite3LoadError = null;
+// Latched true once `new Database()` throws at runtime (e.g. ABI mismatch
+// after Node version change). Once latched, loadBetterSqlite3 returns null
+// permanently so callers take the spawn fallback instead of repeatedly
+// crashing in the constructor.
+let _betterSqlite3Broken = false;
 
 /**
  * Lazily load better-sqlite3. Returns the constructor or null when the native
  * binding is unavailable; call sites then fall back to the legacy `sqlite3`
  * subprocess path. The load attempt is cached so missing optional builds do
  * not re-try `require()` on every query.
+ *
+ * Two failure modes are handled here:
+ *   1. `require("better-sqlite3")` itself throws (no native binding shipped).
+ *   2. The require succeeds but `new Database()` later throws because the
+ *      native .node file was built against a different Node ABI (e.g. nvm
+ *      flipped versions). The lower-level openFts*Db helpers call
+ *      `markBetterSqlite3Broken()` when that happens, after which we
+ *      permanently return null here so callers consistently take the spawn
+ *      fallback instead of repeatedly hitting the same constructor crash.
  */
 function loadBetterSqlite3() {
+  if (_betterSqlite3Broken) return null;
   if (_betterSqlite3Loaded) return _betterSqlite3;
   _betterSqlite3Loaded = true;
   try {
@@ -69,19 +96,31 @@ function betterSqlite3LoadError() {
   return _betterSqlite3LoadError;
 }
 
+function markBetterSqlite3Broken(error) {
+  _betterSqlite3Broken = true;
+  if (error && !_betterSqlite3LoadError) _betterSqlite3LoadError = error;
+}
+
+function resetBetterSqlite3State() {
+  // Test-only: clear the load+broken latches so a test can simulate fresh
+  // module load with different `require` behavior. Not part of any public
+  // API contract.
+  _betterSqlite3 = null;
+  _betterSqlite3Loaded = false;
+  _betterSqlite3LoadError = null;
+  _betterSqlite3Broken = false;
+}
+
 function buildIndex(env = process.env) {
-  const sessions = listSessions(env);
+  const snapshot = listSessionsSnapshot(env);
+  const sessions = snapshot.sessions;
   const docs = [];
   const postings = Object.create(null);
   const df = Object.create(null);
   let totalLength = 0;
 
   for (const session of sessions) {
-    if (session.conversationPath && !fs.existsSync(session.conversationPath)) ensureConversationMarkdown(session, env);
-    const events = readEvents(session);
-    const eventDocs = events.length ? docsForEvents(session, events) : [];
-    if (!eventDocs.length) ensureConversationMarkdown(session, env);
-    const sourceDocs = eventDocs.length ? eventDocs : docsForTranscript(session, readTranscript(session.transcriptPath));
+    const sourceDocs = canonicalDocsForSession(session);
     for (const sourceDoc of sourceDocs) {
       const indexText = normalizeIndexText(sourceDoc.text);
       if (!indexText) continue;
@@ -111,6 +150,7 @@ function buildIndex(env = process.env) {
   const index = {
     version: INDEX_VERSION,
     builtAt: new Date().toISOString(),
+    sessionFingerprint: snapshot.fingerprint,
     docCount: docs.length,
     avgDocLength: docs.length ? totalLength / docs.length : 0,
     df,
@@ -139,6 +179,7 @@ function summarizeIndex(index) {
   return {
     version: index.version,
     builtAt: index.builtAt,
+    sessionFingerprint: index.sessionFingerprint,
     docCount: index.docCount,
     avgDocLength: index.avgDocLength,
     summaryOnly: true
@@ -155,6 +196,7 @@ function readIndexSummary(indexPath) {
     const summary = {
       version: readJsonHeaderNumber(header, "version"),
       builtAt: readJsonHeaderString(header, "builtAt"),
+      sessionFingerprint: readJsonHeaderString(header, "sessionFingerprint"),
       docCount: readJsonHeaderNumber(header, "docCount"),
       avgDocLength: readJsonHeaderNumber(header, "avgDocLength"),
       summaryOnly: true
@@ -190,9 +232,12 @@ const FTS_SCHEMA_SQL = [
   "  session_id TEXT,",
   "  provider TEXT,",
   "  source_type TEXT,",
+  "  conversation_kind TEXT,",
+  "  is_subagent INTEGER,",
   "  repo_canonical TEXT,",
   "  repo_display TEXT,",
   "  scope_canonical TEXT,",
+  "  storage_scope TEXT,",
   "  cwd TEXT,",
   "  title TEXT,",
   "  started_at TEXT,",
@@ -229,6 +274,7 @@ function buildFtsIndex(index, env = process.env) {
     insertFtsMetaRows(handle, [
       { key: "version", value: String(INDEX_VERSION) },
       { key: "builtAt", value: index.builtAt || "" },
+      { key: "sessionFingerprint", value: index.sessionFingerprint || "" },
       { key: "docCount", value: String(index.docCount || 0) }
     ]);
     insertFtsDocs(handle, tmpPath, index.docs || [], 1);
@@ -252,6 +298,7 @@ function buildFtsIndex(index, env = process.env) {
 }
 
 function buildFtsIndexSummary(env = process.env) {
+  const snapshot = listSessionsSnapshot(env);
   const ftsPath = paths(env).ftsIndex;
   const tmpPath = `${ftsPath}.${process.pid}.tmp`;
   const builtAt = new Date().toISOString();
@@ -265,16 +312,13 @@ function buildFtsIndexSummary(env = process.env) {
     execFtsBuildSql(handle, FTS_SCHEMA_SQL);
     insertFtsMetaRows(handle, [
       { key: "version", value: String(INDEX_VERSION) },
-      { key: "builtAt", value: builtAt }
+      { key: "builtAt", value: builtAt },
+      { key: "sessionFingerprint", value: snapshot.fingerprint || "" }
     ]);
 
     let batch = [];
-    for (const session of listSessions(env)) {
-      if (session.conversationPath && !fs.existsSync(session.conversationPath)) ensureConversationMarkdown(session, env);
-      const events = readEvents(session);
-      const eventDocs = events.length ? docsForEvents(session, events) : [];
-      if (!eventDocs.length) ensureConversationMarkdown(session, env);
-      const sourceDocs = eventDocs.length ? eventDocs : docsForTranscript(session, readTranscript(session.transcriptPath));
+    for (const session of snapshot.sessions) {
+      const sourceDocs = canonicalDocsForSession(session);
       for (const sourceDoc of sourceDocs) {
         const indexText = normalizeIndexText(sourceDoc.text);
         if (!indexText) continue;
@@ -319,6 +363,7 @@ function buildFtsIndexSummary(env = process.env) {
   return {
     version: INDEX_VERSION,
     builtAt,
+    sessionFingerprint: snapshot.fingerprint,
     docCount,
     avgDocLength: docCount ? totalLength / docCount : 0,
     summaryOnly: true
@@ -326,8 +371,8 @@ function buildFtsIndexSummary(env = process.env) {
 }
 
 const FTS_DOCS_INSERT_SQL =
-  "INSERT INTO docs(rowid, doc_id, session_id, provider, source_type, repo_canonical, repo_display, scope_canonical, cwd, title, started_at, occurred_at, role, event_id, event_kind, message_index, path, matched_text) " +
-  "VALUES (@rowid, @doc_id, @session_id, @provider, @source_type, @repo_canonical, @repo_display, @scope_canonical, @cwd, @title, @started_at, @occurred_at, @role, @event_id, @event_kind, @message_index, @path, @matched_text)";
+  "INSERT INTO docs(rowid, doc_id, session_id, provider, source_type, conversation_kind, is_subagent, repo_canonical, repo_display, scope_canonical, storage_scope, cwd, title, started_at, occurred_at, role, event_id, event_kind, message_index, path, matched_text) " +
+  "VALUES (@rowid, @doc_id, @session_id, @provider, @source_type, @conversation_kind, @is_subagent, @repo_canonical, @repo_display, @scope_canonical, @storage_scope, @cwd, @title, @started_at, @occurred_at, @role, @event_id, @event_kind, @message_index, @path, @matched_text)";
 const FTS_FTS_INSERT_SQL = "INSERT INTO docs_fts(rowid, text) VALUES (@rowid, @text)";
 
 function docInsertParams(doc, rowid) {
@@ -337,9 +382,16 @@ function docInsertParams(doc, rowid) {
     session_id: doc.sessionId || "",
     provider: doc.provider || "",
     source_type: doc.sourceType || "",
+    conversation_kind: doc.conversationKind || "",
+    // Precompute the subagent flag once at index time using the same JS
+    // predicate the rest of the codebase uses. Querying an integer column is
+    // ~50x faster than `NOT GLOB '*_subagent'`, which scans every row's
+    // string char-by-char because of the leading wildcard.
+    is_subagent: isSubagentSession(doc) ? 1 : 0,
     repo_canonical: doc.repoCanonical || "",
     repo_display: doc.repoDisplay || "",
     scope_canonical: doc.scopeCanonical || "",
+    storage_scope: doc.storageScope || "",
     cwd: doc.cwd || "",
     title: doc.title || "",
     started_at: doc.startedAt || "",
@@ -414,16 +466,19 @@ function insertFtsDocs(handleOrPath, dbPathHint, docs, rowidStart = 1) {
         const doc = batch[offset];
         const sqlString = (value) => `'${String(value == null ? "" : value).replace(/'/g, "''")}'`;
         statements.push(
-          "INSERT INTO docs(rowid, doc_id, session_id, provider, source_type, repo_canonical, repo_display, scope_canonical, cwd, title, started_at, occurred_at, role, event_id, event_kind, message_index, path, matched_text) VALUES (" +
+          "INSERT INTO docs(rowid, doc_id, session_id, provider, source_type, conversation_kind, is_subagent, repo_canonical, repo_display, scope_canonical, storage_scope, cwd, title, started_at, occurred_at, role, event_id, event_kind, message_index, path, matched_text) VALUES (" +
             [
               rowid,
               sqlString(doc.id || ""),
               sqlString(doc.sessionId || ""),
               sqlString(doc.provider || ""),
               sqlString(doc.sourceType || ""),
+              sqlString(doc.conversationKind || ""),
+              isSubagentSession(doc) ? 1 : 0,
               sqlString(doc.repoCanonical || ""),
               sqlString(doc.repoDisplay || ""),
               sqlString(doc.scopeCanonical || ""),
+              sqlString(doc.storageScope || ""),
               sqlString(doc.cwd || ""),
               sqlString(doc.title || ""),
               sqlString(doc.startedAt || ""),
@@ -455,12 +510,22 @@ function insertFtsDocs(handleOrPath, dbPathHint, docs, rowidStart = 1) {
 function openFtsBuildDb(dbPath) {
   const Database = loadBetterSqlite3();
   if (Database) {
-    const db = new Database(dbPath);
-    db.pragma("journal_mode = OFF");
-    db.pragma("synchronous = OFF");
-    db.pragma("temp_store = MEMORY");
-    db.pragma("locking_mode = EXCLUSIVE");
-    return { kind: "native", db };
+    // `new Database()` can throw on ABI mismatch (different Node version
+    // than the one better-sqlite3 was compiled for). Without this guard the
+    // exception propagates up to buildFtsIndex's catch block, which then
+    // deletes the FTS index file — hard-breaking search until next reindex.
+    // Falling back to the spawn path preserves search instead.
+    try {
+      const db = new Database(dbPath);
+      db.pragma("journal_mode = OFF");
+      db.pragma("synchronous = OFF");
+      db.pragma("temp_store = MEMORY");
+      db.pragma("locking_mode = EXCLUSIVE");
+      return { kind: "native", db };
+    } catch (error) {
+      markBetterSqlite3Broken(error);
+      // Fall through to spawn handle.
+    }
   }
   return { kind: "spawn", dbPath };
 }
@@ -502,7 +567,11 @@ function openFtsReadDb(ftsPath, stat) {
   try {
     db = new Database(ftsPath, { readonly: true, fileMustExist: true });
     db.pragma("query_only = ON");
-  } catch {
+  } catch (error) {
+    // ABI-mismatch crashes look like ordinary throws here. Latch the module
+    // as broken so other readers/builders take the spawn path immediately
+    // instead of repeatedly hitting the same crash.
+    markBetterSqlite3Broken(error);
     return null;
   }
   _ftsReadConn.path = ftsPath;
@@ -586,6 +655,10 @@ function inlineSqlParams(query, params) {
   });
 }
 
+function escapeSqlLike(value) {
+  return String(value).replace(/[\\%_]/g, (match) => `\\${match}`);
+}
+
 function rememberFtsCache(ftsPath, available) {
   let stat = null;
   try {
@@ -605,14 +678,14 @@ function readFtsMeta(ftsPath, stat) {
   const conn = openFtsReadDb(ftsPath, stat);
   if (conn) {
     try {
-      const stmt = prepareFtsStatement(conn, "SELECT key, value FROM meta WHERE key IN ('version', 'docCount')");
+      const stmt = prepareFtsStatement(conn, "SELECT key, value FROM meta WHERE key IN ('version', 'docCount', 'sessionFingerprint')");
       return stmt.all();
     } catch {
       closeFtsReadDb();
       return null;
     }
   }
-  return legacySqliteJson(ftsPath, "SELECT key, value FROM meta WHERE key IN ('version', 'docCount');");
+  return legacySqliteJson(ftsPath, "SELECT key, value FROM meta WHERE key IN ('version', 'docCount', 'sessionFingerprint');");
 }
 
 function ftsIndexAvailable(env = process.env, options = {}) {
@@ -651,13 +724,20 @@ function docsForEvents(session, events) {
   const indexedKinds = new Set([
     EVENT_KINDS.PROMPT_SUBMITTED,
     EVENT_KINDS.RESPONSE_GENERATED,
-    EVENT_KINDS.TOOL_CALLED
+    EVENT_KINDS.TOOL_CALLED,
+    EVENT_KINDS.TOOL_COMPLETED
   ]);
   return events
     .filter((event) => indexedKinds.has(event.kind))
     .map((event) => {
-      const renderedText = renderEventText(event);
+      let renderedText = renderEventText(event);
       if (!String(renderedText || "").trim()) return null;
+      // Cap tool output to its head only. The first ~600 chars hold the
+      // error message / first lines of a diff / start of a stack trace;
+      // the tail is verbose machine output with near-zero recall value.
+      if (event.kind === EVENT_KINDS.TOOL_COMPLETED && renderedText.length > TOOL_COMPLETED_INDEX_MAX_CHARS) {
+        renderedText = `${renderedText.slice(0, TOOL_COMPLETED_INDEX_MAX_CHARS)} [truncated]`;
+      }
       return {
         id: event.eventId,
         eventId: event.eventId,
@@ -666,9 +746,11 @@ function docsForEvents(session, events) {
         sessionId: session.sessionId,
         provider: event.provider || session.provider,
         sourceType: event.sourceType || session.sourceType || "",
+        conversationKind: session.conversationKind || "",
         repoCanonical: event.repoCanonical || session.repoCanonical || "",
         repoDisplay: displayRepoLabel(session),
         scopeCanonical: event.scopeCanonical || session.scopeCanonical || "",
+        storageScope: session.storageScope || "",
         cwd: session.cwd || "",
         title: session.title || event.indexed?.title || "",
         eventTitle: event.indexed?.title || "",
@@ -684,6 +766,11 @@ function docsForEvents(session, events) {
     .filter(Boolean);
 }
 
+function canonicalDocsForSession(session) {
+  const events = readEvents(session);
+  return events.length ? docsForEvents(session, events) : [];
+}
+
 function docsForTranscript(session, messages) {
   const docs = [];
   for (const message of messages) {
@@ -692,9 +779,11 @@ function docsForTranscript(session, messages) {
       sessionId: session.sessionId,
       provider: session.provider,
       sourceType: session.sourceType || "",
+      conversationKind: session.conversationKind || "",
       repoCanonical: session.repoCanonical || "",
       repoDisplay: displayRepoLabel(session),
       scopeCanonical: session.scopeCanonical || "",
+      storageScope: session.storageScope || "",
       cwd: session.cwd || "",
       title: session.title || "",
       startedAt: session.startedAt,
@@ -758,11 +847,12 @@ function rememberIndexCache(indexPath, index) {
 }
 
 function searchPastSessions(query, options = {}, env = process.env) {
+  const allowMarkdownFallback = Boolean(options.markdownFallback || options.legacyMarkdownFallback || options.allowMarkdownFallback);
   try {
     const eventResults = searchIndexedSessions(query, options, env);
-    if (eventResults.length || options.skipMarkdownFallback) return eventResults;
+    if (eventResults.length || !allowMarkdownFallback || options.skipMarkdownFallback) return eventResults;
   } catch {
-    if (options.skipMarkdownFallback) return [];
+    if (!allowMarkdownFallback || options.skipMarkdownFallback) return [];
     // Fall through to the legacy markdown path below.
   }
   return searchMarkdownSessions(query, options, env);
@@ -772,6 +862,7 @@ function searchMarkdownSessions(query, options = {}, env = process.env) {
   const limit = Math.max(1, Math.min(Number(options.limit || 10), 50));
   const maxMatches = Math.max(limit * 8, 40);
   const includeWebChats = Boolean(options.includeWebChats);
+  const includeSubagents = Boolean(options.includeSubagents || options.include_subagents);
   const filter = normalizeSessionFilter(options);
   const repo = filter.repo || inferCallingRepo(options.cwd || process.cwd());
   const since = parseSinceFilter(options.since);
@@ -780,7 +871,7 @@ function searchMarkdownSessions(query, options = {}, env = process.env) {
   if (!queryTokens.length && !phrase) return [];
 
   const sessions = listSessions(env).filter((session) => {
-    if (!matchesSessionFilter(session, { ...filter, includeWebChats, since })) return false;
+    if (!matchesSessionFilter(session, { ...filter, includeWebChats, includeSubagents, since })) return false;
     const conversationPath = ensureConversationMarkdown(session, env);
     const searchPath = conversationPath || session.transcriptPath;
     session._searchPath = searchPath;
@@ -838,22 +929,23 @@ function searchMarkdownSessions(query, options = {}, env = process.env) {
 function searchIndexedSessions(query, options = {}, env = process.env) {
   const limit = Math.max(1, Math.min(Number(options.limit || 10), 50));
   const includeWebChats = Boolean(options.includeWebChats);
+  const includeSubagents = Boolean(options.includeSubagents || options.include_subagents);
   const filter = normalizeSessionFilter(options);
   const repo = filter.repo || inferCallingRepo(options.cwd || process.cwd());
   const since = parseSinceFilter(options.since);
   const queryTokens = tokenize(query);
   const phrase = String(query || "").trim().toLowerCase();
   if (!queryTokens.length && !phrase) return [];
-  const ftsResults = searchFtsSessions(query, queryTokens, { limit, includeWebChats, filter, repo, since, options }, env);
+  const ftsResults = searchFtsSessions(query, queryTokens, { limit, includeWebChats, includeSubagents, filter, repo, since, options }, env);
   if (ftsResults) return ftsResults;
-  if (options.skipJsonIndex) return [];
+  if (options.skipJsonIndex || !(options.legacyJsonIndex || options.allowJsonIndex)) return [];
   const index = loadIndex(env, { noRebuild: Boolean(options.noRebuild) });
   if (!index) return [];
 
   const scored = [];
   const candidates = candidateDocsForQuery(index, queryTokens, phrase);
   for (const { doc, docIndex } of candidates) {
-    if (!matchesSessionFilter(doc, { ...filter, includeWebChats, since })) continue;
+    if (!matchesSessionFilter(doc, { ...filter, includeWebChats, includeSubagents, since })) continue;
 
     let score = bm25Score(doc, queryTokens, index, docIndex, candidates.termFrequencies);
     if (phrase && doc.text.toLowerCase().includes(phrase)) score += 2.5;
@@ -887,32 +979,44 @@ function searchIndexedSessions(query, options = {}, env = process.env) {
   }));
 }
 
-function searchFtsSessions(query, queryTokens, context, env = process.env) {
-  const ftsPath = paths(env).ftsIndex;
-  if (!ftsIndexAvailable(env, { noStaleCheck: Boolean(context.options.noRebuild || context.options.allowStaleFts) })) return null;
-  const matchQuery = ftsMatchQuery(query);
-  if (!matchQuery) return [];
-  // Pull a single bounded prefix in rank order rather than paging with growing
-  // OFFSET. FTS5 with bm25 ORDER BY recomputes the entire ranked set per
-  // OFFSET query, which is quadratic for broad terms (e.g. "react*" matching
-  // 1700+ docs across 7 sessions). One query + JS dedupe is O(N) and stays
-  // under FTS_MAX_SCAN_ROWS to bound memory for catastrophically broad terms.
-  const scanLimit = Math.max(
-    Math.max(context.limit * 20, FTS_SEARCH_BATCH_SIZE),
-    Math.min(FTS_MAX_SCAN_ROWS, context.limit * 500)
-  );
-  const rows = ftsSearchRows(ftsPath, matchQuery, { limit: scanLimit, offset: 0, context });
-  if (!rows) return null;
+function dedupeFtsRowsBySession(rows, context) {
   const bySession = new Map();
   for (const row of rows) {
     const doc = ftsRowToDoc(row);
-    if (!matchesSessionFilter(doc, { ...context.filter, includeWebChats: context.includeWebChats, since: context.since })) continue;
+    if (!matchesSessionFilter(doc, { ...context.filter, includeWebChats: context.includeWebChats, includeSubagents: context.includeSubagents, since: context.since })) continue;
     if (!context.options.repo && context.repo && doc.repoCanonical === context.repo) {
       row.rank = Number(row.rank || 0) - 0.05;
     }
     if (!bySession.has(doc.sessionId)) bySession.set(doc.sessionId, { doc, row });
     if (bySession.size >= context.limit) break;
   }
+  return bySession;
+}
+
+function searchFtsSessions(query, queryTokens, context, env = process.env) {
+  const ftsPath = paths(env).ftsIndex;
+  if (!ftsIndexAvailable(env, { noStaleCheck: Boolean(context.options.noRebuild || context.options.allowStaleFts) })) return null;
+  const matchQuery = ftsMatchQuery(query);
+  if (!matchQuery) return [];
+  // Single bounded prefix in rank order — FTS5 with bm25 + OFFSET is quadratic,
+  // so we pull one shot and dedupe in JS. Two-tier scan: primary is small and
+  // fast (limit*100, typically 1000), wide cap is only used if dedupe couldn't
+  // fill `limit` distinct sessions from the primary.
+  const primaryScanLimit = Math.max(context.limit * FTS_PRIMARY_SCAN_MULTIPLIER, FTS_SEARCH_BATCH_SIZE);
+  const wideScanLimit = Math.max(primaryScanLimit, Math.min(FTS_MAX_SCAN_ROWS, context.limit * 500));
+
+  let rows = ftsSearchRows(ftsPath, matchQuery, { limit: primaryScanLimit, offset: 0, context });
+  if (!rows) return null;
+  let bySession = dedupeFtsRowsBySession(rows, context);
+  if (bySession.size < context.limit && primaryScanLimit < wideScanLimit) {
+    // Under-filled (e.g. broad term where many top-ranked docs collapse onto
+    // the same session, or filters reject most of the primary set). Re-query
+    // at the wide cap. SQLite re-evaluates the MATCH but the surrounding
+    // prepared statement + OS page cache make the second pass cheap.
+    rows = ftsSearchRows(ftsPath, matchQuery, { limit: wideScanLimit, offset: 0, context });
+    if (!rows) return null;
+    bySession = dedupeFtsRowsBySession(rows, context);
+  }
   return [...bySession.values()].slice(0, context.limit).map(({ doc, row }) => ({
     session_id: doc.sessionId,
     provider: doc.provider,
@@ -947,11 +1051,31 @@ function ftsSearchRows(ftsPath, matchQuery, options) {
     clauses.push(`d.source_type IN (${sourceTypes.map(() => "?").join(", ")})`);
     for (const value of sourceTypes) params.push(value);
   }
+  if (!options.context?.includeSubagents) {
+    // Precomputed integer flag (set in docInsertParams). Equivalent to the
+    // old `NOT GLOB '*_subagent'` but a 1-byte compare instead of a
+    // leading-wildcard string scan over every FTS-matched row.
+    clauses.push("(d.is_subagent IS NULL OR d.is_subagent = 0)");
+  }
   if (options.context?.since) {
     const since = options.context.since.toISOString();
     clauses.push("(d.started_at >= ? OR (d.started_at = '' AND d.occurred_at >= ?))");
     params.push(since, since);
   }
+  if (filter.repo) {
+    const repo = String(filter.repo).toLowerCase();
+    const repoLike = `%${escapeSqlLike(repo)}%`;
+    const repoColumns = ["d.repo_canonical", "d.repo_display", "d.scope_canonical", "d.cwd"];
+    clauses.push(
+      "(" +
+        repoColumns.map((column) => `(LOWER(${column}) = ? OR LOWER(${column}) LIKE ? ESCAPE '\\')`).join(" OR ") +
+        ")"
+    );
+    for (const column of repoColumns) {
+      void column;
+      params.push(repo, repoLike);
+    }
+  }
   const limit = Math.max(1, Number(options.limit) || FTS_SEARCH_BATCH_SIZE);
   const offset = Math.max(0, Number(options.offset) || 0);
   // LIMIT/OFFSET stay parameterized so a single prepared-statement shape
@@ -959,8 +1083,8 @@ function ftsSearchRows(ftsPath, matchQuery, options) {
   // per call but the resulting prepared statement is cached on the connection.
   const sql = [
     "SELECT",
-    "  d.doc_id, d.session_id, d.provider, d.source_type, d.repo_canonical, d.repo_display,",
-    "  d.scope_canonical, d.cwd, d.title, d.started_at, d.occurred_at, d.role,",
+    "  d.doc_id, d.session_id, d.provider, d.source_type, d.conversation_kind, d.is_subagent, d.repo_canonical, d.repo_display,",
+    "  d.scope_canonical, d.storage_scope, d.cwd, d.title, d.started_at, d.occurred_at, d.role,",
     "  d.event_id, d.event_kind, d.message_index, d.path, d.matched_text,",
     "  snippet(docs_fts, 0, '', '', '...', 32) AS excerpt,",
     "  bm25(docs_fts) AS rank",
@@ -992,9 +1116,12 @@ function ftsRowToDoc(row) {
     sessionId: row.session_id || "",
     provider: row.provider || "",
     sourceType: row.source_type || "",
+    conversationKind: row.conversation_kind || "",
+    isSubagent: row.is_subagent == null ? undefined : Number(row.is_subagent),
     repoCanonical: row.repo_canonical || "",
     repoDisplay: row.repo_display || "",
     scopeCanonical: row.scope_canonical || "",
+    storageScope: row.storage_scope || "",
     cwd: row.cwd || "",
     title: row.title || "",
     startedAt: row.started_at || "",
@@ -1220,11 +1347,16 @@ function inferCallingRepo(cwd) {
 }
 
 function listHistorySessions(options = {}, env = process.env) {
+  return listHistorySessionsFromSessions(listSessions(env), options);
+}
+
+function listHistorySessionsFromSessions(sessions, options = {}) {
   const since = parseSinceFilter(options.since);
   const includeWebChats = options.includeWebChats !== false;
+  const includeSubagents = Boolean(options.includeSubagents || options.include_subagents);
   const filter = normalizeSessionFilter(options);
-  return listSessions(env)
-    .filter((session) => matchesSessionFilter(session, { ...filter, includeWebChats, since }))
+  return (Array.isArray(sessions) ? sessions : [])
+    .filter((session) => matchesSessionFilter(session, { ...filter, includeWebChats, includeSubagents, since }))
     .map(historySessionSummary)
     .sort((a, b) => String(historySessionSortTime(b)).localeCompare(String(historySessionSortTime(a))));
 }
@@ -1245,6 +1377,7 @@ function historySessionSummary(session) {
     chat_display_name: session.chatDisplayName || undefined,
     chat_project_path: session.chatProjectPath || undefined,
     conversation_kind: session.conversationKind || undefined,
+    parent_composer_id: session.parentComposerId || undefined,
     pinned: Boolean(session.pinned) || undefined,
     cwd: session.cwd || undefined,
     title: session.title || undefined,
@@ -1257,9 +1390,25 @@ function historySessionSummary(session) {
     usage: session.usage || undefined,
     estimatedUsage: session.estimatedUsage || undefined,
     models: session.models || undefined,
+    toolUsage: session.toolUsage || undefined,
+    outputTokenWork: session.outputTokenWork || undefined,
+    outcomes: session.outcomes || undefined,
     cursorCommandTypeCounts: session.cursorCommandTypeCounts || undefined,
+    codexTurnContext: session.sessionSummary?.codexTurnContext || undefined,
+    sidecarEffort: session.sessionSummary?.claudeCodeSidecar?.effort || undefined,
+    claudeTasks: session.sessionSummary?.claudeTasks
+      ? { count: session.sessionSummary.claudeTasks.count, completed: session.sessionSummary.claudeTasks.completed }
+      : undefined,
+    cursorFileHistory: session.sessionSummary?.cursorFileHistory
+      ? { snapshots: session.sessionSummary.cursorFileHistory.snapshots, bytes: session.sessionSummary.cursorFileHistory.bytes }
+      : undefined,
+    artifactCount: Array.isArray(session.artifacts) && session.artifacts.length ? session.artifacts.length : undefined,
+    artifactBytes: Array.isArray(session.artifacts) && session.artifacts.length
+      ? session.artifacts.reduce((sum, item) => sum + (Number(item?.bytes) || 0), 0)
+      : undefined,
     conversation: session.conversationPath,
-    transcript: session.transcriptPath
+    transcript: session.transcriptPath,
+    events: session.eventPath
   };
 }
 
@@ -1282,6 +1431,7 @@ function displayRepoLabel(session) {
 function displayScopeLabel(scope) {
   return {
     "claude-desktop/uncategorized": "Claude Desktop (uncategorized)",
+    "claude-cowork/uncategorized": "Claude Cowork (uncategorized)",
     "claude-code-desktop/uncategorized": "Claude Code Desktop (uncategorized)"
   }[scope] || scope;
 }
@@ -1317,6 +1467,10 @@ function listRecentSessions(limit = 20, options = {}, env = process.env) {
   return listHistorySessions(options, env).slice(0, limit);
 }
 
+function listRecentSessionsFromSessions(sessions, limit = 20, options = {}) {
+  return listHistorySessionsFromSessions(sessions, options).slice(0, limit);
+}
+
 function normalizeSessionFilter(options = {}) {
   const providerInput = options.provider || options.source || "";
   const normalizedProvider = normalizeProviderFilter(providerInput);
@@ -1337,6 +1491,8 @@ function normalizeProviderFilter(value) {
     claude_cli: { provider: "claude_code" },
     claude_desktop: { provider: "claude_desktop" },
     claude_code_desktop: { provider: "claude_desktop", sourceType: "claude-code-desktop-metadata", sourceTypes: ["claude-code-desktop-metadata"] },
+    claude_cowork: { provider: "claude_desktop", sourceType: "claude-workspace-desktop", sourceTypes: ["claude-workspace-desktop"] },
+    claude_cowork_desktop: { provider: "claude_desktop", sourceType: "claude-workspace-desktop", sourceTypes: ["claude-workspace-desktop"] },
     claude_workspace: { provider: "claude_desktop", sourceType: "claude-workspace-desktop", sourceTypes: ["claude-workspace-desktop"] },
     claude_workspace_desktop: { provider: "claude_desktop", sourceType: "claude-workspace-desktop", sourceTypes: ["claude-workspace-desktop"] },
     claude_sdk: { provider: "claude_sdk" },
@@ -1353,10 +1509,21 @@ function normalizeProviderFilter(value) {
     aider: { provider: "aider", sourceType: "aider-chat-history", sourceTypes: ["aider-chat-history"] },
     devin: { provider: "devin" },
     devin_cli: { provider: "devin", sourceType: "devin-cli-history", sourceTypes: ["devin-cli-history"] },
+    devin_desktop: { provider: "devin", sourceType: "devin-desktop-acp-events", sourceTypes: ["devin-desktop-acp-events"] },
     gemini: { provider: "gemini_cli" },
     gemini_cli: { provider: "gemini_cli" },
     windsurf: { provider: "windsurf" },
     antigravity: { provider: "antigravity" },
+    antigravity_cli: { provider: "antigravity_cli" },
+    antigravity_ide: { provider: "antigravity_ide" },
+    copilot: { provider: "copilot" },
+    copilot_cli: { provider: "copilot" },
+    factory: { provider: "factory" },
+    factory_droid: { provider: "factory" },
+    droid: { provider: "factory" },
+    grok: { provider: "grok" },
+    grok_build: { provider: "grok" },
+    pi: { provider: "pi" },
     chatgpt: { provider: "chatgpt" },
     claude_web: { provider: "claude_web" },
     claude_ai: { provider: "claude_web" }
@@ -1368,6 +1535,7 @@ function matchesSessionFilter(session, filter) {
   if (filter.provider && session.provider !== filter.provider) return false;
   const sourceTypes = filter.sourceTypes?.length ? filter.sourceTypes : filter.sourceType ? [filter.sourceType] : [];
   if (sourceTypes.length && !sourceTypes.includes(session.sourceType)) return false;
+  if (!filter.includeSubagents && isSubagentSession(session)) return false;
   if (!filter.includeWebChats && session.scopeCanonical && session.storageScope !== "local") return false;
   if (filter.repo && !matchesRepoFilter(session, filter.repo)) return false;
   if (filter.since) {
@@ -1378,6 +1546,10 @@ function matchesSessionFilter(session, filter) {
   return true;
 }
 
+function isSubagentSession(session) {
+  return /_subagent$/.test(String(session?.conversationKind || session?.conversation_kind || ""));
+}
+
 function matchesRepoFilter(session, repoFilter) {
   const wanted = String(repoFilter || "").toLowerCase();
   if (!wanted) return true;
@@ -1442,22 +1614,24 @@ function rebuildIndexSummary(env = process.env, options = {}) {
 }
 
 function indexIsStale(indexPath, env = process.env) {
-  let indexStat;
-  try {
-    indexStat = fs.statSync(indexPath);
-  } catch {
-    return true;
-  }
-  for (const session of listSessions(env)) {
+  const expected = listSessionsSnapshot(env).fingerprint;
+  const actual = indexSessionFingerprint(indexPath, env);
+  return !actual || actual !== expected;
+}
+
+function indexSessionFingerprint(indexPath, env = process.env) {
+  const ftsPath = paths(env).ftsIndex;
+  if (path.resolve(indexPath) === path.resolve(ftsPath)) {
+    let stat;
     try {
-      if (fs.statSync(session.transcriptPath).mtimeMs > indexStat.mtimeMs) return true;
-      if (fs.statSync(session.metadataPath).mtimeMs > indexStat.mtimeMs) return true;
-      if (session.eventPath && fs.existsSync(session.eventPath) && fs.statSync(session.eventPath).mtimeMs > indexStat.mtimeMs) return true;
+      stat = fs.statSync(ftsPath);
     } catch {
-      return true;
+      return "";
     }
+    const rows = readFtsMeta(ftsPath, stat);
+    return rows?.find((row) => row.key === "sessionFingerprint")?.value || "";
   }
-  return false;
+  return readIndexSummary(indexPath)?.sessionFingerprint || "";
 }
 
 function parseSinceFilter(value) {
@@ -1498,11 +1672,23 @@ module.exports = {
   buildIndexSummary,
   chunkText,
   listHistorySessions,
+  listHistorySessionsFromSessions,
   listRecentSessions,
+  listRecentSessionsFromSessions,
   loadIndex,
   readIndexSummary,
+  rebuildIndexSummary,
   reindexIfNeeded,
   sessionHistoryTime,
   searchPastSessions,
-  tokenize
+  tokenize,
+  // Test-only handles so we can simulate native-binding failures without
+  // touching the on-disk .node file. Not part of any public contract.
+  _searchInternals: {
+    openFtsBuildDb,
+    openFtsReadDb,
+    loadBetterSqlite3,
+    markBetterSqlite3Broken,
+    resetBetterSqlite3State
+  }
 };