agentel 0.2.0 → 0.2.3

package/src/search.js

@@ -10,10 +10,33 @@ const { loadConfig } = require("./config");
 const { paths, ensureDir, readJson, writeJson } = require("./paths");
 const { canonicalRepo } = require("./repo");
 
+const INDEX_VERSION = 3;
+const INDEX_STALE_CHECK_TTL_MS = 5000;
+const SQLITE_QUERY_TIMEOUT_MS = 5000;
+const SQLITE_BUILD_BATCH_SIZE = 100;
+const RIPGREP_SEARCH_TIMEOUT_MS = 8000;
+const RIPGREP_BATCH_FILE_COUNT = 200;
+const MARKDOWN_MATCHES_PER_FILE = 3;
+const _indexCache = {
+  path: "",
+  mtimeMs: 0,
+  size: 0,
+  checkedAtMs: 0,
+  index: null
+};
+const _ftsCache = {
+  path: "",
+  mtimeMs: 0,
+  size: 0,
+  checkedAtMs: 0,
+  available: false
+};
+
 function buildIndex(env = process.env) {
   const sessions = listSessions(env);
   const docs = [];
-  const df = {};
+  const postings = Object.create(null);
+  const df = Object.create(null);
   let totalLength = 0;
 
   for (const session of sessions) {
@@ -28,16 +51,20 @@ function buildIndex(env = process.env) {
       for (const chunk of chunkText(indexText)) {
         const tokens = tokenize(chunk);
         if (!tokens.length) continue;
-        const tf = {};
+        const tf = Object.create(null);
         for (const token of tokens) tf[token] = (tf[token] || 0) + 1;
-        for (const token of new Set(tokens)) df[token] = (df[token] || 0) + 1;
+        const docIndex = docs.length;
+        for (const token of new Set(tokens)) {
+          df[token] = (df[token] || 0) + 1;
+          if (!postings[token]) postings[token] = [];
+          postings[token].push([docIndex, tf[token]]);
+        }
         totalLength += tokens.length;
         docs.push({
           ...sourceDoc,
           id: sourceDoc.id || `${session.sessionId}:${sourceDoc.messageIndex ?? docs.length}:${docs.length}`,
           text: chunk,
           matchedText: chunk,
-          tf,
           length: tokens.length
         });
       }
@@ -45,19 +72,360 @@ function buildIndex(env = process.env) {
   }
 
   const index = {
-    version: 2,
+    version: INDEX_VERSION,
     builtAt: new Date().toISOString(),
     docCount: docs.length,
     avgDocLength: docs.length ? totalLength / docs.length : 0,
     df,
+    postings,
     docs
   };
   const indexPath = paths(env).index;
   ensureDir(path.dirname(indexPath));
-  writeJson(indexPath, index);
+  writeJson(indexPath, index, { pretty: false });
+  buildFtsIndex(index, env);
+  rememberIndexCache(indexPath, index);
   return index;
 }
 
+function buildIndexSummary(env = process.env) {
+  const summary = buildFtsIndexSummary(env);
+  const indexPath = paths(env).index;
+  ensureDir(path.dirname(indexPath));
+  writeJson(indexPath, summary, { pretty: false });
+  rememberIndexCache(indexPath, summary);
+  return summary;
+}
+
+function summarizeIndex(index) {
+  if (!index) return null;
+  return {
+    version: index.version,
+    builtAt: index.builtAt,
+    docCount: index.docCount,
+    avgDocLength: index.avgDocLength,
+    summaryOnly: true
+  };
+}
+
+function readIndexSummary(indexPath) {
+  let fd;
+  try {
+    fd = fs.openSync(indexPath, "r");
+    const buffer = Buffer.alloc(8192);
+    const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0);
+    const header = buffer.toString("utf8", 0, bytesRead);
+    const summary = {
+      version: readJsonHeaderNumber(header, "version"),
+      builtAt: readJsonHeaderString(header, "builtAt"),
+      docCount: readJsonHeaderNumber(header, "docCount"),
+      avgDocLength: readJsonHeaderNumber(header, "avgDocLength"),
+      summaryOnly: true
+    };
+    return summary.version == null && summary.docCount == null ? null : summary;
+  } catch (error) {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  } finally {
+    if (fd != null) fs.closeSync(fd);
+  }
+}
+
+function readJsonHeaderNumber(header, key) {
+  const match = header.match(new RegExp(`"${escapeRegExp(key)}"\\s*:\\s*(-?\\d+(?:\\.\\d+)?)`));
+  return match ? Number(match[1]) : undefined;
+}
+
+function readJsonHeaderString(header, key) {
+  const match = header.match(new RegExp(`"${escapeRegExp(key)}"\\s*:\\s*"([^"]*)"`));
+  return match ? match[1] : undefined;
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function buildFtsIndex(index, env = process.env) {
+  const ftsPath = paths(env).ftsIndex;
+  const tmpPath = `${ftsPath}.${process.pid}.tmp`;
+  try {
+    ensureDir(path.dirname(ftsPath));
+    for (const file of [tmpPath, `${tmpPath}-journal`, `${tmpPath}-wal`, `${tmpPath}-shm`]) {
+      try {
+        fs.rmSync(file, { force: true });
+      } catch {
+        // Best effort cleanup before rebuilding the sidecar index.
+      }
+    }
+    runSqliteScript(tmpPath, [
+      "PRAGMA journal_mode=OFF;",
+      "PRAGMA synchronous=OFF;",
+      "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);",
+      "CREATE TABLE docs(",
+      "  rowid INTEGER PRIMARY KEY,",
+      "  doc_id TEXT,",
+      "  session_id TEXT,",
+      "  provider TEXT,",
+      "  source_type TEXT,",
+      "  repo_canonical TEXT,",
+      "  repo_display TEXT,",
+      "  scope_canonical TEXT,",
+      "  cwd TEXT,",
+      "  title TEXT,",
+      "  started_at TEXT,",
+      "  occurred_at TEXT,",
+      "  role TEXT,",
+      "  event_id TEXT,",
+      "  event_kind TEXT,",
+      "  message_index INTEGER,",
+      "  path TEXT,",
+      "  matched_text TEXT",
+      ");",
+      "CREATE VIRTUAL TABLE docs_fts USING fts5(text, tokenize='unicode61', prefix='2 3 4');",
+      `INSERT INTO meta(key, value) VALUES ('version', ${sqliteString(String(INDEX_VERSION))});`,
+      `INSERT INTO meta(key, value) VALUES ('builtAt', ${sqliteString(index.builtAt || "")});`,
+      `INSERT INTO meta(key, value) VALUES ('docCount', ${sqliteString(String(index.docCount || 0))});`
+	    ].join("\n"));
+
+    insertFtsDocs(tmpPath, index.docs || [], 1);
+    runSqliteScript(tmpPath, "INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
+    fs.renameSync(tmpPath, ftsPath);
+    rememberFtsCache(ftsPath, true);
+    return true;
+  } catch {
+    try {
+      fs.rmSync(tmpPath, { force: true });
+    } catch {
+      // Ignore optional FTS cleanup failure.
+    }
+    try {
+      fs.rmSync(ftsPath, { force: true });
+    } catch {
+      // Ignore optional FTS cleanup failure.
+    }
+    rememberFtsCache(ftsPath, false);
+    return false;
+  }
+}
+
+function buildFtsIndexSummary(env = process.env) {
+  const ftsPath = paths(env).ftsIndex;
+  const tmpPath = `${ftsPath}.${process.pid}.tmp`;
+  const builtAt = new Date().toISOString();
+  let docCount = 0;
+  let totalLength = 0;
+  try {
+    ensureDir(path.dirname(ftsPath));
+    for (const file of [tmpPath, `${tmpPath}-journal`, `${tmpPath}-wal`, `${tmpPath}-shm`]) {
+      try {
+        fs.rmSync(file, { force: true });
+      } catch {
+        // Best effort cleanup before rebuilding the sidecar index.
+      }
+    }
+    runSqliteScript(tmpPath, [
+      "PRAGMA journal_mode=OFF;",
+      "PRAGMA synchronous=OFF;",
+      "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);",
+      "CREATE TABLE docs(",
+      "  rowid INTEGER PRIMARY KEY,",
+      "  doc_id TEXT,",
+      "  session_id TEXT,",
+      "  provider TEXT,",
+      "  source_type TEXT,",
+      "  repo_canonical TEXT,",
+      "  repo_display TEXT,",
+      "  scope_canonical TEXT,",
+      "  cwd TEXT,",
+      "  title TEXT,",
+      "  started_at TEXT,",
+      "  occurred_at TEXT,",
+      "  role TEXT,",
+      "  event_id TEXT,",
+      "  event_kind TEXT,",
+      "  message_index INTEGER,",
+      "  path TEXT,",
+      "  matched_text TEXT",
+      ");",
+      "CREATE VIRTUAL TABLE docs_fts USING fts5(text, tokenize='unicode61', prefix='2 3 4');",
+      `INSERT INTO meta(key, value) VALUES ('version', ${sqliteString(String(INDEX_VERSION))});`,
+      `INSERT INTO meta(key, value) VALUES ('builtAt', ${sqliteString(builtAt)});`
+    ].join("\n"));
+
+    let batch = [];
+    for (const session of listSessions(env)) {
+      if (session.conversationPath && !fs.existsSync(session.conversationPath)) ensureConversationMarkdown(session, env);
+      const events = readEvents(session);
+      const eventDocs = events.length ? docsForEvents(session, events) : [];
+      if (!eventDocs.length) ensureConversationMarkdown(session, env);
+      const sourceDocs = eventDocs.length ? eventDocs : docsForTranscript(session, readTranscript(session.transcriptPath));
+      for (const sourceDoc of sourceDocs) {
+        const indexText = normalizeIndexText(sourceDoc.text);
+        if (!indexText) continue;
+        for (const chunk of chunkText(indexText)) {
+          const tokens = tokenize(chunk);
+          if (!tokens.length) continue;
+          const docIndex = docCount;
+          totalLength += tokens.length;
+          docCount++;
+          batch.push({
+            ...sourceDoc,
+            id: sourceDoc.id || `${session.sessionId}:${sourceDoc.messageIndex ?? docIndex}:${docIndex}`,
+            text: chunk,
+            matchedText: chunk,
+            length: tokens.length
+          });
+          if (batch.length >= SQLITE_BUILD_BATCH_SIZE) {
+            insertFtsDocs(tmpPath, batch, docCount - batch.length + 1);
+            batch = [];
+          }
+        }
+      }
+    }
+    if (batch.length) insertFtsDocs(tmpPath, batch, docCount - batch.length + 1);
+    runSqliteScript(tmpPath, [
+      `INSERT INTO meta(key, value) VALUES ('docCount', ${sqliteString(String(docCount))});`,
+      "INSERT INTO docs_fts(docs_fts) VALUES('optimize');"
+    ].join("\n"));
+    fs.renameSync(tmpPath, ftsPath);
+    rememberFtsCache(ftsPath, true);
+  } catch (error) {
+    try {
+      fs.rmSync(tmpPath, { force: true });
+    } catch {
+      // Ignore optional FTS cleanup failure.
+    }
+    try {
+      fs.rmSync(ftsPath, { force: true });
+    } catch {
+      // Ignore optional FTS cleanup failure.
+    }
+    rememberFtsCache(ftsPath, false);
+    throw error;
+  }
+  return {
+    version: INDEX_VERSION,
+    builtAt,
+    docCount,
+    avgDocLength: docCount ? totalLength / docCount : 0,
+    summaryOnly: true
+  };
+}
+
+function insertFtsDocs(dbPath, docs, rowidStart = 1) {
+  for (let start = 0; start < docs.length; start += SQLITE_BUILD_BATCH_SIZE) {
+    const statements = ["BEGIN;"];
+    const batch = docs.slice(start, start + SQLITE_BUILD_BATCH_SIZE);
+    for (let offset = 0; offset < batch.length; offset++) {
+      const rowid = rowidStart + start + offset;
+      const doc = batch[offset];
+      statements.push(
+        `INSERT INTO docs(rowid, doc_id, session_id, provider, source_type, repo_canonical, repo_display, scope_canonical, cwd, title, started_at, occurred_at, role, event_id, event_kind, message_index, path, matched_text) VALUES (` +
+          [
+            rowid,
+            sqliteString(doc.id || ""),
+            sqliteString(doc.sessionId || ""),
+            sqliteString(doc.provider || ""),
+            sqliteString(doc.sourceType || ""),
+            sqliteString(doc.repoCanonical || ""),
+            sqliteString(doc.repoDisplay || ""),
+            sqliteString(doc.scopeCanonical || ""),
+            sqliteString(doc.cwd || ""),
+            sqliteString(doc.title || ""),
+            sqliteString(doc.startedAt || ""),
+            sqliteString(doc.occurredAt || ""),
+            sqliteString(doc.role || ""),
+            sqliteString(doc.eventId || ""),
+            sqliteString(doc.eventKind || ""),
+            Number.isFinite(Number(doc.messageIndex)) ? Number(doc.messageIndex) : "NULL",
+            sqliteString(doc.path || ""),
+            sqliteString(doc.matchedText || "")
+          ].join(", ") +
+          ");"
+      );
+      statements.push(`INSERT INTO docs_fts(rowid, text) VALUES (${rowid}, ${sqliteString(doc.text || "")});`);
+    }
+    statements.push("COMMIT;");
+    runSqliteScript(dbPath, statements.join("\n"));
+  }
+}
+
+function runSqliteScript(dbPath, script) {
+  const result = spawnSync("sqlite3", [dbPath], {
+    argv0: "agentlog-sqlite",
+    input: script,
+    encoding: "utf8",
+    maxBuffer: 1024 * 1024 * 20,
+    timeout: SQLITE_QUERY_TIMEOUT_MS
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error(String(result.stderr || result.stdout || "sqlite3 failed").trim());
+}
+
+function sqliteJson(dbPath, query) {
+  const result = spawnSync("sqlite3", [dbPath, "-json", query], {
+    argv0: "agentlog-sqlite",
+    encoding: "utf8",
+    maxBuffer: 1024 * 1024 * 20,
+    timeout: SQLITE_QUERY_TIMEOUT_MS
+  });
+  if (result.error || result.status !== 0) return null;
+  try {
+    return result.stdout.trim() ? JSON.parse(result.stdout) : [];
+  } catch {
+    return null;
+  }
+}
+
+function sqliteString(value) {
+  return `'${String(value == null ? "" : value).replace(/'/g, "''")}'`;
+}
+
+function rememberFtsCache(ftsPath, available) {
+  let stat = null;
+  try {
+    stat = fs.statSync(ftsPath);
+  } catch {
+    stat = null;
+  }
+  _ftsCache.path = ftsPath;
+  _ftsCache.mtimeMs = stat?.mtimeMs || 0;
+  _ftsCache.size = stat?.size || 0;
+  _ftsCache.checkedAtMs = Date.now();
+  _ftsCache.available = Boolean(available && stat);
+}
+
+function ftsIndexAvailable(env = process.env, options = {}) {
+  const ftsPath = paths(env).ftsIndex;
+  let stat = null;
+  try {
+    stat = fs.statSync(ftsPath);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+    return false;
+  }
+  if (
+    _ftsCache.path === ftsPath &&
+    _ftsCache.available &&
+    _ftsCache.mtimeMs === stat.mtimeMs &&
+    _ftsCache.size === stat.size &&
+    Date.now() - _ftsCache.checkedAtMs < INDEX_STALE_CHECK_TTL_MS
+  ) {
+    return true;
+  }
+  const rows = sqliteJson(ftsPath, "SELECT key, value FROM meta WHERE key IN ('version', 'docCount');");
+  if (!rows || !rows.some((row) => row.key === "version" && Number(row.value) === INDEX_VERSION)) {
+    rememberFtsCache(ftsPath, false);
+    return false;
+  }
+  if (!options.noStaleCheck && indexIsStale(ftsPath, env)) {
+    rememberFtsCache(ftsPath, false);
+    return false;
+  }
+  rememberFtsCache(ftsPath, true);
+  return true;
+}
+
 function docsForEvents(session, events) {
   const indexedKinds = new Set([
     EVENT_KINDS.PROMPT_SUBMITTED,
@@ -120,18 +488,60 @@ function docsForTranscript(session, messages) {
   return docs;
 }
 
-function loadIndex(env = process.env) {
+function loadIndex(env = process.env, options = {}) {
+  const allowRebuild = !options.noRebuild;
   const indexPath = paths(env).index;
+  let stat = null;
+  try {
+    stat = fs.statSync(indexPath);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  if (!stat) return allowRebuild ? buildIndex(env) : null;
+  if (
+    _indexCache.path === indexPath &&
+    _indexCache.index?.version === INDEX_VERSION &&
+    _indexCache.mtimeMs === stat.mtimeMs &&
+    _indexCache.size === stat.size
+  ) {
+    if (Date.now() - _indexCache.checkedAtMs < INDEX_STALE_CHECK_TTL_MS) return _indexCache.index;
+    if (!indexIsStale(indexPath, env)) {
+      _indexCache.checkedAtMs = Date.now();
+      return _indexCache.index;
+    }
+    if (!allowRebuild) return null;
+  }
+  const summary = readIndexSummary(indexPath);
+  if (summary?.version !== INDEX_VERSION) return allowRebuild ? buildIndex(env) : null;
+  if (indexIsStale(indexPath, env)) return allowRebuild ? buildIndex(env) : null;
   const existing = readJson(indexPath, null);
-  if (existing && existing.version === 2 && !indexIsStale(indexPath, env)) return existing;
-  return buildIndex(env);
+  if (existing && existing.version === INDEX_VERSION) {
+    rememberIndexCache(indexPath, existing);
+    return existing;
+  }
+  return allowRebuild ? buildIndex(env) : null;
+}
+
+function rememberIndexCache(indexPath, index) {
+  let stat = null;
+  try {
+    stat = fs.statSync(indexPath);
+  } catch {
+    stat = null;
+  }
+  _indexCache.path = indexPath;
+  _indexCache.mtimeMs = stat?.mtimeMs || 0;
+  _indexCache.size = stat?.size || 0;
+  _indexCache.checkedAtMs = Date.now();
+  _indexCache.index = index;
 }
 
 function searchPastSessions(query, options = {}, env = process.env) {
   try {
     const eventResults = searchIndexedSessions(query, options, env);
-    if (eventResults.length) return eventResults;
+    if (eventResults.length || options.skipMarkdownFallback) return eventResults;
   } catch {
+    if (options.skipMarkdownFallback) return [];
     // Fall through to the legacy markdown path below.
   }
   return searchMarkdownSessions(query, options, env);
@@ -139,6 +549,7 @@ function searchPastSessions(query, options = {}, env = process.env) {
 
 function searchMarkdownSessions(query, options = {}, env = process.env) {
   const limit = Math.max(1, Math.min(Number(options.limit || 10), 50));
+  const maxMatches = Math.max(limit * 8, 40);
   const includeWebChats = Boolean(options.includeWebChats);
   const filter = normalizeSessionFilter(options);
   const repo = filter.repo || inferCallingRepo(options.cwd || process.cwd());
@@ -161,16 +572,18 @@ function searchMarkdownSessions(query, options = {}, env = process.env) {
     sessionByPath.set(path.resolve(session._searchPath), session);
   }
 
-  const matches = ripgrepMatches(queryTokens, [...sessionByPath.keys()]) || jsLineMatches(queryTokens, [...sessionByPath.keys()]);
+  const searchFiles = [...sessionByPath.keys()];
+  const matches = ripgrepMatches(queryTokens, searchFiles, { maxMatches }) || jsLineMatches(queryTokens, searchFiles, { maxMatches });
   const ranked = [];
   const seen = new Set();
+  const lineCache = new Map();
   for (const match of matches) {
     const session = sessionByPath.get(path.resolve(match.path));
     if (!session) continue;
     const key = `${session.sessionId}:${match.line}`;
     if (seen.has(key)) continue;
     seen.add(key);
-    const excerptText = readLineWindow(match.path, match.line, 4);
+    const excerptText = readLineWindow(match.path, match.line, 4, lineCache);
     const lower = excerptText.toLowerCase();
     const matchedTokens = queryTokens.filter((token) => lower.includes(token));
     let score = matchedTokens.length * 5 + (phrase && lower.includes(phrase) ? 10 : 0);
@@ -185,15 +598,20 @@ function searchMarkdownSessions(query, options = {}, env = process.env) {
       cwd: session.cwd || undefined,
       title: session.title || undefined,
       started_at: session.startedAt,
-      role: inferRoleFromMarkdown(match.path, match.line),
+      role: inferRoleFromMarkdown(match.path, match.line, lineCache),
       excerpt: excerptText.replace(/\s+/g, " ").trim(),
       score: Number(score.toFixed(4)),
       session_link: session._searchPath
     });
+    if (ranked.length >= maxMatches) break;
   }
 
   ranked.sort((a, b) => b.score - a.score || String(b.started_at).localeCompare(String(a.started_at)));
-  return ranked.slice(0, limit);
+  const bySession = new Map();
+  for (const item of ranked) {
+    if (!bySession.has(item.session_id)) bySession.set(item.session_id, item);
+  }
+  return [...bySession.values()].slice(0, limit);
 }
 
 function searchIndexedSessions(query, options = {}, env = process.env) {
@@ -202,16 +620,21 @@ function searchIndexedSessions(query, options = {}, env = process.env) {
   const filter = normalizeSessionFilter(options);
   const repo = filter.repo || inferCallingRepo(options.cwd || process.cwd());
   const since = parseSinceFilter(options.since);
-  const index = loadIndex(env);
   const queryTokens = tokenize(query);
   const phrase = String(query || "").trim().toLowerCase();
   if (!queryTokens.length && !phrase) return [];
+  const ftsResults = searchFtsSessions(query, queryTokens, { limit, includeWebChats, filter, repo, since, options }, env);
+  if (ftsResults) return ftsResults;
+  if (options.skipJsonIndex) return [];
+  const index = loadIndex(env, { noRebuild: Boolean(options.noRebuild) });
+  if (!index) return [];
 
   const scored = [];
-  for (const doc of index.docs || []) {
+  const candidates = candidateDocsForQuery(index, queryTokens, phrase);
+  for (const { doc, docIndex } of candidates) {
     if (!matchesSessionFilter(doc, { ...filter, includeWebChats, since })) continue;
 
-    let score = bm25Score(doc, queryTokens, index);
+    let score = bm25Score(doc, queryTokens, index, docIndex, candidates.termFrequencies);
     if (phrase && doc.text.toLowerCase().includes(phrase)) score += 2.5;
     if (!options.repo && repo && doc.repoCanonical === repo) score *= 1.25;
     if (score > 0) scored.push({ doc, score });
@@ -243,35 +666,167 @@ function searchIndexedSessions(query, options = {}, env = process.env) {
   }));
 }
 
-function ripgrepMatches(queryTokens, files) {
+function searchFtsSessions(query, queryTokens, context, env = process.env) {
+  const ftsPath = paths(env).ftsIndex;
+  if (!ftsIndexAvailable(env, { noStaleCheck: Boolean(context.options.noRebuild || context.options.allowStaleFts) })) return null;
+  const matchQuery = ftsMatchQuery(query);
+  if (!matchQuery) return [];
+  const candidateLimit = Math.max(context.limit * 8, 80);
+  const rows = sqliteJson(
+    ftsPath,
+    [
+      "SELECT",
+      "  d.doc_id, d.session_id, d.provider, d.source_type, d.repo_canonical, d.repo_display,",
+      "  d.scope_canonical, d.cwd, d.title, d.started_at, d.occurred_at, d.role,",
+      "  d.event_id, d.event_kind, d.message_index, d.path, d.matched_text,",
+      "  snippet(docs_fts, 0, '', '', '...', 32) AS excerpt,",
+      "  bm25(docs_fts) AS rank",
+      "FROM docs_fts",
+      "JOIN docs d ON d.rowid = docs_fts.rowid",
+      `WHERE docs_fts MATCH ${sqliteString(matchQuery)}`,
+      "ORDER BY rank ASC, d.occurred_at DESC, d.started_at DESC",
+      `LIMIT ${candidateLimit};`
+    ].join("\n")
+  );
+  if (!rows) return null;
+  const bySession = new Map();
+  for (const row of rows) {
+    const doc = ftsRowToDoc(row);
+    if (!matchesSessionFilter(doc, { ...context.filter, includeWebChats: context.includeWebChats, since: context.since })) continue;
+    if (!context.options.repo && context.repo && doc.repoCanonical === context.repo) {
+      row.rank = Number(row.rank || 0) - 0.05;
+    }
+    if (!bySession.has(doc.sessionId)) bySession.set(doc.sessionId, { doc, row });
+    if (bySession.size >= context.limit) break;
+  }
+  return [...bySession.values()].slice(0, context.limit).map(({ doc, row }) => ({
+    session_id: doc.sessionId,
+    provider: doc.provider,
+    source_type: doc.sourceType || undefined,
+    repo: doc.repoCanonical || undefined,
+    repo_display: doc.repoDisplay || doc.repoCanonical || undefined,
+    scope: doc.scopeCanonical || undefined,
+    cwd: doc.cwd || undefined,
+    title: doc.title || undefined,
+    started_at: doc.startedAt,
+    role: doc.role,
+    event_id: doc.eventId || undefined,
+    event_kind: doc.eventKind || undefined,
+    message_index: doc.messageIndex ?? undefined,
+    matched_text: doc.matchedText ? excerpt(doc.matchedText, queryTokens) : undefined,
+    excerpt: row.excerpt || excerpt(doc.matchedText, queryTokens),
+    score: Number((-Number(row.rank || 0)).toFixed(4)),
+    session_link: doc.path
+  }));
+}
+
+function ftsRowToDoc(row) {
+  return {
+    id: row.doc_id || "",
+    sessionId: row.session_id || "",
+    provider: row.provider || "",
+    sourceType: row.source_type || "",
+    repoCanonical: row.repo_canonical || "",
+    repoDisplay: row.repo_display || "",
+    scopeCanonical: row.scope_canonical || "",
+    cwd: row.cwd || "",
+    title: row.title || "",
+    startedAt: row.started_at || "",
+    occurredAt: row.occurred_at || "",
+    role: row.role || "",
+    eventId: row.event_id || "",
+    eventKind: row.event_kind || "",
+    messageIndex: row.message_index == null ? undefined : Number(row.message_index),
+    path: row.path || "",
+    matchedText: row.matched_text || ""
+  };
+}
+
+function ftsMatchQuery(query) {
+  const tokens = String(query || "")
+    .toLowerCase()
+    .match(/[a-z0-9_]{2,}/g);
+  if (!tokens?.length) return "";
+  return [...new Set(tokens.filter((token) => !STOP_WORDS.has(token)))]
+    .slice(0, 12)
+    .map((token) => `"${token.replace(/"/g, '""')}"*`)
+    .join(" ");
+}
+
+function candidateDocsForQuery(index, queryTokens, phrase) {
+  const docs = index.docs || [];
+  const postings = index.postings || null;
+  const uniqueTokens = [...new Set(queryTokens)];
+  const termFrequencies = new Map();
+  const candidateIndexes = new Set();
+
+  if (postings && uniqueTokens.length) {
+    for (const token of uniqueTokens) {
+      const tokenPostings = postings[token] || [];
+      if (!tokenPostings.length) continue;
+      const frequencies = new Map();
+      for (const posting of tokenPostings) {
+        const docIndex = Array.isArray(posting) ? posting[0] : posting.doc;
+        const tf = Array.isArray(posting) ? posting[1] : posting.tf;
+        if (!Number.isInteger(docIndex) || !tf) continue;
+        frequencies.set(docIndex, tf);
+        candidateIndexes.add(docIndex);
+      }
+      termFrequencies.set(token, frequencies);
+    }
+  }
+
+  if (candidateIndexes.size) {
+    const candidates = [...candidateIndexes]
+      .map((docIndex) => ({ doc: docs[docIndex], docIndex }))
+      .filter((item) => item.doc);
+    candidates.termFrequencies = termFrequencies;
+    return candidates;
+  }
+
+  const allDocs = docs.map((doc, docIndex) => ({ doc, docIndex }));
+  allDocs.termFrequencies = termFrequencies;
+  if (phrase && !uniqueTokens.length) return allDocs;
+  if (!postings) return allDocs;
+  return [];
+}
+
+function ripgrepMatches(queryTokens, files, options = {}) {
   if (!queryTokens.length || !files.length) return null;
   const pattern = queryTokens.map(escapeRegex).join("|");
-  const result = spawnSync(
-    "rg",
-    ["--json", "--ignore-case", "--line-number", "-e", pattern, "--", ...files],
-    { encoding: "utf8", maxBuffer: 1024 * 1024 * 50 }
-  );
-  if (result.error && result.error.code === "ENOENT") return null;
-  if (result.status !== 0 && result.status !== 1) return null;
   const matches = [];
-  for (const line of String(result.stdout || "").split(/\r?\n/)) {
-    if (!line.trim()) continue;
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      continue;
+  const maxMatches = Math.max(1, Number(options.maxMatches || 200));
+  for (let start = 0; start < files.length; start += RIPGREP_BATCH_FILE_COUNT) {
+    const batch = files.slice(start, start + RIPGREP_BATCH_FILE_COUNT);
+    const result = spawnSync(
+      "rg",
+      ["--json", "--ignore-case", "--line-number", "--max-count", String(MARKDOWN_MATCHES_PER_FILE), "-e", pattern, "--", ...batch],
+      { encoding: "utf8", maxBuffer: 1024 * 1024 * 10, timeout: RIPGREP_SEARCH_TIMEOUT_MS }
+    );
+    if (result.error && result.error.code === "ENOENT") return null;
+    if (result.error && !String(result.stdout || "").trim()) return matches;
+    if (result.status !== 0 && result.status !== 1 && !String(result.stdout || "").trim()) return matches;
+    for (const line of String(result.stdout || "").split(/\r?\n/)) {
+      if (!line.trim()) continue;
+      let event;
+      try {
+        event = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (event.type !== "match") continue;
+      const file = event.data?.path?.text;
+      const lineNumber = event.data?.line_number;
+      if (file && lineNumber) matches.push({ path: file, line: lineNumber });
+      if (matches.length >= maxMatches) return matches;
     }
-    if (event.type !== "match") continue;
-    const file = event.data?.path?.text;
-    const lineNumber = event.data?.line_number;
-    if (file && lineNumber) matches.push({ path: file, line: lineNumber });
   }
   return matches;
 }
 
-function jsLineMatches(queryTokens, files) {
+function jsLineMatches(queryTokens, files, options = {}) {
   const matches = [];
+  const maxMatches = Math.max(1, Number(options.maxMatches || 200));
   for (const file of files) {
     let lines;
     try {
@@ -279,17 +834,22 @@ function jsLineMatches(queryTokens, files) {
     } catch {
       continue;
     }
+    let fileMatches = 0;
     for (let index = 0; index < lines.length; index++) {
       const lower = lines[index].toLowerCase();
-      if (queryTokens.some((token) => lower.includes(token))) matches.push({ path: file, line: index + 1 });
+      if (!queryTokens.some((token) => lower.includes(token))) continue;
+      matches.push({ path: file, line: index + 1 });
+      fileMatches++;
+      if (matches.length >= maxMatches) return matches;
+      if (fileMatches >= MARKDOWN_MATCHES_PER_FILE) break;
     }
   }
   return matches;
 }
 
-function readLineWindow(file, lineNumber, radius = 2) {
+function readLineWindow(file, lineNumber, radius = 2, cache) {
   try {
-    const lines = fs.readFileSync(file, "utf8").split(/\r?\n/);
+    const lines = readMarkdownLines(file, cache);
     const start = Math.max(0, lineNumber - 1 - radius);
     const end = Math.min(lines.length, lineNumber + radius);
     return lines.slice(start, end).join("\n").trim();
@@ -298,9 +858,9 @@ function readLineWindow(file, lineNumber, radius = 2) {
   }
 }
 
-function inferRoleFromMarkdown(file, lineNumber) {
+function inferRoleFromMarkdown(file, lineNumber, cache) {
   try {
-    const lines = fs.readFileSync(file, "utf8").split(/\r?\n/);
+    const lines = readMarkdownLines(file, cache);
     for (let index = Math.min(lineNumber - 1, lines.length - 1); index >= 0; index--) {
       const match = lines[index].match(/^##\s+([A-Za-z_ -]+)\s+-\s+/);
       if (match) return match[1].trim().toLowerCase().replace(/\s+/g, "_");
@@ -311,18 +871,26 @@ function inferRoleFromMarkdown(file, lineNumber) {
   return "unknown";
 }
 
+function readMarkdownLines(file, cache) {
+  const key = path.resolve(file);
+  if (cache?.has(key)) return cache.get(key);
+  const lines = fs.readFileSync(file, "utf8").split(/\r?\n/);
+  if (cache) cache.set(key, lines);
+  return lines;
+}
+
 function escapeRegex(value) {
   return String(value).replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
 }
 
-function bm25Score(doc, queryTokens, index) {
+function bm25Score(doc, queryTokens, index, docIndex, termFrequencies) {
   const k1 = 1.2;
   const b = 0.75;
   const n = Math.max(1, index.docCount || 1);
   const avgdl = Math.max(1, index.avgDocLength || 1);
   let score = 0;
   for (const token of queryTokens) {
-    const tf = doc.tf?.[token] || 0;
+    const tf = termFrequencyForDoc(doc, token, docIndex, termFrequencies);
     if (!tf) continue;
     const df = index.df?.[token] || 0;
     const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
@@ -331,6 +899,12 @@ function bm25Score(doc, queryTokens, index) {
   return score;
 }
 
+function termFrequencyForDoc(doc, token, docIndex, termFrequencies) {
+  const mapped = termFrequencies?.get(token)?.get(docIndex);
+  if (mapped) return mapped;
+  return doc.tf?.[token] || 0;
+}
+
 function chunkText(text, maxTokens = 220, overlap = 40) {
   const words = String(text || "").split(/\s+/).filter(Boolean);
   if (words.length <= maxTokens) return [words.join(" ")].filter(Boolean);
@@ -415,7 +989,9 @@ function historySessionSummary(session) {
     messages: session.messageCount,
     user_messages: Number.isFinite(Number(session.userMessageCount)) ? Number(session.userMessageCount) : undefined,
     usage: session.usage || undefined,
+    estimatedUsage: session.estimatedUsage || undefined,
     models: session.models || undefined,
+    cursorCommandTypeCounts: session.cursorCommandTypeCounts || undefined,
     conversation: session.conversationPath,
     transcript: session.transcriptPath
   };
@@ -503,7 +1079,7 @@ function normalizeProviderFilter(value) {
     codex_desktop: { provider: "codex", sourceType: "codex-desktop-history", sourceTypes: ["codex-desktop-history"] },
     cursor: { provider: "cursor" },
     cline: { provider: "cline", sourceType: "cline-task-history", sourceTypes: ["cline-task-history"] },
-    opencode: { provider: "opencode", sourceType: "opencode-history", sourceTypes: ["opencode-history"] },
+    opencode: { provider: "opencode", sourceTypes: ["opencode-history", "opencode-sqlite-history"] },
     aider: { provider: "aider", sourceType: "aider-chat-history", sourceTypes: ["aider-chat-history"] },
     devin: { provider: "devin" },
     devin_cli: { provider: "devin", sourceType: "devin-cli-history", sourceTypes: ["devin-cli-history"] },
@@ -541,17 +1117,58 @@ function matchesRepoFilter(session, repoFilter) {
     .some((value) => value === wanted || value.includes(wanted));
 }
 
-function reindexIfNeeded(env = process.env) {
+function reindexIfNeeded(env = process.env, options = {}) {
   const cfg = loadConfig(env);
-  if (cfg.index.paused) return { paused: true, index: readJson(paths(env).index, null) };
   const indexPath = paths(env).index;
+  if (cfg.index.paused) return { paused: true, index: readIndexSummary(indexPath) };
+  let stat = null;
   try {
-    const stat = fs.statSync(indexPath);
-    if (Date.now() - stat.mtimeMs < 10 * 60 * 1000) return { paused: false, index: readJson(indexPath, null) };
-  } catch {
-    // Missing index: build below.
+    stat = fs.statSync(indexPath);
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
   }
-  return { paused: false, index: buildIndex(env) };
+  if (stat) {
+    const summary = readIndexSummary(indexPath);
+    if (summary?.version !== INDEX_VERSION) return { paused: false, index: rebuildIndexSummary(env, options), rebuilt: true };
+    const intervalMs = configuredIndexIntervalMs(cfg);
+    if (Date.now() - stat.mtimeMs < intervalMs) return { paused: false, index: summary };
+    if (!indexIsStale(indexPath, env)) return { paused: false, index: summary };
+  }
+  return { paused: false, index: rebuildIndexSummary(env, options), rebuilt: true };
+}
+
+function configuredIndexIntervalMs(config) {
+  const minutes = Number(config?.index?.intervalMinutes ?? 10);
+  return Math.max(1, Number.isFinite(minutes) ? minutes : 10) * 60 * 1000;
+}
+
+function buildIndexInChild(env = process.env) {
+  const script = `
+process.title = "agentlog-index";
+const { buildIndexSummary } = require(${JSON.stringify(__filename)});
+const index = buildIndexSummary(process.env);
+process.stdout.write(JSON.stringify(index));
+`;
+  const result = spawnSync(process.execPath, ["-e", script], {
+    argv0: "agentlog-index",
+    env,
+    encoding: "utf8",
+    maxBuffer: 1024 * 1024
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) {
+    const message = String(result.stderr || result.stdout || `index rebuild exited with status ${result.status}`).trim();
+    throw new Error(message);
+  }
+  try {
+    return JSON.parse(result.stdout || "null") || null;
+  } catch (error) {
+    throw new Error(`index rebuild returned invalid summary: ${error.message}`);
+  }
+}
+
+function rebuildIndexSummary(env = process.env, options = {}) {
+  return options.rebuildInProcess ? buildIndexSummary(env) : buildIndexInChild(env);
 }
 
 function indexIsStale(indexPath, env = process.env) {
@@ -608,10 +1225,12 @@ const STOP_WORDS = new Set([
 
 module.exports = {
   buildIndex,
+  buildIndexSummary,
   chunkText,
   listHistorySessions,
   listRecentSessions,
   loadIndex,
+  readIndexSummary,
   reindexIfNeeded,
   sessionHistoryTime,
   searchPastSessions,