@threadbase-sh/scanner 0.12.4 → 0.14.0

package/dist/index.cjs

@@ -212,18 +212,43 @@ function readGitBranch(projectPath) {
 var import_flexsearch = __toESM(require("flexsearch"), 1);
 
 // src/search-matches.ts
+var FTS_HIT_OPEN = "";
+var FTS_HIT_CLOSE = "";
+var FTS_SNIPPET_TOKENS = 16;
+var FTS_ELLIPSIS = "\u2026";
+var CONTENT_FIELD = "content";
+function parseFtsSnippet(raw) {
+  if (!raw?.includes(FTS_HIT_OPEN)) return null;
+  const highlights = [];
+  let snippet = "";
+  let openAt = -1;
+  for (const ch of raw) {
+    if (ch === FTS_HIT_OPEN) {
+      openAt = snippet.length;
+      continue;
+    }
+    if (ch === FTS_HIT_CLOSE) {
+      if (openAt >= 0 && snippet.length > openAt) {
+        highlights.push({ start: openAt, end: snippet.length });
+      }
+      openAt = -1;
+      continue;
+    }
+    snippet += ch;
+  }
+  return highlights.length > 0 ? { snippet, highlights } : null;
+}
 function generateMatches(meta, query) {
   const matches = [];
   const lowerQuery = query.toLowerCase();
   const fields = [
-    ["contentSnippet", meta.contentSnippet],
-    ["projectName", meta.projectName],
-    ["sessionId", meta.sessionId],
     ["sessionName", meta.sessionName],
-    ["account", meta.account],
-    ["model", meta.model || ""],
+    ["projectName", meta.projectName],
     ["gitBranch", meta.gitBranch || ""],
-    ["toolNames", meta.toolNames.join(" ")]
+    ["toolNames", meta.toolNames.join(" ")],
+    ["model", meta.model || ""],
+    ["account", meta.account],
+    ["sessionId", meta.sessionId]
   ];
   for (const [field, value] of fields) {
     const idx = value.toLowerCase().indexOf(lowerQuery);
@@ -238,6 +263,20 @@ function generateMatches(meta, query) {
   }
   return matches.length > 0 ? matches : [{ field: "preview", snippet: meta.preview }];
 }
+function buildContentMatch(searchContent, query) {
+  if (!searchContent || !query.trim()) return null;
+  const idx = searchContent.toLowerCase().indexOf(query.toLowerCase());
+  if (idx === -1) return null;
+  const start = Math.max(0, idx - 80);
+  const end = Math.min(searchContent.length, idx + query.length + 120);
+  const body = searchContent.slice(start, end).replace(/\s+/g, " ").trim();
+  const hitAt = body.toLowerCase().indexOf(query.toLowerCase());
+  const prefix = start > 0 ? FTS_ELLIPSIS : "";
+  const suffix = end < searchContent.length ? FTS_ELLIPSIS : "";
+  const snippet = `${prefix}${body}${suffix}`;
+  const highlights = hitAt === -1 ? [] : [{ start: hitAt + prefix.length, end: hitAt + prefix.length + query.length }];
+  return { field: CONTENT_FIELD, snippet, highlights };
+}
 
 // src/indexer.ts
 var FlexSearch = import_flexsearch.default.default ?? import_flexsearch.default;
@@ -245,6 +284,10 @@ var SearchIndexer = class {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   index;
   documents = /* @__PURE__ */ new Map();
+  // The indexed body per conversation, kept so a hit can produce a real excerpt
+  // instead of falling back to an unrelated preview. Sized by the caller (the
+  // scanner tail-caps it to the content tier) — see the note on addDocument.
+  searchContents = /* @__PURE__ */ new Map();
   constructor() {
     this.index = this.createIndex();
   }
@@ -270,25 +313,25 @@ var SearchIndexer = class {
       cache: 100
     });
   }
-  addDocument(meta) {
+  // `searchContent` is the combined search document (text + thinking + tools),
+  // NOT meta.contentSnippet.
+  //
+  // It arrives pre-capped. This index is `tokenize: "forward"` at resolution 9,
+  // which stores every prefix of every token, so it is a resident-memory
+  // structure whose cost is very different from the on-disk FTS index. Feeding
+  // it the full ~128 KB budget across a few hundred conversations would be a
+  // multi-hundred-MB-to-GB index. The scanner therefore caps this to the active
+  // tier's snippetMax and accepts that `persistent: false` has lower recall than
+  // SQLite — an honest, documented divergence rather than a silent one.
+  addDocument(meta, searchContent = "") {
     this.documents.set(meta.id, meta);
-    this.index.add({
-      id: meta.id,
-      content: meta.contentSnippet,
-      projectName: meta.projectName,
-      projectPath: meta.projectPath,
-      sessionId: meta.sessionId,
-      sessionName: meta.sessionName,
-      account: meta.account,
-      model: meta.model || "",
-      gitBranch: meta.gitBranch || "",
-      toolNames: meta.toolNames.join(" ")
-    });
+    this.searchContents.set(meta.id, searchContent);
+    this.index.add(toIndexDoc(meta, searchContent));
   }
-  buildIndex(metas) {
+  buildIndex(metas, searchContents) {
     this.clear();
     for (const meta of metas) {
-      this.addDocument(meta);
+      this.addDocument(meta, searchContents?.get(meta.id) ?? "");
     }
     getLogger().debug({ docCount: metas.length }, "indexer: built");
   }
@@ -308,14 +351,26 @@ var SearchIndexer = class {
         seen.add(id);
         const meta = this.documents.get(id);
         if (!meta) continue;
-        const matches = generateMatches(meta, query);
-        searchResults.push({ meta, score: 1, matches });
+        searchResults.push({
+          meta,
+          score: 1,
+          matches: this.matchesFor(meta, query)
+        });
         if (searchResults.length >= limit) break;
       }
       if (searchResults.length >= limit) break;
     }
     return searchResults;
   }
+  // Body context first (that is what explains why the result appeared), then any
+  // metadata matches. Only when neither hits does generateMatches' preview
+  // fallback stand in.
+  matchesFor(meta, query) {
+    const contentMatch = buildContentMatch(this.searchContents.get(meta.id) ?? "", query);
+    const metaMatches = generateMatches(meta, query);
+    if (!contentMatch) return metaMatches;
+    return [contentMatch, ...metaMatches.filter((m) => m.field !== "preview")];
+  }
   getRecent(limit) {
     return Array.from(this.documents.values()).sort((a, b) => b.timestamp.localeCompare(a.timestamp)).slice(0, limit).map((meta) => ({
       meta,
@@ -329,31 +384,37 @@ var SearchIndexer = class {
   // Replace an already-indexed document in place. FlexSearch's `add` does not
   // overwrite an existing id, so a single-file refresh must go through
   // `update` to avoid stale matches lingering in the index.
-  updateDocument(meta) {
+  updateDocument(meta, searchContent = "") {
     this.documents.set(meta.id, meta);
-    this.index.update({
-      id: meta.id,
-      content: meta.contentSnippet,
-      projectName: meta.projectName,
-      projectPath: meta.projectPath,
-      sessionId: meta.sessionId,
-      sessionName: meta.sessionName,
-      account: meta.account,
-      model: meta.model || "",
-      gitBranch: meta.gitBranch || "",
-      toolNames: meta.toolNames.join(" ")
-    });
+    this.searchContents.set(meta.id, searchContent);
+    this.index.update(toIndexDoc(meta, searchContent));
   }
   removeDocument(id) {
     this.documents.delete(id);
+    this.searchContents.delete(id);
     this.index.remove(id);
   }
   clear() {
     this.documents.clear();
+    this.searchContents.clear();
     this.index = this.createIndex();
     getLogger().trace("indexer: cleared");
   }
 };
+function toIndexDoc(meta, searchContent) {
+  return {
+    id: meta.id,
+    content: searchContent,
+    projectName: meta.projectName,
+    projectPath: meta.projectPath,
+    sessionId: meta.sessionId,
+    sessionName: meta.sessionName,
+    account: meta.account,
+    model: meta.model || "",
+    gitBranch: meta.gitBranch || "",
+    toolNames: meta.toolNames.join(" ")
+  };
+}
 
 // src/parser.ts
 var import_fs2 = require("fs");
@@ -519,7 +580,7 @@ function cleanSystemTags(text) {
 }
 
 // src/parser.ts
-async function parseMeta(filePath, account, tier) {
+async function parseMeta(filePath, account, tier, onEntry) {
   const log = getLogger();
   log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
   const state = initialReducerState();
@@ -536,6 +597,7 @@ async function parseMeta(filePath, account, tier) {
         continue;
       }
       reduceLine(state, entry, tier);
+      onEntry?.(entry);
     }
   } catch (err) {
     log.warn({ filePath, err }, "parseMeta: read failed");
@@ -1290,7 +1352,7 @@ var import_promises5 = require("timers/promises");
 var import_fs5 = require("fs");
 var import_promises4 = require("timers/promises");
 var YIELD_EVERY_LINES = 500;
-async function tailReduce(filePath, startOffset, startLine, state, tier) {
+async function tailReduce(filePath, startOffset, startLine, state, tier, onEntry) {
   const stream = (0, import_fs5.createReadStream)(filePath, { start: startOffset, encoding: "utf8" });
   let buffer = "";
   let offset = startOffset;
@@ -1306,7 +1368,9 @@ async function tailReduce(filePath, startOffset, startLine, state, tier) {
       buffer = buffer.slice(nl + 1);
       if (text.length > 0) {
         try {
-          reduceLine(state, JSON.parse(text), tier);
+          const entry = JSON.parse(text);
+          reduceLine(state, entry, tier);
+          onEntry?.(entry);
         } catch {
           state.badJsonLines++;
         }
@@ -1542,7 +1606,7 @@ var import_fs10 = require("fs");
 // src/providers/parse.ts
 var import_fs8 = require("fs");
 var import_readline3 = require("readline");
-async function parseMetaWithProvider(provider, filePath, account, tier) {
+async function parseMetaWithProvider(provider, filePath, account, tier, onEntry) {
   const log = getLogger();
   const acc = provider.createEmptyAccumulator();
   const rl = (0, import_readline3.createInterface)({
@@ -1560,6 +1624,7 @@ async function parseMetaWithProvider(provider, filePath, account, tier) {
       }
       try {
         provider.reduceEntry(acc, entry, tier);
+        onEntry?.(entry);
       } catch (err) {
         log.warn({ filePath, provider: provider.name, err }, "provider reduce threw; line skipped");
       }
@@ -1571,6 +1636,151 @@ async function parseMetaWithProvider(provider, filePath, account, tier) {
   return provider.finalize(acc, filePath, account, tier);
 }
 
+// src/search-document.ts
+var SEARCH_BUDGET = {
+  textMax: 64 * 1024,
+  thinkingMax: 32 * 1024,
+  toolsMax: 32 * 1024,
+  toolPayloadMax: 4 * 1024
+};
+var SEP = "\n\n";
+function emptySearchDocument() {
+  return { text: "", thinking: "", tools: "" };
+}
+function appendSearchDelta(doc, delta) {
+  return {
+    text: tailAppend(doc.text, delta.text, SEARCH_BUDGET.textMax),
+    thinking: tailAppend(doc.thinking, delta.thinking, SEARCH_BUDGET.thinkingMax),
+    tools: tailAppend(doc.tools, delta.tools, SEARCH_BUDGET.toolsMax)
+  };
+}
+function tailAppend(current, incoming, max) {
+  if (!incoming) return current;
+  const joined = current ? current + SEP + incoming : incoming;
+  return joined.length <= max ? joined : joined.slice(-max);
+}
+function combineSearchContent(doc) {
+  return [doc.text, doc.thinking, doc.tools].filter(Boolean).join(SEP);
+}
+function capToolPayload(value) {
+  const raw = stringifyPayload(value);
+  if (!raw) return "";
+  return raw.length > SEARCH_BUDGET.toolPayloadMax ? raw.slice(0, SEARCH_BUDGET.toolPayloadMax) : raw;
+}
+function stringifyPayload(value) {
+  if (value === null || value === void 0) return "";
+  if (typeof value === "string") return value;
+  try {
+    return JSON.stringify(value) ?? "";
+  } catch {
+    return "";
+  }
+}
+function extractSearchDelta(entry) {
+  if (entry.type === "response_item" || entry.type === "session_meta") {
+    return extractCodexDelta(entry);
+  }
+  return extractClaudeDelta(entry);
+}
+function extractClaudeDelta(entry) {
+  const type = entry.type;
+  if (type !== "user" && type !== "assistant") return emptySearchDocument();
+  if (entry.isMeta) return emptySearchDocument();
+  const msg = entry.message;
+  const content = msg?.content;
+  const tools = [
+    extractClaudeToolContent(content),
+    // Claude stores the rich/structured tool result at the JSONL entry's top
+    // level, not inside message.content — indexing only message.content would
+    // miss most real tool output (file reads, command stdout).
+    capToolPayload(entry.toolUseResult)
+  ].filter(Boolean).join(SEP);
+  return {
+    text: extractClaudeText(content),
+    thinking: type === "assistant" ? extractThinking(content).content : "",
+    tools
+  };
+}
+function extractClaudeText(content) {
+  if (typeof content === "string") return cleanSystemTags(content);
+  if (!Array.isArray(content)) return "";
+  const parts = [];
+  for (const item of content) {
+    if (typeof item === "string") {
+      const cleaned = cleanSystemTags(item);
+      if (cleaned) parts.push(cleaned);
+    } else if (item?.type === "text" && typeof item.text === "string") {
+      const cleaned = cleanSystemTags(item.text);
+      if (cleaned) parts.push(cleaned);
+    }
+  }
+  return parts.join(SEP);
+}
+function extractClaudeToolContent(content) {
+  if (!Array.isArray(content)) return "";
+  const parts = [];
+  for (const item of content) {
+    if (item?.type === "tool_use") {
+      const capped = capToolPayload(item.input);
+      if (capped) parts.push(capped);
+    } else if (item?.type === "tool_result") {
+      const capped = capToolPayload(item.content);
+      if (capped) parts.push(capped);
+    }
+  }
+  return parts.join(SEP);
+}
+function extractCodexDelta(entry) {
+  const payload = entry.payload;
+  if (!payload || typeof payload !== "object") return emptySearchDocument();
+  const ptype = payload.type;
+  if (ptype === "function_call" || ptype === "custom_tool_call") {
+    return { text: "", thinking: "", tools: capToolPayload(payload.arguments) };
+  }
+  if (ptype === "function_call_output" || ptype === "custom_tool_call_output") {
+    return { text: "", thinking: "", tools: capToolPayload(payload.output) };
+  }
+  if (ptype === "reasoning") {
+    return { text: "", thinking: extractCodexReasoning(payload), tools: "" };
+  }
+  if (ptype === "message") {
+    const role = payload.role;
+    if (role !== "user" && role !== "assistant") return emptySearchDocument();
+    return { text: extractCodexText2(payload.content), thinking: "", tools: "" };
+  }
+  return emptySearchDocument();
+}
+function extractCodexReasoning(payload) {
+  const parts = [];
+  for (const key of ["summary", "content"]) {
+    const blocks = payload[key];
+    if (!Array.isArray(blocks)) continue;
+    for (const block of blocks) {
+      if (typeof block === "string") parts.push(block);
+      else if (typeof block?.text === "string") parts.push(block.text);
+    }
+  }
+  return parts.filter(Boolean).join(SEP);
+}
+function extractCodexText2(content) {
+  if (typeof content === "string") return cleanSystemTags(content);
+  if (!Array.isArray(content)) return "";
+  const parts = [];
+  for (const item of content) {
+    if (typeof item === "string") {
+      const cleaned = cleanSystemTags(item);
+      if (cleaned) parts.push(cleaned);
+      continue;
+    }
+    const t = item?.type;
+    if ((t === "input_text" || t === "output_text" || t === "text") && typeof item.text === "string") {
+      const cleaned = cleanSystemTags(item.text);
+      if (cleaned) parts.push(cleaned);
+    }
+  }
+  return parts.join(SEP);
+}
+
 // src/tiers.ts
 var DEFAULT_TIERS = {
   standard: { name: "standard", previewMax: 200, snippetMax: 5e3 },
@@ -1592,7 +1802,7 @@ var import_fs9 = require("fs");
 var import_path8 = require("path");
 
 // src/persistent/schema.ts
-var SCHEMA_VERSION = 4;
+var SCHEMA_VERSION = 5;
 var SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS conversation_files (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1692,13 +1902,27 @@ CREATE INDEX IF NOT EXISTS idx_conversations_account_recent ON conversations(acc
 CREATE INDEX IF NOT EXISTS idx_conversations_subagent_recent ON conversations(is_subagent, timestamp DESC);
 CREATE INDEX IF NOT EXISTS idx_conversations_team_recent ON conversations(team_name, timestamp DESC);
 
--- Full-text search index over conversation content + metadata. Kept separate
--- from the metadata tables so list-screen queries stay small and fast.
--- source_path is UNINDEXED (stored, not tokenized) and links back to a
--- conversations row. One FTS row per conversation, replaced on each upsert.
+-- Full-text search index over conversation body + metadata. Kept separate from
+-- the metadata tables so list-screen queries stay small and fast. One FTS row
+-- per conversation, replaced on each upsert.
+--
+-- The row's rowid IS conversation_files.id. That is load-bearing, not cosmetic:
+-- FTS5's query planner only handles MATCH, rowid and rank, so any other
+-- constraint (including a source_path equality on an UNINDEXED column) has no
+-- index and linear-scans the whole table. At ~128 KB of body per row that would
+-- mean scanning the entire corpus on every append. Look rows up by rowid.
+-- source_path stays stored-but-UNINDEXED so a search hit can resolve back to a
+-- conversations row.
+--
+-- Body is three columns, not one: text > thinking > tools priority has to
+-- survive an append (a new user message belongs in the text column, not after
+-- the tool output already written). They are deliberately NOT concatenated into
+-- a fourth column - that would double-weight body hits and inflate bm25 length.
 CREATE VIRTUAL TABLE IF NOT EXISTS conversation_messages_fts USING fts5(
   source_path UNINDEXED,
-  content,
+  text,
+  thinking,
+  tools,
   project_name,
   session_id,
   session_name,
@@ -1768,6 +1992,24 @@ function runMigrations(db) {
   if (current >= 1 && current < 3 && tableExists(db, "conversations")) {
     db.exec("UPDATE conversations SET provider = 'claude-code' WHERE provider = 'threadbase'");
   }
+  if (current >= 1 && current < 5) {
+    db.exec("DROP TABLE IF EXISTS conversation_messages_fts");
+    if (tableExists(db, "conversation_files")) {
+      const assignments = [];
+      if (hasColumn(db, "conversation_files", "last_indexed_offset")) {
+        assignments.push("last_indexed_offset = 0");
+      }
+      if (hasColumn(db, "conversation_files", "last_indexed_line")) {
+        assignments.push("last_indexed_line = 0");
+      }
+      if (hasColumn(db, "conversation_files", "reducer_state")) {
+        assignments.push("reducer_state = NULL");
+      }
+      if (assignments.length > 0) {
+        db.exec(`UPDATE conversation_files SET ${assignments.join(", ")}`);
+      }
+    }
+  }
   db.exec(SCHEMA_SQL);
   db.pragma(`user_version = ${SCHEMA_VERSION}`);
 }
@@ -1784,15 +2026,46 @@ function openDatabase(dbPath) {
   if (dbPath !== ":memory:") {
     (0, import_fs9.mkdirSync)((0, import_path8.dirname)(dbPath), { recursive: true });
   }
-  const db = new import_better_sqlite3.default(dbPath);
+  let db;
+  try {
+    db = new import_better_sqlite3.default(dbPath);
+  } catch (err) {
+    if (isNativeBindingFailure(err)) throw nativeBindingError(err, dbPath);
+    throw err;
+  }
   db.pragma("journal_mode = WAL");
   db.pragma("synchronous = NORMAL");
   db.pragma("temp_store = MEMORY");
   db.pragma("foreign_keys = ON");
+  db.pragma("busy_timeout = 5000");
   runMigrations(db);
   getLogger().debug({ dbPath }, "db: opened");
   return db;
 }
+function isNativeBindingFailure(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return /could not locate the bindings file|NODE_MODULE_VERSION|was compiled against|\.node['"\s]/i.test(
+    message
+  );
+}
+function nativeBindingError(err, dbPath) {
+  const detail = err instanceof Error ? err.message : String(err);
+  return new Error(
+    [
+      `Could not load better-sqlite3's native binary, so the persistent index at ${dbPath} cannot be opened.`,
+      "This usually means the binary was never built or downloaded \u2014 not that your Node version is wrong.",
+      "",
+      "Try, in order:",
+      "  1. npm rebuild better-sqlite3",
+      "  2. rm -rf node_modules && npm install",
+      "     (npm 12 blocks package install scripts by default; approve better-sqlite3 if asked)",
+      "  3. Run without SQLite entirely: new ConversationScanner({ persistent: false })",
+      "     or pass --no-persist on the CLI. Search falls back to the in-memory index.",
+      "",
+      `Original error: ${detail}`
+    ].join("\n")
+  );
+}
 
 // src/persistent/dir-watermark.ts
 var import_promises6 = require("fs/promises");
@@ -2236,22 +2509,31 @@ var ConversationsRepo = class {
 };
 
 // src/persistent/repositories/fts.repo.ts
+var BODY_COLUMNS = [
+  { name: "text", index: 1 },
+  { name: "thinking", index: 2 },
+  { name: "tools", index: 3 }
+];
 var FtsRepo = class {
   constructor(db) {
     this.db = db;
   }
   db;
-  upsert(meta) {
+  upsert(rowId, meta, doc) {
     const sourcePath = canonicalPath(meta.id);
     const tx = this.db.transaction(() => {
-      this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(sourcePath);
+      this.db.prepare("DELETE FROM conversation_messages_fts WHERE rowid = ?").run(rowId);
       this.db.prepare(
         `INSERT INTO conversation_messages_fts
-             (source_path, content, project_name, session_id, session_name, account, model, branch, tool_names)
-           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+             (rowid, source_path, text, thinking, tools,
+              project_name, session_id, session_name, account, model, branch, tool_names)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
       ).run(
+        rowId,
         sourcePath,
-        meta.contentSnippet ?? "",
+        doc.text,
+        doc.thinking,
+        doc.tools,
         meta.projectName ?? "",
         meta.sessionId ?? "",
         meta.sessionName ?? "",
@@ -2263,26 +2545,94 @@ var FtsRepo = class {
     });
     tx();
   }
-  remove(sourcePath) {
-    this.db.prepare("DELETE FROM conversation_messages_fts WHERE source_path = ?").run(canonicalPath(sourcePath));
+  // Current durable buckets for a conversation, so an append can tail-extend
+  // them without reparsing the file. Returns null when no row exists yet.
+  readDocument(rowId) {
+    const row = this.db.prepare("SELECT text, thinking, tools FROM conversation_messages_fts WHERE rowid = ?").get(rowId);
+    if (!row) return null;
+    return { text: row.text ?? "", thinking: row.thinking ?? "", tools: row.tools ?? "" };
   }
-  // Ranked source_paths matching the query, best first. Returns [] on an empty
-  // query (callers fall back to a recency listing).
-  search(query, limit) {
+  // True when the stored row already equals what we would write, so an append
+  // can skip re-tokenizing ~128 KB of body for nothing.
+  //
+  // The check covers the metadata columns too, not just the buckets: a
+  // newly-seen tool name or a late-resolved session_name changes metadata while
+  // the body is byte-identical, and nothing else ever rewrites this row — so
+  // skipping on "body unchanged" alone would strand that stale value forever.
+  isCurrent(rowId, meta, doc) {
+    const row = this.db.prepare(
+      `SELECT text, thinking, tools,
+                project_name, session_id, session_name, account, model, branch, tool_names
+         FROM conversation_messages_fts WHERE rowid = ?`
+    ).get(rowId);
+    if (!row) return false;
+    return row.text === doc.text && row.thinking === doc.thinking && row.tools === doc.tools && row.project_name === (meta.projectName ?? "") && row.session_id === (meta.sessionId ?? "") && row.session_name === (meta.sessionName ?? "") && row.account === (meta.account ?? "") && row.model === (meta.model ?? "") && row.branch === (meta.gitBranch ?? "") && row.tool_names === meta.toolNames.join(" ");
+  }
+  remove(rowId) {
+    this.db.prepare("DELETE FROM conversation_messages_fts WHERE rowid = ?").run(rowId);
+  }
+  // Ranked hits matching the query, best first. Filters run in SQL BEFORE LIMIT:
+  // with a wide corpus a popular term matches far more conversations than one
+  // page, so post-filtering an already-truncated list would report "no results"
+  // for queries that do have them.
+  search(query, limit, filters = {}) {
     const match = toMatchQuery(query);
     if (!match) return [];
+    const snippetSelects = BODY_COLUMNS.map(
+      (c) => `snippet(conversation_messages_fts, ${c.index}, ?, ?, ?, ?) AS snippet_${c.name}`
+    ).join(", ");
+    const params = [];
+    for (const _col of BODY_COLUMNS) {
+      params.push(FTS_HIT_OPEN, FTS_HIT_CLOSE, FTS_ELLIPSIS, FTS_SNIPPET_TOKENS);
+    }
+    params.push(match);
+    const where = ["conversation_messages_fts MATCH ?", "c.status = 'active'"];
+    if (filters.account) {
+      where.push("c.account = ?");
+      params.push(filters.account);
+    }
+    if (filters.provider) {
+      where.push("c.provider = ?");
+      params.push(filters.provider);
+    }
+    if (filters.project) {
+      where.push("(lower(c.project_path) LIKE ? OR lower(c.project_name) LIKE ?)");
+      const like = `%${filters.project.toLowerCase()}%`;
+      params.push(like, like);
+    }
+    if (filters.since) {
+      where.push("c.timestamp >= ?");
+      params.push(filters.since.toISOString());
+    }
+    if (filters.include === "conversations") {
+      where.push("c.is_subagent = 0 AND c.is_teammate = 0");
+    } else if (filters.include === "subagents") {
+      where.push("c.is_subagent = 1");
+    } else if (filters.include === "teammates") {
+      where.push("c.is_teammate = 1");
+    }
+    params.push(limit);
     const rows = this.db.prepare(
-      `SELECT source_path FROM conversation_messages_fts
-         WHERE conversation_messages_fts MATCH ?
+      `SELECT conversation_messages_fts.source_path AS source_path, ${snippetSelects}
+         FROM conversation_messages_fts
+         JOIN conversations c ON c.source_path = conversation_messages_fts.source_path
+         WHERE ${where.join(" AND ")}
          ORDER BY rank
          LIMIT ?`
-    ).all(match, limit);
-    return rows.map((r) => r.source_path);
+    ).all(...params);
+    return rows.map((row) => ({ sourcePath: row.source_path, body: pickBodySnippet(row) }));
   }
   count() {
     return this.db.prepare("SELECT COUNT(*) AS n FROM conversation_messages_fts").get().n;
   }
 };
+function pickBodySnippet(row) {
+  for (const col of BODY_COLUMNS) {
+    const parsed = parseFtsSnippet(row[`snippet_${col.name}`]);
+    if (parsed) return parsed;
+  }
+  return null;
+}
 function toMatchQuery(query) {
   const terms = query.trim().split(/\s+/).map((t) => t.replace(/"/g, "").trim()).filter(Boolean);
   if (terms.length === 0) return "";
@@ -2462,9 +2812,13 @@ var PersistentEngine = class {
     const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
     const startOffset = resume ? existing.last_indexed_offset : 0;
     const startLine = resume ? existing.last_indexed_line : 0;
+    let searchDelta = emptySearchDocument();
+    const collectSearch = (entry) => {
+      searchDelta = appendSearchDelta(searchDelta, extractSearchDelta(entry));
+    };
     let result;
     try {
-      result = await tailReduce(filePath, startOffset, startLine, state, tier);
+      result = await tailReduce(filePath, startOffset, startLine, state, tier, collectSearch);
     } catch (err) {
       log.warn({ filePath, err }, "persistent: tail read failed");
       return { meta: null, change };
@@ -2477,9 +2831,13 @@ var PersistentEngine = class {
     meta.gitBranch = resolveGitBranch(meta.projectPath);
     const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
     const fileId = this.files.ensure(filePath, account);
+    const base = resume ? this.fts.readDocument(fileId) ?? emptySearchDocument() : emptySearchDocument();
+    const searchDoc = appendSearchDelta(base, searchDelta);
     const upsert = this.db.transaction(() => {
       this.conversations.upsert(fileId, meta, state.pageMessageCount);
-      this.fts.upsert(meta);
+      if (!this.fts.isCurrent(fileId, meta, searchDoc)) {
+        this.fts.upsert(fileId, meta, searchDoc);
+      }
       if (!resume) this.checkpoints.remove(filePath);
       this.files.updateCursor(fileId, {
         sizeBytes: stat4.size,
@@ -2522,7 +2880,10 @@ var PersistentEngine = class {
   // any change reparses from 0 again. No reducer_state is persisted.
   async indexFileWithProvider(provider, filePath, account, tier, stat4, resolveGitBranch) {
     const log = getLogger();
-    const meta = await parseMetaWithProvider(provider, filePath, account, tier);
+    let searchDoc = emptySearchDocument();
+    const meta = await parseMetaWithProvider(provider, filePath, account, tier, (entry) => {
+      searchDoc = appendSearchDelta(searchDoc, extractSearchDelta(entry));
+    });
     if (!meta) {
       this.markDeleted(filePath);
       return null;
@@ -2534,7 +2895,9 @@ var PersistentEngine = class {
     const fileId = this.files.ensure(filePath, account);
     const upsert = this.db.transaction(() => {
       this.conversations.upsert(fileId, meta, meta.messageCount);
-      this.fts.upsert(meta);
+      if (!this.fts.isCurrent(fileId, meta, searchDoc)) {
+        this.fts.upsert(fileId, meta, searchDoc);
+      }
       this.checkpoints.remove(filePath);
       this.files.updateCursor(fileId, {
         sizeBytes: stat4.size,
@@ -2560,7 +2923,7 @@ var PersistentEngine = class {
     if (!existing) return;
     const tx = this.db.transaction(() => {
       this.conversations.deleteByFileId(existing.id);
-      this.fts.remove(filePath);
+      this.fts.remove(existing.id);
       this.checkpoints.remove(filePath);
       this.files.setStatus(existing.id, "deleted");
     });
@@ -2576,20 +2939,24 @@ var PersistentEngine = class {
   getAllBySessionId(sessionId) {
     return this.conversations.getAllBySessionId(sessionId);
   }
-  // Ranked metas matching the FTS query, best first. Empty query returns the
-  // most recent conversations (mirroring the in-memory indexer's empty-query
-  // behavior). Resolves each FTS hit to its active conversation row.
-  searchMetas(query, limit) {
-    if (!query.trim()) {
-      return this.conversations.recent(limit);
-    }
-    const paths = this.fts.search(query, limit);
-    const metas = [];
-    for (const path of paths) {
-      const meta = this.conversations.getBySourcePath(path);
-      if (meta) metas.push(meta);
+  // Ranked hits matching the FTS query, best first, each already resolved to its
+  // active conversation row and carrying the body excerpt when the match was in
+  // the conversation body.
+  //
+  // Filters are passed down into SQL rather than applied to the result: with a
+  // wide corpus, filtering an already-LIMITed list drops conversations that
+  // would have matched.
+  searchHits(query, limit, filters = {}) {
+    const hits = [];
+    for (const hit of this.fts.search(query, limit, filters)) {
+      const meta = this.conversations.getBySourcePath(hit.sourcePath);
+      if (meta) hits.push({ meta, body: hit.body });
     }
-    return metas;
+    return hits;
+  }
+  // Empty-query listing, mirroring the in-memory indexer's behavior.
+  recentMetas(limit) {
+    return this.conversations.recent(limit);
   }
   getProjects() {
     return this.conversations.distinctProjects();
@@ -2750,6 +3117,10 @@ var DEFAULT_CONFIG_PATH = "~/.config/threadbase-scanner";
 function defaultDbPath() {
   return process.env.TB_SCANNER_DB ?? (0, import_path11.join)((0, import_os2.homedir)(), ".config", "threadbase-scanner", "index.db");
 }
+function capForMemory(doc, max) {
+  const combined = combineSearchContent(doc);
+  return combined.length > max ? combined.slice(-max) : combined;
+}
 var ConversationScanner = class {
   metadataCache = /* @__PURE__ */ new Map();
   // Parsed conversations plus (persistent claude-code entries only) the resume
@@ -2761,6 +3132,10 @@ var ConversationScanner = class {
   sessionIdIndex = /* @__PURE__ */ new Map();
   projects = /* @__PURE__ */ new Set();
   indexer = new SearchIndexer();
+  // Search body per conversation for the in-memory path, already tail-capped to
+  // the content tier. Survives scan() so a statCache hit — which skips the parse
+  // entirely — can still index a body rather than an empty string.
+  searchContents = /* @__PURE__ */ new Map();
   // Tier the most recent scan() ran with, so refreshFile() re-parses a single
   // file at the same content depth. Defaults to the standard tier.
   lastTier = resolveTier("standard");
@@ -2921,34 +3296,42 @@ var ConversationScanner = class {
               try {
                 const s = (0, import_fs11.statSync)(filePath);
                 if (s.mtimeMs === cached.stat.mtimeMs && s.size === cached.stat.size) {
-                  return cached.meta;
+                  return {
+                    meta: cached.meta,
+                    searchContent: this.searchContents.get(cached.meta.id)
+                  };
                 }
               } catch {
               }
             }
           }
           try {
-            const meta = await parseMetaWithProvider(provider, filePath, account, tier);
+            let doc = emptySearchDocument();
+            const meta = await parseMetaWithProvider(provider, filePath, account, tier, (entry) => {
+              doc = appendSearchDelta(doc, extractSearchDelta(entry));
+            });
             if (meta && meta.gitBranch === null && meta.projectPath) {
               meta.gitBranch = resolveGitBranch(meta.projectPath);
             }
-            return meta;
+            return { meta, searchContent: capForMemory(doc, tier.snippetMax) };
           } catch (err) {
             parseFailures++;
             log.warn({ filePath, account, provider: provider.name, err }, "scan: parse threw");
-            return null;
+            return { meta: null, searchContent: void 0 };
           }
         })
       );
       const batchMetas = [];
-      for (const meta of results) {
+      for (const { meta, searchContent } of results) {
         if (meta && meta.messageCount > 0) {
           this.metadataCache.set(meta.id, meta);
           this.addToSessionIndex(meta);
           this.projects.add(meta.projectPath);
           allMetas.push(meta);
           batchMetas.push(meta);
-          this.indexer.addDocument(meta);
+          const content = searchContent ?? this.searchContents.get(meta.id) ?? "";
+          this.searchContents.set(meta.id, content);
+          this.indexer.addDocument(meta, content);
         }
       }
       if (batchMetas.length > 0) {
@@ -2985,12 +3368,29 @@ var ConversationScanner = class {
         const activeProfiles = profiles.filter((p) => p.enabled && p.scanHistory !== false);
         await engine.indexAll(activeProfiles, { ...options, limit: void 0, offset: void 0 });
       }
-      const metas = engine.searchMetas(query, (options.limit ?? 50) * 2);
-      results = query.trim() ? metas.map((meta) => ({ meta, score: 1, matches: generateMatches(meta, query) })) : metas.map((meta) => ({
-        meta,
-        score: 1,
-        matches: [{ field: "timestamp", snippet: meta.preview }]
-      }));
+      if (query.trim()) {
+        const want = (options.limit ?? 50) + (options.offset ?? 0);
+        results = engine.searchHits(query, want, {
+          account: options.account,
+          provider: options.provider,
+          project: options.project,
+          since: options.since ? parseSinceCutoff(options.since) : void 0,
+          include: options.include
+        }).map(({ meta, body }) => ({
+          meta,
+          score: 1,
+          matches: body ? [
+            { field: CONTENT_FIELD, snippet: body.snippet, highlights: body.highlights },
+            ...generateMatches(meta, query).filter((m) => m.field !== "preview")
+          ] : generateMatches(meta, query)
+        }));
+      } else {
+        results = engine.recentMetas((options.limit ?? 50) + (options.offset ?? 0)).map((meta) => ({
+          meta,
+          score: 1,
+          matches: [{ field: "timestamp", snippet: meta.preview }]
+        }));
+      }
     } else {
       if (this.indexer.getDocumentCount() === 0) {
         log.debug("search: index empty, triggering scan");
@@ -3184,8 +3584,11 @@ var ConversationScanner = class {
     const previous = this.metadataCache.get(filePath);
     const resolvedAccount = account ?? previous?.account ?? "default";
     let meta = null;
+    let searchDoc = emptySearchDocument();
     try {
-      meta = await parseMeta(filePath, resolvedAccount, this.lastTier);
+      meta = await parseMeta(filePath, resolvedAccount, this.lastTier, (entry) => {
+        searchDoc = appendSearchDelta(searchDoc, extractSearchDelta(entry));
+      });
     } catch (err) {
       log.warn({ filePath, err }, "refreshFile: parseMeta threw");
       meta = null;
@@ -3202,6 +3605,7 @@ var ConversationScanner = class {
         this.metadataCache.delete(previous.id);
         this.removeFromSessionIndex(previous);
         this.indexer.removeDocument(previous.id);
+        this.searchContents.delete(previous.id);
       }
       log.debug({ filePath }, "refreshFile: dropped (no parseable messages)");
       return null;
@@ -3211,10 +3615,12 @@ var ConversationScanner = class {
     this.metadataCache.set(meta.id, meta);
     this.addToSessionIndex(meta);
     this.projects.add(meta.projectPath);
+    const searchContent = capForMemory(searchDoc, this.lastTier.snippetMax);
+    this.searchContents.set(meta.id, searchContent);
     if (previous) {
-      this.indexer.updateDocument(meta);
+      this.indexer.updateDocument(meta, searchContent);
     } else {
-      this.indexer.addDocument(meta);
+      this.indexer.addDocument(meta, searchContent);
     }
     log.debug(
       { filePath, messageCount: meta.messageCount },