cogpit-memory 0.1.7 → 0.1.9

package/dist/cli.js

@@ -951,19 +951,32 @@ function extractSessionMetadata(messages) {
   }
   return meta;
 }
-function parseSession(jsonlText) {
+function emptyStats(turnCount) {
+  return {
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    totalCacheCreationTokens: 0,
+    totalCacheReadTokens: 0,
+    totalCostUSD: 0,
+    toolCallCounts: {},
+    errorCount: 0,
+    totalDurationMs: 0,
+    turnCount
+  };
+}
+function parseSession(jsonlText, opts) {
   if (isCodexSessionText(jsonlText)) {
     return parseCodexSession(jsonlText);
   }
   const rawMessages = parseLines(jsonlText);
   const metadata = extractSessionMetadata(rawMessages);
   const turns = buildTurns(rawMessages);
-  const stats = computeStats(turns);
+  const stats = opts?.skipStats ? emptyStats(turns.length) : computeStats(turns);
   return {
     ...metadata,
     turns,
     stats,
-    rawMessages,
+    rawMessages: opts?.skipStats ? [] : rawMessages,
     agentKind: "claude"
   };
 }
@@ -974,6 +987,10 @@ function getUserMessageText(content) {
 }
 
 // src/lib/search-index.ts
+var MAX_CONTENT_LEN = 4096;
+function truncContent(text) {
+  return text.length > MAX_CONTENT_LEN ? text.slice(0, MAX_CONTENT_LEN) : text;
+}
 var SearchIndex = class {
   db;
   dbPath;
@@ -1007,7 +1024,7 @@ var SearchIndex = class {
         source_file,
         location,
         content,
-        tokenize = 'trigram'
+        tokenize = 'unicode61'
       )`);
     }
   }
@@ -1034,6 +1051,56 @@ var SearchIndex = class {
       lastUpdate: this._lastUpdate
     };
   }
+  /** Index tool call inputs and results under the given location prefix. */
+  insertToolCalls(insert, sessionId, filePath, prefix, toolCalls) {
+    for (const tc of toolCalls) {
+      const inputStr = JSON.stringify(tc.input);
+      if (inputStr && inputStr !== "{}") {
+        insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/input`, truncContent(inputStr));
+      }
+      if (tc.result) {
+        insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/result`, truncContent(tc.result));
+      }
+    }
+  }
+  /**
+   * Insert all searchable content from a parsed session into the FTS5 index.
+   * Shared by both `indexFile` (single-file) and `buildFull` (batch).
+   */
+  insertSessionContent(insert, sessionId, filePath, session) {
+    for (let i = 0; i < session.turns.length; i++) {
+      const turn = session.turns[i];
+      const prefix = `turn/${i}`;
+      const userText = getUserMessageText(turn.userMessage);
+      if (userText.trim()) {
+        insert.run(sessionId, filePath, `${prefix}/userMessage`, truncContent(userText));
+      }
+      const assistantJoined = turn.assistantText.join("\n\n").trim();
+      if (assistantJoined) {
+        insert.run(sessionId, filePath, `${prefix}/assistantMessage`, truncContent(assistantJoined));
+      }
+      const thinkingText = turn.thinking.filter((t) => t.thinking && t.thinking.length > 0).map((t) => t.thinking).join("\n\n").trim();
+      if (thinkingText) {
+        insert.run(sessionId, filePath, `${prefix}/thinking`, truncContent(thinkingText));
+      }
+      this.insertToolCalls(insert, sessionId, filePath, prefix, turn.toolCalls);
+      for (const sa of turn.subAgentActivity) {
+        const saPrefix = `agent/${sa.agentId}`;
+        const saText = sa.text.join("\n\n").trim();
+        if (saText) {
+          insert.run(sessionId, filePath, `${saPrefix}/assistantMessage`, truncContent(saText));
+        }
+        const saThinking = sa.thinking.filter((t) => t.length > 0).join("\n\n").trim();
+        if (saThinking) {
+          insert.run(sessionId, filePath, `${saPrefix}/thinking`, truncContent(saThinking));
+        }
+        this.insertToolCalls(insert, sessionId, filePath, saPrefix, sa.toolCalls);
+      }
+      if (turn.compactionSummary) {
+        insert.run(sessionId, filePath, `${prefix}/compactionSummary`, truncContent(turn.compactionSummary));
+      }
+    }
+  }
   /**
    * Parse a JSONL file and insert all searchable content into the FTS5 index.
    * Idempotent: deletes old data for the file before re-indexing.
@@ -1047,7 +1114,7 @@ var SearchIndex = class {
       mtimeMs = (0, import_node_fs.statSync)(filePath).mtimeMs;
     }
     const content = (0, import_node_fs.readFileSync)(filePath, "utf-8");
-    const session = parseSession(content);
+    const session = parseSession(content, { skipStats: true });
     const isSubagent = opts?.isSubagent ? 1 : 0;
     const parentSessionId = opts?.parentSessionId ?? null;
     const insert = this.db.prepare(
@@ -1065,54 +1132,7 @@ var SearchIndex = class {
     const txn = this.db.transaction(() => {
       deleteContent.run(filePath);
       deleteFile.run(filePath);
-      for (let i = 0; i < session.turns.length; i++) {
-        const turn = session.turns[i];
-        const prefix = `turn/${i}`;
-        const userText = getUserMessageText(turn.userMessage);
-        if (userText.trim()) {
-          insert.run(sessionId, filePath, `${prefix}/userMessage`, userText);
-        }
-        const assistantJoined = turn.assistantText.join("\n\n").trim();
-        if (assistantJoined) {
-          insert.run(sessionId, filePath, `${prefix}/assistantMessage`, assistantJoined);
-        }
-        const thinkingText = turn.thinking.filter((t) => t.thinking && t.thinking.length > 0).map((t) => t.thinking).join("\n\n").trim();
-        if (thinkingText) {
-          insert.run(sessionId, filePath, `${prefix}/thinking`, thinkingText);
-        }
-        for (const tc of turn.toolCalls) {
-          const inputStr = JSON.stringify(tc.input);
-          if (inputStr && inputStr !== "{}") {
-            insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/input`, inputStr);
-          }
-          if (tc.result) {
-            insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/result`, tc.result);
-          }
-        }
-        for (const sa of turn.subAgentActivity) {
-          const saPrefix = `agent/${sa.agentId}`;
-          const saText = sa.text.join("\n\n").trim();
-          if (saText) {
-            insert.run(sessionId, filePath, `${saPrefix}/assistantMessage`, saText);
-          }
-          const saThinking = sa.thinking.filter((t) => t.length > 0).join("\n\n").trim();
-          if (saThinking) {
-            insert.run(sessionId, filePath, `${saPrefix}/thinking`, saThinking);
-          }
-          for (const tc of sa.toolCalls) {
-            const inputStr = JSON.stringify(tc.input);
-            if (inputStr && inputStr !== "{}") {
-              insert.run(sessionId, filePath, `${saPrefix}/toolCall/${tc.id}/input`, inputStr);
-            }
-            if (tc.result) {
-              insert.run(sessionId, filePath, `${saPrefix}/toolCall/${tc.id}/result`, tc.result);
-            }
-          }
-        }
-        if (turn.compactionSummary) {
-          insert.run(sessionId, filePath, `${prefix}/compactionSummary`, turn.compactionSummary);
-        }
-      }
+      this.insertSessionContent(insert, sessionId, filePath, session);
       insertFile.run(filePath, mtimeMs, sessionId, isSubagent, parentSessionId);
     });
     txn();
@@ -1121,7 +1141,7 @@ var SearchIndex = class {
   /**
    * Query the FTS5 index and return structured search results.
    *
-   * - FTS5 trigram tokenizer is case-insensitive by default.
+   * - FTS5 unicode61 tokenizer is case-insensitive by default.
    * - When `caseSensitive` is true, a post-filter checks the original query
    *   against the snippet text (exact case match).
    * - When `maxAgeMs` is provided, only files whose mtime in `indexed_files`
@@ -1163,7 +1183,7 @@ var SearchIndex = class {
       location: row.location,
       snippet: row.snippet,
       matchCount: 1
-      // FTS5 trigram doesn't expose per-row match count; 1 = "at least one match"
+      // FTS5 doesn't expose per-row match count; 1 = "at least one match"
     }));
     if (caseSensitive) {
       hits = hits.filter((h) => h.snippet.includes(query));
@@ -1202,19 +1222,67 @@ var SearchIndex = class {
    * Structure: projectsDir/{projectName}/{sessionId}.jsonl
    * Subagents:  projectsDir/{projectName}/{sessionId}/subagents/agent-{id}.jsonl
    *
+   * Optimized: discovers all files first, then processes them in a single
+   * SQLite transaction with pre-prepared statements. This avoids the overhead
+   * of 3000+ individual transactions (each forcing a disk sync).
+   *
    * Stores `projectsDir` as a class field so `rebuild()` can reuse it.
    */
   buildFull(projectsDir) {
     this.projectsDir = projectsDir;
-    this.db.exec("DELETE FROM search_content");
-    this.db.exec("DELETE FROM indexed_files");
-    this.indexProjectsDir(projectsDir);
+    this.db.close();
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath);
+    } catch {
+    }
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath + "-wal");
+    } catch {
+    }
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath + "-shm");
+    } catch {
+    }
+    this.db = new Database(this.dbPath);
+    this.db.exec("PRAGMA journal_mode = WAL");
+    this.db.exec("PRAGMA synchronous = OFF");
+    this.db.exec("PRAGMA cache_size = -64000");
+    this.db.exec("PRAGMA temp_store = MEMORY");
+    this.db.exec("PRAGMA mmap_size = 268435456");
+    this.initSchema();
+    const files = [];
+    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
+      files.push({ path: filePath, sessionId, mtimeMs, isSubagent, parentSessionId });
+    });
+    const insert = this.db.prepare(
+      "INSERT INTO search_content (session_id, source_file, location, content) VALUES (?, ?, ?, ?)"
+    );
+    const insertFile = this.db.prepare(
+      "INSERT OR REPLACE INTO indexed_files (file_path, mtime_ms, session_id, is_subagent, parent_session_id) VALUES (?, ?, ?, ?, ?)"
+    );
+    const txn = this.db.transaction(() => {
+      for (const file of files) {
+        try {
+          const content = (0, import_node_fs.readFileSync)(file.path, "utf-8");
+          const session = parseSession(content, { skipStats: true });
+          this.insertSessionContent(insert, file.sessionId, file.path, session);
+          insertFile.run(file.path, file.mtimeMs, file.sessionId, file.isSubagent ? 1 : 0, file.parentSessionId);
+        } catch {
+        }
+      }
+    });
+    txn();
+    this.db.exec("PRAGMA synchronous = NORMAL");
     this._lastFullBuild = (/* @__PURE__ */ new Date()).toISOString();
     this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
   }
   /**
    * Incrementally re-index only files whose mtime has changed since last index.
    * New files (not in indexed_files) are always indexed.
+   *
+   * WARNING: This walks ALL files under projectsDir and stats each one.
+   * On large session stores (3000+ files) this can take minutes.
+   * Prefer `updateRecent()` for CLI search paths.
    */
   updateStale(projectsDir) {
     this.projectsDir = projectsDir;
@@ -1241,6 +1309,46 @@ var SearchIndex = class {
       this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
     }
   }
+  /**
+   * Lightweight incremental update for CLI search paths.
+   *
+   * Still walks and stats all files via `discoverFiles`, but skips DB lookups
+   * for files with mtime <= the high-water mark that are already indexed.
+   * Caps re-indexing to `maxFiles` to prevent blocking on large backlogs
+   * (run `index rebuild` for a full catch-up).
+   */
+  updateRecent(projectsDir, maxFiles = 50) {
+    this.projectsDir = projectsDir;
+    const row = this.db.prepare(
+      "SELECT MAX(mtime_ms) as max_mtime FROM indexed_files"
+    ).get();
+    const highWater = row?.max_mtime ?? 0;
+    const getIndexed = this.db.prepare(
+      "SELECT mtime_ms FROM indexed_files WHERE file_path = ?"
+    );
+    const filesToIndex = [];
+    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
+      if (mtimeMs <= highWater) {
+        const existing = getIndexed.get(filePath);
+        if (existing && existing.mtime_ms >= mtimeMs) return;
+      }
+      filesToIndex.push({ path: filePath, sessionId, mtimeMs, isSubagent, parentSessionId });
+    });
+    filesToIndex.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    const batch = filesToIndex.slice(0, maxFiles);
+    for (const file of batch) {
+      try {
+        this.indexFile(file.path, file.sessionId, file.mtimeMs, {
+          isSubagent: file.isSubagent,
+          parentSessionId: file.parentSessionId
+        });
+      } catch {
+      }
+    }
+    if (batch.length > 0) {
+      this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
+    }
+  }
   /**
    * Re-run buildFull using the previously stored projectsDir.
    * No-op if projectsDir was never set.
@@ -1249,18 +1357,6 @@ var SearchIndex = class {
     if (!this.projectsDir) return;
     this.buildFull(this.projectsDir);
   }
-  /**
-   * Walk all project directories under `projectsDir` and index every discovered
-   * JSONL file (both sessions and subagents).
-   */
-  indexProjectsDir(projectsDir) {
-    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
-      try {
-        this.indexFile(filePath, sessionId, mtimeMs, { isSubagent, parentSessionId });
-      } catch {
-      }
-    });
-  }
   /**
    * Walk the projects directory tree and invoke `callback` for every JSONL file.
    *
@@ -1276,29 +1372,23 @@ var SearchIndex = class {
   discoverFiles(projectsDir, callback) {
     let entries;
     try {
-      entries = (0, import_node_fs.readdirSync)(projectsDir);
+      entries = (0, import_node_fs.readdirSync)(projectsDir, { withFileTypes: true });
     } catch {
       return;
     }
-    for (const projectName of entries) {
-      if (projectName === "memory") continue;
-      const projectDir = (0, import_node_path.join)(projectsDir, projectName);
-      try {
-        const s = (0, import_node_fs.statSync)(projectDir);
-        if (!s.isDirectory()) continue;
-      } catch {
-        continue;
-      }
+    for (const entry of entries) {
+      if (entry.name === "memory" || !entry.isDirectory()) continue;
+      const projectDir = (0, import_node_path.join)(projectsDir, entry.name);
       let files;
       try {
-        files = (0, import_node_fs.readdirSync)(projectDir);
+        files = (0, import_node_fs.readdirSync)(projectDir, { withFileTypes: true });
       } catch {
         continue;
       }
       for (const file of files) {
-        if (!file.endsWith(".jsonl")) continue;
-        const filePath = (0, import_node_path.join)(projectDir, file);
-        const sessionId = (0, import_node_path.basename)(file, ".jsonl");
+        if (!file.name.endsWith(".jsonl") || !file.isFile()) continue;
+        const filePath = (0, import_node_path.join)(projectDir, file.name);
+        const sessionId = (0, import_node_path.basename)(file.name, ".jsonl");
         try {
           const s = (0, import_node_fs.statSync)(filePath);
           callback(filePath, sessionId, s.mtimeMs, false, null);
@@ -1544,11 +1634,19 @@ async function searchSessions(query, opts, searchIndex) {
   const depth = Math.min(Math.max(1, opts.depth ?? 4), 4);
   let index = searchIndex ?? null;
   let ownedIndex = false;
-  if (!index && (0, import_node_fs2.existsSync)(DEFAULT_DB_PATH)) {
+  if (!index && searchIndex === void 0) {
     try {
+      const dbExists = (0, import_node_fs2.existsSync)(DEFAULT_DB_PATH);
+      if (!dbExists) {
+        (0, import_node_fs2.mkdirSync)((0, import_node_path4.dirname)(DEFAULT_DB_PATH), { recursive: true });
+      }
       index = new SearchIndex(DEFAULT_DB_PATH);
       ownedIndex = true;
-      index.updateStale(dirs.PROJECTS_DIR);
+      if (!dbExists) {
+        index.buildFull(dirs.PROJECTS_DIR);
+      } else {
+        index.updateRecent(dirs.PROJECTS_DIR);
+      }
     } catch {
     }
   }

package/dist/index.js

@@ -967,19 +967,32 @@ function extractSessionMetadata(messages) {
   }
   return meta;
 }
-function parseSession(jsonlText) {
+function emptyStats(turnCount) {
+  return {
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    totalCacheCreationTokens: 0,
+    totalCacheReadTokens: 0,
+    totalCostUSD: 0,
+    toolCallCounts: {},
+    errorCount: 0,
+    totalDurationMs: 0,
+    turnCount
+  };
+}
+function parseSession(jsonlText, opts) {
   if (isCodexSessionText(jsonlText)) {
     return parseCodexSession(jsonlText);
   }
   const rawMessages = parseLines(jsonlText);
   const metadata = extractSessionMetadata(rawMessages);
   const turns = buildTurns(rawMessages);
-  const stats = computeStats(turns);
+  const stats = opts?.skipStats ? emptyStats(turns.length) : computeStats(turns);
   return {
     ...metadata,
     turns,
     stats,
-    rawMessages,
+    rawMessages: opts?.skipStats ? [] : rawMessages,
     agentKind: "claude"
   };
 }
@@ -1072,6 +1085,10 @@ function getUserMessageImages(content) {
 }
 
 // src/lib/search-index.ts
+var MAX_CONTENT_LEN = 4096;
+function truncContent(text) {
+  return text.length > MAX_CONTENT_LEN ? text.slice(0, MAX_CONTENT_LEN) : text;
+}
 var SearchIndex = class {
   db;
   dbPath;
@@ -1105,7 +1122,7 @@ var SearchIndex = class {
         source_file,
         location,
         content,
-        tokenize = 'trigram'
+        tokenize = 'unicode61'
       )`);
     }
   }
@@ -1132,6 +1149,56 @@ var SearchIndex = class {
       lastUpdate: this._lastUpdate
     };
   }
+  /** Index tool call inputs and results under the given location prefix. */
+  insertToolCalls(insert, sessionId, filePath, prefix, toolCalls) {
+    for (const tc of toolCalls) {
+      const inputStr = JSON.stringify(tc.input);
+      if (inputStr && inputStr !== "{}") {
+        insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/input`, truncContent(inputStr));
+      }
+      if (tc.result) {
+        insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/result`, truncContent(tc.result));
+      }
+    }
+  }
+  /**
+   * Insert all searchable content from a parsed session into the FTS5 index.
+   * Shared by both `indexFile` (single-file) and `buildFull` (batch).
+   */
+  insertSessionContent(insert, sessionId, filePath, session) {
+    for (let i = 0; i < session.turns.length; i++) {
+      const turn = session.turns[i];
+      const prefix = `turn/${i}`;
+      const userText = getUserMessageText(turn.userMessage);
+      if (userText.trim()) {
+        insert.run(sessionId, filePath, `${prefix}/userMessage`, truncContent(userText));
+      }
+      const assistantJoined = turn.assistantText.join("\n\n").trim();
+      if (assistantJoined) {
+        insert.run(sessionId, filePath, `${prefix}/assistantMessage`, truncContent(assistantJoined));
+      }
+      const thinkingText = turn.thinking.filter((t) => t.thinking && t.thinking.length > 0).map((t) => t.thinking).join("\n\n").trim();
+      if (thinkingText) {
+        insert.run(sessionId, filePath, `${prefix}/thinking`, truncContent(thinkingText));
+      }
+      this.insertToolCalls(insert, sessionId, filePath, prefix, turn.toolCalls);
+      for (const sa of turn.subAgentActivity) {
+        const saPrefix = `agent/${sa.agentId}`;
+        const saText = sa.text.join("\n\n").trim();
+        if (saText) {
+          insert.run(sessionId, filePath, `${saPrefix}/assistantMessage`, truncContent(saText));
+        }
+        const saThinking = sa.thinking.filter((t) => t.length > 0).join("\n\n").trim();
+        if (saThinking) {
+          insert.run(sessionId, filePath, `${saPrefix}/thinking`, truncContent(saThinking));
+        }
+        this.insertToolCalls(insert, sessionId, filePath, saPrefix, sa.toolCalls);
+      }
+      if (turn.compactionSummary) {
+        insert.run(sessionId, filePath, `${prefix}/compactionSummary`, truncContent(turn.compactionSummary));
+      }
+    }
+  }
   /**
    * Parse a JSONL file and insert all searchable content into the FTS5 index.
    * Idempotent: deletes old data for the file before re-indexing.
@@ -1145,7 +1212,7 @@ var SearchIndex = class {
       mtimeMs = (0, import_node_fs.statSync)(filePath).mtimeMs;
     }
     const content = (0, import_node_fs.readFileSync)(filePath, "utf-8");
-    const session = parseSession(content);
+    const session = parseSession(content, { skipStats: true });
     const isSubagent = opts?.isSubagent ? 1 : 0;
     const parentSessionId = opts?.parentSessionId ?? null;
     const insert = this.db.prepare(
@@ -1163,54 +1230,7 @@ var SearchIndex = class {
     const txn = this.db.transaction(() => {
       deleteContent.run(filePath);
       deleteFile.run(filePath);
-      for (let i = 0; i < session.turns.length; i++) {
-        const turn = session.turns[i];
-        const prefix = `turn/${i}`;
-        const userText = getUserMessageText(turn.userMessage);
-        if (userText.trim()) {
-          insert.run(sessionId, filePath, `${prefix}/userMessage`, userText);
-        }
-        const assistantJoined = turn.assistantText.join("\n\n").trim();
-        if (assistantJoined) {
-          insert.run(sessionId, filePath, `${prefix}/assistantMessage`, assistantJoined);
-        }
-        const thinkingText = turn.thinking.filter((t) => t.thinking && t.thinking.length > 0).map((t) => t.thinking).join("\n\n").trim();
-        if (thinkingText) {
-          insert.run(sessionId, filePath, `${prefix}/thinking`, thinkingText);
-        }
-        for (const tc of turn.toolCalls) {
-          const inputStr = JSON.stringify(tc.input);
-          if (inputStr && inputStr !== "{}") {
-            insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/input`, inputStr);
-          }
-          if (tc.result) {
-            insert.run(sessionId, filePath, `${prefix}/toolCall/${tc.id}/result`, tc.result);
-          }
-        }
-        for (const sa of turn.subAgentActivity) {
-          const saPrefix = `agent/${sa.agentId}`;
-          const saText = sa.text.join("\n\n").trim();
-          if (saText) {
-            insert.run(sessionId, filePath, `${saPrefix}/assistantMessage`, saText);
-          }
-          const saThinking = sa.thinking.filter((t) => t.length > 0).join("\n\n").trim();
-          if (saThinking) {
-            insert.run(sessionId, filePath, `${saPrefix}/thinking`, saThinking);
-          }
-          for (const tc of sa.toolCalls) {
-            const inputStr = JSON.stringify(tc.input);
-            if (inputStr && inputStr !== "{}") {
-              insert.run(sessionId, filePath, `${saPrefix}/toolCall/${tc.id}/input`, inputStr);
-            }
-            if (tc.result) {
-              insert.run(sessionId, filePath, `${saPrefix}/toolCall/${tc.id}/result`, tc.result);
-            }
-          }
-        }
-        if (turn.compactionSummary) {
-          insert.run(sessionId, filePath, `${prefix}/compactionSummary`, turn.compactionSummary);
-        }
-      }
+      this.insertSessionContent(insert, sessionId, filePath, session);
       insertFile.run(filePath, mtimeMs, sessionId, isSubagent, parentSessionId);
     });
     txn();
@@ -1219,7 +1239,7 @@ var SearchIndex = class {
   /**
    * Query the FTS5 index and return structured search results.
    *
-   * - FTS5 trigram tokenizer is case-insensitive by default.
+   * - FTS5 unicode61 tokenizer is case-insensitive by default.
    * - When `caseSensitive` is true, a post-filter checks the original query
    *   against the snippet text (exact case match).
    * - When `maxAgeMs` is provided, only files whose mtime in `indexed_files`
@@ -1261,7 +1281,7 @@ var SearchIndex = class {
       location: row.location,
       snippet: row.snippet,
       matchCount: 1
-      // FTS5 trigram doesn't expose per-row match count; 1 = "at least one match"
+      // FTS5 doesn't expose per-row match count; 1 = "at least one match"
     }));
     if (caseSensitive) {
       hits = hits.filter((h) => h.snippet.includes(query));
@@ -1300,19 +1320,67 @@ var SearchIndex = class {
    * Structure: projectsDir/{projectName}/{sessionId}.jsonl
    * Subagents:  projectsDir/{projectName}/{sessionId}/subagents/agent-{id}.jsonl
    *
+   * Optimized: discovers all files first, then processes them in a single
+   * SQLite transaction with pre-prepared statements. This avoids the overhead
+   * of 3000+ individual transactions (each forcing a disk sync).
+   *
    * Stores `projectsDir` as a class field so `rebuild()` can reuse it.
    */
   buildFull(projectsDir) {
     this.projectsDir = projectsDir;
-    this.db.exec("DELETE FROM search_content");
-    this.db.exec("DELETE FROM indexed_files");
-    this.indexProjectsDir(projectsDir);
+    this.db.close();
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath);
+    } catch {
+    }
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath + "-wal");
+    } catch {
+    }
+    try {
+      (0, import_node_fs.unlinkSync)(this.dbPath + "-shm");
+    } catch {
+    }
+    this.db = new Database(this.dbPath);
+    this.db.exec("PRAGMA journal_mode = WAL");
+    this.db.exec("PRAGMA synchronous = OFF");
+    this.db.exec("PRAGMA cache_size = -64000");
+    this.db.exec("PRAGMA temp_store = MEMORY");
+    this.db.exec("PRAGMA mmap_size = 268435456");
+    this.initSchema();
+    const files = [];
+    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
+      files.push({ path: filePath, sessionId, mtimeMs, isSubagent, parentSessionId });
+    });
+    const insert = this.db.prepare(
+      "INSERT INTO search_content (session_id, source_file, location, content) VALUES (?, ?, ?, ?)"
+    );
+    const insertFile = this.db.prepare(
+      "INSERT OR REPLACE INTO indexed_files (file_path, mtime_ms, session_id, is_subagent, parent_session_id) VALUES (?, ?, ?, ?, ?)"
+    );
+    const txn = this.db.transaction(() => {
+      for (const file of files) {
+        try {
+          const content = (0, import_node_fs.readFileSync)(file.path, "utf-8");
+          const session = parseSession(content, { skipStats: true });
+          this.insertSessionContent(insert, file.sessionId, file.path, session);
+          insertFile.run(file.path, file.mtimeMs, file.sessionId, file.isSubagent ? 1 : 0, file.parentSessionId);
+        } catch {
+        }
+      }
+    });
+    txn();
+    this.db.exec("PRAGMA synchronous = NORMAL");
     this._lastFullBuild = (/* @__PURE__ */ new Date()).toISOString();
     this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
   }
   /**
    * Incrementally re-index only files whose mtime has changed since last index.
    * New files (not in indexed_files) are always indexed.
+   *
+   * WARNING: This walks ALL files under projectsDir and stats each one.
+   * On large session stores (3000+ files) this can take minutes.
+   * Prefer `updateRecent()` for CLI search paths.
    */
   updateStale(projectsDir) {
     this.projectsDir = projectsDir;
@@ -1339,6 +1407,46 @@ var SearchIndex = class {
       this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
     }
   }
+  /**
+   * Lightweight incremental update for CLI search paths.
+   *
+   * Still walks and stats all files via `discoverFiles`, but skips DB lookups
+   * for files with mtime <= the high-water mark that are already indexed.
+   * Caps re-indexing to `maxFiles` to prevent blocking on large backlogs
+   * (run `index rebuild` for a full catch-up).
+   */
+  updateRecent(projectsDir, maxFiles = 50) {
+    this.projectsDir = projectsDir;
+    const row = this.db.prepare(
+      "SELECT MAX(mtime_ms) as max_mtime FROM indexed_files"
+    ).get();
+    const highWater = row?.max_mtime ?? 0;
+    const getIndexed = this.db.prepare(
+      "SELECT mtime_ms FROM indexed_files WHERE file_path = ?"
+    );
+    const filesToIndex = [];
+    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
+      if (mtimeMs <= highWater) {
+        const existing = getIndexed.get(filePath);
+        if (existing && existing.mtime_ms >= mtimeMs) return;
+      }
+      filesToIndex.push({ path: filePath, sessionId, mtimeMs, isSubagent, parentSessionId });
+    });
+    filesToIndex.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    const batch = filesToIndex.slice(0, maxFiles);
+    for (const file of batch) {
+      try {
+        this.indexFile(file.path, file.sessionId, file.mtimeMs, {
+          isSubagent: file.isSubagent,
+          parentSessionId: file.parentSessionId
+        });
+      } catch {
+      }
+    }
+    if (batch.length > 0) {
+      this._lastUpdate = (/* @__PURE__ */ new Date()).toISOString();
+    }
+  }
   /**
    * Re-run buildFull using the previously stored projectsDir.
    * No-op if projectsDir was never set.
@@ -1347,18 +1455,6 @@ var SearchIndex = class {
     if (!this.projectsDir) return;
     this.buildFull(this.projectsDir);
   }
-  /**
-   * Walk all project directories under `projectsDir` and index every discovered
-   * JSONL file (both sessions and subagents).
-   */
-  indexProjectsDir(projectsDir) {
-    this.discoverFiles(projectsDir, (filePath, sessionId, mtimeMs, isSubagent, parentSessionId) => {
-      try {
-        this.indexFile(filePath, sessionId, mtimeMs, { isSubagent, parentSessionId });
-      } catch {
-      }
-    });
-  }
   /**
    * Walk the projects directory tree and invoke `callback` for every JSONL file.
    *
@@ -1374,29 +1470,23 @@ var SearchIndex = class {
   discoverFiles(projectsDir, callback) {
     let entries;
     try {
-      entries = (0, import_node_fs.readdirSync)(projectsDir);
+      entries = (0, import_node_fs.readdirSync)(projectsDir, { withFileTypes: true });
     } catch {
       return;
     }
-    for (const projectName of entries) {
-      if (projectName === "memory") continue;
-      const projectDir = (0, import_node_path.join)(projectsDir, projectName);
-      try {
-        const s = (0, import_node_fs.statSync)(projectDir);
-        if (!s.isDirectory()) continue;
-      } catch {
-        continue;
-      }
+    for (const entry of entries) {
+      if (entry.name === "memory" || !entry.isDirectory()) continue;
+      const projectDir = (0, import_node_path.join)(projectsDir, entry.name);
       let files;
       try {
-        files = (0, import_node_fs.readdirSync)(projectDir);
+        files = (0, import_node_fs.readdirSync)(projectDir, { withFileTypes: true });
       } catch {
         continue;
       }
       for (const file of files) {
-        if (!file.endsWith(".jsonl")) continue;
-        const filePath = (0, import_node_path.join)(projectDir, file);
-        const sessionId = (0, import_node_path.basename)(file, ".jsonl");
+        if (!file.name.endsWith(".jsonl") || !file.isFile()) continue;
+        const filePath = (0, import_node_path.join)(projectDir, file.name);
+        const sessionId = (0, import_node_path.basename)(file.name, ".jsonl");
         try {
           const s = (0, import_node_fs.statSync)(filePath);
           callback(filePath, sessionId, s.mtimeMs, false, null);
@@ -1647,11 +1737,19 @@ async function searchSessions(query, opts, searchIndex) {
   const depth = Math.min(Math.max(1, opts.depth ?? 4), 4);
   let index = searchIndex ?? null;
   let ownedIndex = false;
-  if (!index && (0, import_node_fs2.existsSync)(DEFAULT_DB_PATH)) {
+  if (!index && searchIndex === void 0) {
     try {
+      const dbExists = (0, import_node_fs2.existsSync)(DEFAULT_DB_PATH);
+      if (!dbExists) {
+        (0, import_node_fs2.mkdirSync)((0, import_node_path4.dirname)(DEFAULT_DB_PATH), { recursive: true });
+      }
       index = new SearchIndex(DEFAULT_DB_PATH);
       ownedIndex = true;
-      index.updateStale(dirs.PROJECTS_DIR);
+      if (!dbExists) {
+        index.buildFull(dirs.PROJECTS_DIR);
+      } else {
+        index.updateRecent(dirs.PROJECTS_DIR);
+      }
     } catch {
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cogpit-memory",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "CLI tool for Claude Code session introspection — search, browse, and drill into past sessions",
   "main": "./dist/index.js",
   "types": "./src/index.ts",