@threadbase-sh/scanner 0.8.4 → 0.9.1

package/dist/index.d.cts

@@ -83,6 +83,11 @@ interface ScanOptions {
         stat: FileStatEntry;
         meta: ConversationMeta;
     }>;
+    /** Persistent mode only: bypass the dir-mtime discovery gate and force a
+     *  full glob of every profile's projectsDir, ignoring stored watermarks.
+     *  Use for an explicit user-triggered refresh that shouldn't trust the
+     *  gate's optimization. */
+    fullRescan?: boolean;
 }
 interface ScanResult {
     conversations: ConversationMeta[] | TreeConversation[] | GroupedConversations;

package/dist/index.d.ts

@@ -83,6 +83,11 @@ interface ScanOptions {
         stat: FileStatEntry;
         meta: ConversationMeta;
     }>;
+    /** Persistent mode only: bypass the dir-mtime discovery gate and force a
+     *  full glob of every profile's projectsDir, ignoring stored watermarks.
+     *  Use for an explicit user-triggered refresh that shouldn't trust the
+     *  gate's optimization. */
+    fullRescan?: boolean;
 }
 interface ScanResult {
     conversations: ConversationMeta[] | TreeConversation[] | GroupedConversations;

package/dist/index.js

@@ -1215,33 +1215,33 @@ function fingerprint(filePath, size) {
   return hash.digest("hex");
 }
 function classify(filePath, existing) {
-  let stat3;
+  let stat4;
   try {
     const s = statSync(filePath);
-    stat3 = { size: s.size, mtimeMs: s.mtimeMs };
+    stat4 = { size: s.size, mtimeMs: s.mtimeMs };
   } catch {
     return { change: "vanished" };
   }
   if (!existing || existing.status !== "active" || existing.last_indexed_offset === 0) {
-    return { change: "reindex", stat: stat3 };
+    return { change: "reindex", stat: stat4 };
   }
-  if (stat3.size < existing.last_indexed_offset) {
-    return { change: "reindex", stat: stat3 };
+  if (stat4.size < existing.last_indexed_offset) {
+    return { change: "reindex", stat: stat4 };
   }
-  if (stat3.size === existing.size_bytes && stat3.mtimeMs === existing.mtime_ms) {
-    return { change: "unchanged", stat: stat3 };
+  if (stat4.size === existing.size_bytes && stat4.mtimeMs === existing.mtime_ms) {
+    return { change: "unchanged", stat: stat4 };
   }
-  if (stat3.size === existing.last_indexed_offset) {
-    const fp = fingerprint(filePath, stat3.size);
+  if (stat4.size === existing.last_indexed_offset) {
+    const fp = fingerprint(filePath, stat4.size);
     if (existing.content_fingerprint && fp !== existing.content_fingerprint) {
-      return { change: "reindex", stat: stat3 };
+      return { change: "reindex", stat: stat4 };
     }
-    return { change: "unchanged", stat: stat3 };
+    return { change: "unchanged", stat: stat4 };
   }
   if (existing.content_fingerprint && existing.size_bytes === existing.last_indexed_offset && fingerprint(filePath, existing.last_indexed_offset) === existing.content_fingerprint) {
-    return { change: "appended", stat: stat3 };
+    return { change: "appended", stat: stat4 };
   }
-  return { change: "reindex", stat: stat3 };
+  return { change: "reindex", stat: stat4 };
 }
 
 // src/providers/parse.ts
@@ -1297,7 +1297,7 @@ import { mkdirSync } from "fs";
 import { dirname as dirname3 } from "path";
 
 // src/persistent/schema.ts
-var SCHEMA_VERSION = 3;
+var SCHEMA_VERSION = 4;
 var SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS conversation_files (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1431,6 +1431,24 @@ CREATE TABLE IF NOT EXISTS message_checkpoints (
 
 CREATE INDEX IF NOT EXISTS idx_message_checkpoints_lookup
   ON message_checkpoints(source_path, message_index);
+
+-- Per-directory mtime watermark for the discovery dir-mtime gate (skips the
+-- glob for a directory whose file/subdir SET hasn't changed; never skips
+-- per-file classify()). One row for a profile's projectsDir itself (parent_root
+-- IS NULL) and one row per immediate project subdirectory discovered under it
+-- (parent_root = the projectsDir path). has_nested marks a project dir known to
+-- contain files below its own top level (e.g. subagents/) \u2014 those always
+-- re-glob regardless of mtime, since a project-dir-level watermark can't see a
+-- change two levels down.
+CREATE TABLE IF NOT EXISTS scanned_dirs (
+  path TEXT PRIMARY KEY,
+  parent_root TEXT,
+  mtime_ms INTEGER NOT NULL,
+  has_nested INTEGER NOT NULL DEFAULT 0,
+  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_scanned_dirs_parent_root ON scanned_dirs(parent_root);
 `;
 
 // src/persistent/migrations.ts
@@ -1481,6 +1499,82 @@ function openDatabase(dbPath) {
   return db;
 }
 
+// src/persistent/dir-watermark.ts
+import { readdir, stat as stat3 } from "fs/promises";
+var FULL_RECONCILE_EVERY_N_SCANS = 20;
+async function discoverJsonlFilesGated(dirs, files, scannedDirs, options = {}) {
+  const log = getLogger();
+  if (options.fullRescan) {
+    return discoverJsonlFiles(dirs);
+  }
+  const results = [];
+  for (const rawDir of dirs) {
+    const { account } = rawDir;
+    const projectsDir = rawDir.projectsDir.replace(/\\/g, "/");
+    const resolved = await resolveProjectDirs(projectsDir, scannedDirs);
+    if (resolved === null) continue;
+    for (const projectDir of resolved.entries) {
+      let dirStat;
+      try {
+        dirStat = await stat3(projectDir);
+      } catch {
+        scannedDirs.remove(projectDir);
+        continue;
+      }
+      const watermark = scannedDirs.get(projectDir);
+      const canReuse = watermark !== void 0 && watermark.mtime_ms === dirStat.mtimeMs && watermark.has_nested === 0;
+      if (canReuse) {
+        for (const row of files.activePathsByParentDir(projectDir)) {
+          results.push({ filePath: row.absolute_path, account: row.account });
+        }
+        continue;
+      }
+      const found = await discoverJsonlFiles([{ projectsDir: projectDir, account }]);
+      const hasNested = found.some((f) => dirnameOf(f.filePath) !== projectDir);
+      scannedDirs.upsert(projectDir, projectsDir, dirStat.mtimeMs, hasNested);
+      results.push(...found);
+    }
+    if (resolved.commitRoot) {
+      scannedDirs.upsert(projectsDir, null, resolved.commitRoot.mtimeMs, false);
+      const seen = new Set(resolved.entries);
+      for (const known of scannedDirs.childrenOf(projectsDir)) {
+        if (!seen.has(known.path)) scannedDirs.remove(known.path);
+      }
+    }
+  }
+  log.debug(
+    { totalFiles: results.length, dirs: dirs.length },
+    "dir-watermark: gated discovery complete"
+  );
+  return results;
+}
+async function resolveProjectDirs(projectsDir, scannedDirs) {
+  let rootStat;
+  try {
+    rootStat = await stat3(projectsDir);
+  } catch {
+    return null;
+  }
+  const rootWatermark = scannedDirs.get(projectsDir);
+  if (rootWatermark !== void 0 && rootWatermark.mtime_ms === rootStat.mtimeMs) {
+    return { entries: scannedDirs.childrenOf(projectsDir).map((row) => row.path) };
+  }
+  let entries;
+  try {
+    entries = (await readdir(projectsDir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => joinPath(projectsDir, e.name));
+  } catch {
+    return null;
+  }
+  return { entries, commitRoot: { mtimeMs: rootStat.mtimeMs } };
+}
+function dirnameOf(filePath) {
+  const idx = filePath.lastIndexOf("/");
+  return idx === -1 ? filePath : filePath.slice(0, idx);
+}
+function joinPath(dir, name) {
+  return dir.endsWith("/") ? `${dir}${name}` : `${dir}/${name}`;
+}
+
 // src/persistent/jsonl-tail-reader.ts
 import { createReadStream as createReadStream4 } from "fs";
 async function tailReduce(filePath, startOffset, startLine, state, tier) {
@@ -1687,6 +1781,14 @@ var ConversationFilesRepo = class {
     const rows = this.db.prepare("SELECT absolute_path FROM conversation_files WHERE status != 'deleted'").all();
     return rows.map((r) => r.absolute_path);
   }
+  // Active files whose immediate parent is exactly parentDir (no nested
+  // subdirectories). Backs the dir-mtime gate's reuse path: a project dir with
+  // an unchanged mtime and no nested files can skip the glob entirely.
+  activePathsByParentDir(parentDir) {
+    return this.db.prepare(
+      "SELECT absolute_path, account FROM conversation_files WHERE parent_dir = ? AND status != 'deleted'"
+    ).all(parentDir);
+  }
 };
 
 // src/persistent/repositories/conversations.repo.ts
@@ -1935,6 +2037,36 @@ function toMatchQuery(query) {
   return terms.map((t) => `"${t}"*`).join(" AND ");
 }
 
+// src/persistent/repositories/scanned-dirs.repo.ts
+var ScannedDirsRepo = class {
+  constructor(db) {
+    this.db = db;
+  }
+  db;
+  get(path) {
+    return this.db.prepare("SELECT * FROM scanned_dirs WHERE path = ?").get(path);
+  }
+  // Known project subdirectories under a root, path ascending (stable order).
+  childrenOf(parentRoot) {
+    return this.db.prepare("SELECT * FROM scanned_dirs WHERE parent_root = ? ORDER BY path ASC").all(parentRoot);
+  }
+  upsert(path, parentRoot, mtimeMs, hasNested) {
+    this.db.prepare(
+      `INSERT INTO scanned_dirs (path, parent_root, mtime_ms, has_nested, updated_at)
+         VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
+         ON CONFLICT(path) DO UPDATE SET
+           parent_root = excluded.parent_root,
+           mtime_ms = excluded.mtime_ms,
+           has_nested = excluded.has_nested,
+           updated_at = CURRENT_TIMESTAMP`
+    ).run(path, parentRoot, mtimeMs, hasNested ? 1 : 0);
+  }
+  // Drop a project subdirectory's watermark row (it vanished from disk).
+  remove(path) {
+    this.db.prepare("DELETE FROM scanned_dirs WHERE path = ?").run(path);
+  }
+};
+
 // src/persistent/index-engine.ts
 var BATCH_SIZE = 12;
 var PersistentEngine = class {
@@ -1943,15 +2075,22 @@ var PersistentEngine = class {
   conversations;
   fts;
   checkpoints;
+  scannedDirs;
   // When true, write a portable <file>.idx.json sidecar next to each indexed
   // JSONL. Off by default.
   sidecar;
+  // Counts indexAll() passes so the dir-mtime gate's full-reconcile backstop
+  // (FULL_RECONCILE_EVERY_N_SCANS) can fire periodically. In-memory only: a
+  // restart just means the first few post-restart scans don't force an early
+  // backstop pass, which is harmless (watermarks themselves persist in the DB).
+  scanCount = 0;
   constructor(dbPath, options = {}) {
     this.db = openDatabase(dbPath);
     this.files = new ConversationFilesRepo(this.db);
     this.conversations = new ConversationsRepo(this.db);
     this.fts = new FtsRepo(this.db);
     this.checkpoints = new CheckpointsRepo(this.db);
+    this.scannedDirs = new ScannedDirsRepo(this.db);
     this.sidecar = options.sidecar ?? false;
   }
   close() {
@@ -1970,7 +2109,12 @@ var PersistentEngine = class {
         projectsDir: getProjectsDir(p),
         account: p.id
       }));
-      for (const f of await discoverJsonlFiles(configDirs)) discovered.push(f);
+      this.scanCount++;
+      const forceFullGlob = options.fullRescan === true || this.scanCount % FULL_RECONCILE_EVERY_N_SCANS === 0;
+      const gated = await discoverJsonlFilesGated(configDirs, this.files, this.scannedDirs, {
+        fullRescan: forceFullGlob
+      });
+      for (const f of gated) discovered.push(f);
     }
     const codex = new CodexCliProvider();
     if (enabled.includes(CODEX_CLI_PROVIDER) && (options.codexRoots?.length ?? 0) > 0) {
@@ -2025,8 +2169,8 @@ var PersistentEngine = class {
     const log = getLogger();
     const tier = resolveTier(tierName, customTiers);
     const existing = this.files.getByPath(filePath);
-    const { change, stat: stat3 } = classify(filePath, existing);
-    if (change === "vanished" || !stat3) {
+    const { change, stat: stat4 } = classify(filePath, existing);
+    if (change === "vanished" || !stat4) {
       this.markDeleted(filePath);
       return null;
     }
@@ -2034,7 +2178,7 @@ var PersistentEngine = class {
       return this.conversations.getBySourcePath(filePath);
     }
     if (provider && provider.name !== CLAUDE_CODE_PROVIDER) {
-      return this.indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch);
+      return this.indexFileWithProvider(provider, filePath, account, tier, stat4, resolveGitBranch);
     }
     const resume = change === "appended" && !force && existing?.reducer_state;
     const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
@@ -2053,15 +2197,15 @@ var PersistentEngine = class {
       return null;
     }
     meta.gitBranch = resolveGitBranch(meta.projectPath);
-    const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
+    const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
     const fileId = this.files.ensure(filePath, account);
     const upsert = this.db.transaction(() => {
       this.conversations.upsert(fileId, meta, state.pageMessageCount);
       this.fts.upsert(meta);
       this.checkpoints.remove(filePath);
       this.files.updateCursor(fileId, {
-        sizeBytes: stat3.size,
-        mtimeMs: stat3.mtimeMs,
+        sizeBytes: stat4.size,
+        mtimeMs: stat4.mtimeMs,
         // Advance only to the last fully-parsed line; a trailing partial line
         // is left for the next pass.
         offset: result.newOffset,
@@ -2078,8 +2222,8 @@ var PersistentEngine = class {
         buildSidecar(
           meta,
           {
-            sizeBytes: stat3.size,
-            mtimeMs: stat3.mtimeMs,
+            sizeBytes: stat4.size,
+            mtimeMs: stat4.mtimeMs,
             offset: result.newOffset,
             line: result.newLine
           },
@@ -2098,7 +2242,7 @@ var PersistentEngine = class {
   // the Threadbase path uses. The cursor records size/mtime (and offset = size)
   // so the next pass classifies an unchanged file as "unchanged" and skips it;
   // any change reparses from 0 again. No reducer_state is persisted.
-  async indexFileWithProvider(provider, filePath, account, tier, stat3, resolveGitBranch) {
+  async indexFileWithProvider(provider, filePath, account, tier, stat4, resolveGitBranch) {
     const log = getLogger();
     const meta = await parseMetaWithProvider(provider, filePath, account, tier);
     if (!meta) {
@@ -2108,18 +2252,18 @@ var PersistentEngine = class {
     if (meta.gitBranch === null && meta.projectPath) {
       meta.gitBranch = resolveGitBranch(meta.projectPath);
     }
-    const fp = stat3.size > 0 ? fingerprint(filePath, stat3.size) : null;
+    const fp = stat4.size > 0 ? fingerprint(filePath, stat4.size) : null;
     const fileId = this.files.ensure(filePath, account);
     const upsert = this.db.transaction(() => {
       this.conversations.upsert(fileId, meta, meta.messageCount);
       this.fts.upsert(meta);
       this.checkpoints.remove(filePath);
       this.files.updateCursor(fileId, {
-        sizeBytes: stat3.size,
-        mtimeMs: stat3.mtimeMs,
+        sizeBytes: stat4.size,
+        mtimeMs: stat4.mtimeMs,
         // offset = size marks the file fully consumed (non-zero so the next pass
         // can classify it "unchanged"); no resumable reducer state is kept.
-        offset: stat3.size,
+        offset: stat4.size,
         line: 0,
         reducerState: null,
         fingerprint: fp,