@threadbase-sh/streamer 1.39.0 → 1.39.1

package/dist/index.cjs

@@ -8867,6 +8867,20 @@ var StreamerServer = class {
   // Set by onConversationChanged while a scan is in-flight; getScanner() does
   // a single rescan after the current one completes instead of restarting it.
   scannerStale = false;
+  // WHICH files scannerStale is about. A directory event names exactly one
+  // JSONL, and the only correct response is refreshFile() on that one file —
+  // but scannerStale alone carries no identity, so honoring it used to mean a
+  // full-tree rescan. On the non-persistent scanner this server actually runs
+  // (buildStatCache => persistent:false, see listen()), scan() opens by
+  // clearing metadataCache AND conversationLRU — so one live session appending
+  // to its own transcript threw away every OTHER conversation's parsed
+  // snapshot, and the next full fetch of an unrelated conversation re-parsed
+  // it from disk (745-2877ms on a 5MB/1112-message history) while the
+  // per-file paginated path stayed at ~20ms throughout. Populated alongside
+  // scannerStale and drained with it by takeStaleFiles(); an armed flag with
+  // an EMPTY set means "stale, source unknown" and still falls back to the
+  // full rescan.
+  staleFiles = /* @__PURE__ */ new Set();
   // Single-flight guard for the background disk reconcile: a burst of list
   // polls during active session writes shares one rescan instead of queueing
   // a full rescan per request.
@@ -9035,7 +9049,10 @@ var StreamerServer = class {
     this.directoryDebounceMs = parseDirScanDebounceEnv(process.env.THREADBASE_DIR_SCAN_DEBOUNCE_MS) ?? config.directoryScanDebounceMs ?? 1e3;
     this.markScannerStaleDebounced = debounce(() => {
       if (this.scannerReady) this.scannerStale = true;
-      else this.scanner = null;
+      else {
+        this.scanner = null;
+        this.staleFiles.clear();
+      }
     }, this.directoryDebounceMs);
     this.includeAgents = parseIncludeAgentsEnv(process.env.THREADBASE_INCLUDE_AGENTS);
     this.agentEntrypoints = parseAgentEntrypointsEnv(process.env.THREADBASE_AGENT_ENTRYPOINTS);
@@ -9128,6 +9145,7 @@ var StreamerServer = class {
         if (!tailed) this.maybeAttachExternalTail(filePath);
         this.sweepIdleExternalTails();
         this.cache?.invalidateByFilePath(filePath, { skipIfTailed: true });
+        this.staleFiles.add(filePath);
         this.markScannerStaleDebounced();
         this.log.debug?.(`Scanner invalidated by directory event: ${filePath}`, {
           filePath,
@@ -10374,21 +10392,46 @@ var StreamerServer = class {
   // so a burst of list polls during active session writes shares one rescan
   // rather than queueing a full rescan each; tracked so close() awaits the
   // in-flight cache write before shutting the DB.
-  startBackgroundConversationReconcile() {
+  startBackgroundConversationReconcile(mode = "full") {
     if (this.conversationReconcileInFlight) return;
-    const task = this.reconcileConversationsCacheFromDisk().finally(() => {
+    const paths = mode === "files" ? this.takeStaleFiles() : [];
+    const task = (paths.length > 0 ? this.reconcileStaleFilesFromDisk(paths) : this.reconcileConversationsCacheFromDisk()).finally(() => {
       this.conversationReconcileInFlight = null;
     });
     this.conversationReconcileInFlight = task;
     this.trackCacheWrite(task);
   }
-  shouldAutoReconcileConversationList() {
-    if (!this.cache) return false;
-    if (this.scannerStale) return true;
-    if (!this.conversationsRepo || !this.cacheMetadataRepo) return false;
+  // "files": a directory event named specific JSONLs, so refresh only those.
+  // "full": disk drifted in ways a per-file refresh can't see (a project dir
+  // appeared, rows vanished), so walk the tree. Order matters — the staleness
+  // check short-circuits first so the HDD freshness probe stays off the hot
+  // poll path, exactly as it did when this returned a boolean.
+  conversationReconcileMode() {
+    if (!this.cache) return null;
+    if (this.scannerStale) return "files";
+    if (!this.conversationsRepo || !this.cacheMetadataRepo) return null;
     return shouldRefreshProjectsFromHdd(this.conversationsRepo, this.cacheMetadataRepo, {
       projectsDirs: this.projectsDirsForFreshnessCheck()
-    });
+    }) ? "full" : null;
+  }
+  // The per-file half of reconcileConversationsCacheFromDisk: re-index just the
+  // changed JSONLs and upsert their rows. No reconcileDeletions here — that
+  // needs the whole live-path set, and deletions already have their own path
+  // (onFileDeleted -> invalidateByFilePath). New projects still arrive via the
+  // HDD-freshness "full" mode.
+  async reconcileStaleFilesFromDisk(paths) {
+    if (!this.cache) return;
+    const scanner = await this.getScanner(true);
+    const metas = await this.refreshStaleFiles(scanner, paths);
+    if (metas.length === 0) return;
+    try {
+      this.cache.upsertFromScannerMeta(metas);
+    } catch (err) {
+      this.log.warn(
+        `stale-file reconcile failed: ${err instanceof Error ? err.message : String(err)}`,
+        { event: "conversations.reconcile_failed" }
+      );
+    }
   }
   async handleListConversations(url, res) {
     if (this.rejectIfWarmingUp(res)) return;
@@ -10398,10 +10441,11 @@ var StreamerServer = class {
     const project = url.searchParams.get("project") ?? void 0;
     const providerFilter = url.searchParams.get("provider") ?? void 0;
     const bustCache = url.searchParams.get("refresh") === "1";
-    if (this.cache && (bustCache || this.shouldAutoReconcileConversationList())) {
+    const reconcileMode = this.conversationReconcileMode();
+    if (this.cache && (bustCache || reconcileMode)) {
       const canServeStale = !bustCache && this.cache.listConversations({ limit: 0, offset: 0 }).total > 0;
       if (canServeStale) {
-        this.startBackgroundConversationReconcile();
+        this.startBackgroundConversationReconcile(reconcileMode ?? "full");
       } else {
         const shouldEmitProgress = createScanProgressThrottle();
         await this.withWarmup(
@@ -10601,21 +10645,54 @@ var StreamerServer = class {
       options ?? (this.scannerPersistenceDisabled ? { persistent: false } : void 0)
     );
   }
+  // Drain the stale set and disarm the flag together. The caller owns the
+  // returned paths: clearing before the refresh means events that land DURING
+  // it re-arm the flag and get their own pass instead of being swallowed.
+  takeStaleFiles() {
+    const paths = [...this.staleFiles];
+    this.staleFiles.clear();
+    this.scannerStale = false;
+    return paths;
+  }
+  // Reconcile exactly the JSONLs a directory event named. Failures are logged
+  // and swallowed per file: one unreadable transcript must not abort the
+  // others, and the file simply stays on its previous snapshot until the next
+  // event — the same outcome the full rescan gave on a parse failure.
+  async refreshStaleFiles(scanner, paths) {
+    const metas = await Promise.all(
+      paths.map(
+        (filePath) => scanner.refreshFile(filePath).catch((err) => {
+          this.log.warn("scanner.refreshFile: failed", {
+            event: "scanner.refresh_failed",
+            filePath,
+            trigger: "directory-event",
+            err
+          });
+          return null;
+        })
+      )
+    );
+    return metas.filter((m) => m !== null);
+  }
   async getScanner(skipStaleRescan = false) {
     if (this.scannerReady) {
       await this.scannerReady;
       if (this.scanner) {
         if (skipStaleRescan) return this.scanner;
         if (this.scannerStale) {
-          this.scannerStale = false;
-          this.scanner = null;
-          this.scannerReady = null;
-          return this.getScanner();
+          const paths = this.takeStaleFiles();
+          if (paths.length === 0) {
+            this.scanner = null;
+            this.scannerReady = null;
+            return this.getScanner();
+          }
+          await this.refreshStaleFiles(this.scanner, paths);
+          return this.scanner ?? this.getScanner();
         }
         return this.scanner;
       }
     }
-    this.scannerStale = false;
+    this.takeStaleFiles();
     const statCache = this.buildStatCache(this.scanner);
     this.scanner = this.newScanner(statCache ? { persistent: false } : void 0);
     this.allScanners.add(this.scanner);
@@ -10649,7 +10726,7 @@ var StreamerServer = class {
   // getScanner() anti-infinite-loop guard is preserved.
   async rescanForRefresh(onProgress) {
     if (this.scannerReady) await this.scannerReady;
-    this.scannerStale = false;
+    this.takeStaleFiles();
     if (!this.scanner) {
       this.scanner = new import_scanner3.ConversationScanner();
       this.allScanners.add(this.scanner);