@threadbase-sh/streamer 1.24.3 → 1.24.5

package/dist/index.d.cts

@@ -515,6 +515,34 @@ declare class ConversationCache {
      * JSONL has been deleted.
      */
     pruneGhostFiles(exists?: (filePath: string) => boolean): string[];
+    /**
+     * Reconcile the cache against the authoritative set of conversation file
+     * paths a fresh scan surfaced: drop any cached row whose `file_path` is not
+     * in `livePaths` (removed from disk, or now filtered out — e.g. became an
+     * agent JSONL). This is the "removed conversations" half of a ?refresh=1
+     * reconcile; the additions/updates half is upsertFromScannerMeta.
+     *
+     * Skip semantics depend on whether the file still exists on disk:
+     *  - File GONE from disk → always removed, tail or not. This matches the old
+     *    invalidate()+rebuild behavior (a deleted conversation must disappear on
+     *    refresh) and keeps refresh=1 truthful about removals. NOTE: this is an
+     *    INTENTIONAL divergence from pruneGhostFiles(), which KEEPS tailed ghosts
+     *    so their cached history stays viewable on a background prune. refresh=1
+     *    has the opposite contract (mobile relies on removals being reflected), so
+     *    do not "unify" the two — they serve different purposes.
+     *  - File STILL on disk but absent from `livePaths` → the CRITICAL #2 race:
+     *    the scan snapshot predates a just-created (and now live-tailed) file.
+     *    A tailed row here is actively maintained from real content, so it is
+     *    kept — dropping it would flicker the active conversation out of
+     *    /api/conversations. An untailed on-disk row not in the snapshot is a
+     *    transient scan/discovery gap; it is left alone (not removed) and the
+     *    next reconcile picks it up, rather than risk removing a real file the
+     *    scan simply hasn't surfaced yet.
+     * Returns the removed IDs.
+     */
+    reconcileDeletions(livePaths: Set<string>, opts?: {
+        exists?: (filePath: string) => boolean;
+    }): string[];
 }
 
 type CacheMetadataKey = "last_conversation_id" | "last_conversation_created_at" | "projects_last_indexed_at" | "conversations_last_indexed_at" | "conversations_dirty";
@@ -840,6 +868,7 @@ declare class StreamerServer {
     private scannerStale;
     private binding;
     private cacheReady;
+    private inFlightCacheWrites;
     private apiKey;
     private apiKeySource;
     private localNoAuth;
@@ -897,6 +926,7 @@ declare class StreamerServer {
     }): Promise<void>;
     private bindWithRetry;
     private bindWithRetryLoop;
+    private trackCacheWrite;
     close(): Promise<void>;
     private handleRequest;
     private handlePairStart;
@@ -917,6 +947,7 @@ declare class StreamerServer {
     private newScanner;
     private getScanner;
     private getFreshScanner;
+    private rescanForRefresh;
     private findJsonlPath;
     private readCwdFromJsonl;
     private findConversationByUuid;
@@ -987,6 +1018,15 @@ declare class ConversationWatcher {
     constructor(events?: ConversationWatcherEvents);
     watch(filePath: string): void;
     unwatch(filePath: string): void;
+    /**
+     * Re-drive the tail read for a file that's already being tailed. A per-file
+     * chokidar handle can die silently (fs.watch stops firing after inode churn)
+     * while the coarser directory watcher keeps reporting changes — calling this
+     * from the directory-event path makes the tail self-healing. Reads are
+     * offset-based and coalesced, so a redundant poke after a normal change
+     * event is a cheap stat + no-op. Returns false for untailed paths.
+     */
+    poke(filePath: string): boolean;
     /**
      * Watch a directory of conversation JSONL files. Fires
      * onConversationChanged for any add/change/unlink event so the caller

package/dist/index.d.ts

@@ -515,6 +515,34 @@ declare class ConversationCache {
      * JSONL has been deleted.
      */
     pruneGhostFiles(exists?: (filePath: string) => boolean): string[];
+    /**
+     * Reconcile the cache against the authoritative set of conversation file
+     * paths a fresh scan surfaced: drop any cached row whose `file_path` is not
+     * in `livePaths` (removed from disk, or now filtered out — e.g. became an
+     * agent JSONL). This is the "removed conversations" half of a ?refresh=1
+     * reconcile; the additions/updates half is upsertFromScannerMeta.
+     *
+     * Skip semantics depend on whether the file still exists on disk:
+     *  - File GONE from disk → always removed, tail or not. This matches the old
+     *    invalidate()+rebuild behavior (a deleted conversation must disappear on
+     *    refresh) and keeps refresh=1 truthful about removals. NOTE: this is an
+     *    INTENTIONAL divergence from pruneGhostFiles(), which KEEPS tailed ghosts
+     *    so their cached history stays viewable on a background prune. refresh=1
+     *    has the opposite contract (mobile relies on removals being reflected), so
+     *    do not "unify" the two — they serve different purposes.
+     *  - File STILL on disk but absent from `livePaths` → the CRITICAL #2 race:
+     *    the scan snapshot predates a just-created (and now live-tailed) file.
+     *    A tailed row here is actively maintained from real content, so it is
+     *    kept — dropping it would flicker the active conversation out of
+     *    /api/conversations. An untailed on-disk row not in the snapshot is a
+     *    transient scan/discovery gap; it is left alone (not removed) and the
+     *    next reconcile picks it up, rather than risk removing a real file the
+     *    scan simply hasn't surfaced yet.
+     * Returns the removed IDs.
+     */
+    reconcileDeletions(livePaths: Set<string>, opts?: {
+        exists?: (filePath: string) => boolean;
+    }): string[];
 }
 
 type CacheMetadataKey = "last_conversation_id" | "last_conversation_created_at" | "projects_last_indexed_at" | "conversations_last_indexed_at" | "conversations_dirty";
@@ -840,6 +868,7 @@ declare class StreamerServer {
     private scannerStale;
     private binding;
     private cacheReady;
+    private inFlightCacheWrites;
     private apiKey;
     private apiKeySource;
     private localNoAuth;
@@ -897,6 +926,7 @@ declare class StreamerServer {
     }): Promise<void>;
     private bindWithRetry;
     private bindWithRetryLoop;
+    private trackCacheWrite;
     close(): Promise<void>;
     private handleRequest;
     private handlePairStart;
@@ -917,6 +947,7 @@ declare class StreamerServer {
     private newScanner;
     private getScanner;
     private getFreshScanner;
+    private rescanForRefresh;
     private findJsonlPath;
     private readCwdFromJsonl;
     private findConversationByUuid;
@@ -987,6 +1018,15 @@ declare class ConversationWatcher {
     constructor(events?: ConversationWatcherEvents);
     watch(filePath: string): void;
     unwatch(filePath: string): void;
+    /**
+     * Re-drive the tail read for a file that's already being tailed. A per-file
+     * chokidar handle can die silently (fs.watch stops firing after inode churn)
+     * while the coarser directory watcher keeps reporting changes — calling this
+     * from the directory-event path makes the tail self-healing. Reads are
+     * offset-based and coalesced, so a redundant poke after a normal change
+     * event is a cheap stat + no-op. Returns false for untailed paths.
+     */
+    poke(filePath: string): boolean;
     /**
      * Watch a directory of conversation JSONL files. Fires
      * onConversationChanged for any add/change/unlink event so the caller

package/dist/index.js

@@ -3768,6 +3768,61 @@ var ConversationCache = class _ConversationCache {
     }
     return ghosts;
   }
+  /**
+   * Reconcile the cache against the authoritative set of conversation file
+   * paths a fresh scan surfaced: drop any cached row whose `file_path` is not
+   * in `livePaths` (removed from disk, or now filtered out — e.g. became an
+   * agent JSONL). This is the "removed conversations" half of a ?refresh=1
+   * reconcile; the additions/updates half is upsertFromScannerMeta.
+   *
+   * Skip semantics depend on whether the file still exists on disk:
+   *  - File GONE from disk → always removed, tail or not. This matches the old
+   *    invalidate()+rebuild behavior (a deleted conversation must disappear on
+   *    refresh) and keeps refresh=1 truthful about removals. NOTE: this is an
+   *    INTENTIONAL divergence from pruneGhostFiles(), which KEEPS tailed ghosts
+   *    so their cached history stays viewable on a background prune. refresh=1
+   *    has the opposite contract (mobile relies on removals being reflected), so
+   *    do not "unify" the two — they serve different purposes.
+   *  - File STILL on disk but absent from `livePaths` → the CRITICAL #2 race:
+   *    the scan snapshot predates a just-created (and now live-tailed) file.
+   *    A tailed row here is actively maintained from real content, so it is
+   *    kept — dropping it would flicker the active conversation out of
+   *    /api/conversations. An untailed on-disk row not in the snapshot is a
+   *    transient scan/discovery gap; it is left alone (not removed) and the
+   *    next reconcile picks it up, rather than risk removing a real file the
+   *    scan simply hasn't surfaced yet.
+   * Returns the removed IDs.
+   */
+  reconcileDeletions(livePaths, opts) {
+    const exists = opts?.exists ?? existsSync5;
+    const rows = this.stmts.allFilePaths.all();
+    const removed = [];
+    const drop = this.db.transaction((ids) => {
+      for (const id of ids) {
+        this.stmts.deleteTailById.run(id);
+        this.stmts.deleteById.run(id);
+      }
+    });
+    for (const row of rows) {
+      if (livePaths.has(row.file_path)) continue;
+      if (exists(row.file_path)) continue;
+      removed.push(row.id);
+    }
+    if (removed.length > 0) {
+      drop(removed);
+      if (this.fileIndexLoaded) {
+        for (const id of removed) {
+          for (const [fp, cid] of this.fileIndex) {
+            if (cid === id) {
+              this.fileIndex.delete(fp);
+              break;
+            }
+          }
+        }
+      }
+    }
+    return removed;
+  }
 };
 
 // src/db/repositories/cacheMetadata.repository.ts
@@ -4127,6 +4182,19 @@ var ConversationWatcher = class {
     void entry.watcher.close();
     this.files.delete(filePath);
   }
+  /**
+   * Re-drive the tail read for a file that's already being tailed. A per-file
+   * chokidar handle can die silently (fs.watch stops firing after inode churn)
+   * while the coarser directory watcher keeps reporting changes — calling this
+   * from the directory-event path makes the tail self-healing. Reads are
+   * offset-based and coalesced, so a redundant poke after a normal change
+   * event is a cheap stat + no-op. Returns false for untailed paths.
+   */
+  poke(filePath) {
+    if (!this.files.has(filePath)) return false;
+    void this.readNewLines(filePath);
+    return true;
+  }
   /**
    * Watch a directory of conversation JSONL files. Fires
    * onConversationChanged for any add/change/unlink event so the caller
@@ -4832,6 +4900,14 @@ var StreamerServer = class {
   // window so the self-healing kickstart-relaunch race doesn't spam warn.
   binding = false;
   cacheReady = false;
+  // Every fire-and-forget task that runs a scan and then writes to this.cache
+  // in an async continuation (startup warm-up, background count refresh, …).
+  // close() awaits all of them before closing this.cache, so a scan's post-scan
+  // cache writes (upsertFromScannerMeta / populateTailFromFile / pruneGhostFiles
+  // / reconcileDeletions) can never hit a cache.db that was already closed
+  // ("database connection is not open"), which would otherwise leave the cache
+  // empty. Register via trackCacheWrite(); each entry removes itself on settle.
+  inFlightCacheWrites = /* @__PURE__ */ new Set();
   apiKey;
   apiKeySource;
   localNoAuth;
@@ -4963,6 +5039,7 @@ var StreamerServer = class {
         }
       },
       onConversationChanged: (filePath) => {
+        this.fileWatcher.poke(filePath);
         this.cache?.invalidateByFilePath(filePath, { skipIfTailed: true });
         this.markScannerStaleDebounced();
         this.log.debug?.(`Scanner invalidated by directory event: ${filePath}`, {
@@ -5399,6 +5476,7 @@ var StreamerServer = class {
         });
       }
     });
+    this.trackCacheWrite(warmUp);
     if (opts?.awaitReady) await warmUp;
   }
   // Bind the HTTP listener, retrying on a transient EADDRINUSE. See the call
@@ -5446,11 +5524,23 @@ var StreamerServer = class {
       }
     }
   }
+  // Register a fire-and-forget task that writes to this.cache after a scan, so
+  // close() can await it before closing cache.db. Removes itself on settle. The
+  // caller keeps its own error handling; this wrapper swallows rejections so a
+  // failed task never rejects close()'s Promise.all.
+  trackCacheWrite(task) {
+    const guarded = task.catch(() => void 0);
+    this.inFlightCacheWrites.add(guarded);
+    void guarded.finally(() => {
+      this.inFlightCacheWrites.delete(guarded);
+    });
+  }
   async close() {
     for (const timer of this.ptyGraceTimers.values()) clearTimeout(timer);
     this.ptyGraceTimers.clear();
     this.markScannerStaleDebounced.cancel();
-    for (const s of this.allScanners) s.close();
+    await Promise.all([...this.inFlightCacheWrites]);
+    await Promise.all([...this.allScanners].map((s) => s.close()));
     this.allScanners.clear();
     this.scanner = null;
     this.cache?.close();
@@ -5590,12 +5680,23 @@ var StreamerServer = class {
     const project = url.searchParams.get("project") ?? void 0;
     const providerFilter = url.searchParams.get("provider") ?? void 0;
     const bustCache = url.searchParams.get("refresh") === "1";
-    if (bustCache) {
-      this.cache?.invalidate();
-      this.scanner = null;
-      this.scannerReady = null;
+    if (bustCache && this.cache) {
+      const scanner2 = await this.rescanForRefresh();
+      const metas2 = [...scanner2.getMetadataCache().values()];
+      try {
+        this.cache.upsertFromScannerMeta(metas2);
+        const livePaths = new Set(
+          metas2.map((m) => m.filePath).filter((p) => Boolean(p))
+        );
+        this.cache.reconcileDeletions(livePaths);
+      } catch (err) {
+        this.log.warn(
+          `refresh reconcile failed: ${err instanceof Error ? err.message : String(err)}`,
+          { event: "conversations.reconcile_failed" }
+        );
+      }
     }
-    if (this.cache && !bustCache) {
+    if (this.cache) {
       const { conversations, total: total2 } = this.cache.listConversations({
         project,
         provider: providerFilter,
@@ -5660,12 +5761,6 @@ var StreamerServer = class {
       };
     });
     json(res, 200, { conversations: adapted, hasMore: offset + limit < total, offset, total });
-    if (this.cache && bustCache) {
-      try {
-        this.cache.upsertFromScannerMeta([...scanner.getMetadataCache().values()]);
-      } catch {
-      }
-    }
   }
   async handleConversationsCount(url, res) {
     const project = url.searchParams.get("project") ?? void 0;
@@ -5694,19 +5789,21 @@ var StreamerServer = class {
   // later count reflects new/removed conversations. Never awaited by the request
   // path — refresh=1 returns the cached total synchronously and this catches up.
   refreshCountInBackground() {
-    void (async () => {
-      try {
-        const scanner = await this.getFreshScanner();
-        if (this.cache) {
-          this.cache.upsertFromScannerMeta([...scanner.getMetadataCache().values()]);
+    this.trackCacheWrite(
+      (async () => {
+        try {
+          const scanner = await this.getFreshScanner();
+          if (this.cache) {
+            this.cache.upsertFromScannerMeta([...scanner.getMetadataCache().values()]);
+          }
+        } catch (err) {
+          this.log.warn(
+            `Background count refresh failed: ${err instanceof Error ? err.message : String(err)}`,
+            { event: "count.refresh_failed" }
+          );
         }
-      } catch (err) {
-        this.log.warn(
-          `Background count refresh failed: ${err instanceof Error ? err.message : String(err)}`,
-          { event: "count.refresh_failed" }
-        );
-      }
-    })();
+      })()
+    );
   }
   handleSessionsCount(res) {
     json(res, 200, { total: this.sessionStore.list(this.ptyAttachedIds()).length });
@@ -5819,6 +5916,30 @@ var StreamerServer = class {
     this.scannerReady = null;
     return this.getScanner();
   }
+  // refresh=1's scan: reuse the WARM persistent scanner (its index.db + cursors
+  // survive, so classify() still skips unchanged files) and re-run its scan
+  // with fullRescan:true — the escape hatch that bypasses the scanner's
+  // dir-mtime discovery gate, since an explicit user pull-to-refresh is exactly
+  // the "don't trust the gate, check disk for real" signal. Unlike
+  // getFreshScanner() this does NOT discard the warm scanner. scannerReady is
+  // only ever reassigned to a live scan promise (never nulled mid-scan), so the
+  // getScanner() anti-infinite-loop guard is preserved.
+  async rescanForRefresh() {
+    if (this.scannerReady) await this.scannerReady;
+    this.scannerStale = false;
+    if (!this.scanner) {
+      this.scanner = new ConversationScanner();
+      this.allScanners.add(this.scanner);
+    }
+    const scanner = this.scanner;
+    this.scannerReady = scanner.scan({
+      ...this.scanProfiles ? { profiles: this.scanProfiles } : {},
+      ...this.codexScanOpts(),
+      fullRescan: true
+    });
+    await this.scannerReady;
+    return scanner;
+  }
   findJsonlPath(uuid) {
     const projectsDir = join12(homedir5(), ".claude", "projects");
     if (!existsSync7(projectsDir)) return null;