@threadbase-sh/streamer 1.27.3 → 1.28.0

package/dist/index.cjs

@@ -2317,10 +2317,10 @@ async function readGitBranch(dir) {
 // src/server.ts
 var import_node_ws = require("@hono/node-ws");
 var import_client = require("@temporalio/client");
-var import_scanner2 = require("@threadbase-sh/scanner");
+var import_scanner3 = require("@threadbase-sh/scanner");
 var import_events = require("events");
 var import_fs12 = require("fs");
-var import_promises5 = require("fs/promises");
+var import_promises7 = require("fs/promises");
 var import_http = require("http");
 var import_os6 = require("os");
 var import_path13 = require("path");
@@ -3162,9 +3162,12 @@ async function createDirectory(parentAbsolutePath, name) {
 }
 
 // src/conversation-cache.ts
+var import_scanner2 = require("@threadbase-sh/scanner");
 var import_better_sqlite3 = __toESM(require("better-sqlite3"), 1);
 var import_fs8 = require("fs");
+var import_promises3 = require("fs/promises");
 var import_path10 = require("path");
+var import_promises4 = require("timers/promises");
 
 // src/db/sqlite-migrate.ts
 var import_fs6 = require("fs");
@@ -3277,6 +3280,33 @@ function parseAgentEntrypointsEnv(raw) {
   return new Set(parts);
 }
 
+// src/utils/fileIdentity.ts
+var import_crypto5 = require("crypto");
+function fileIdentity(stat3, headBytes) {
+  if (stat3.ino && stat3.ino > 0) return `inode:${stat3.dev}:${stat3.ino}`;
+  const head = headBytes ?? Buffer.alloc(0);
+  return `fp:${(0, import_crypto5.createHash)("sha1").update(head).digest("hex")}`;
+}
+function splitCompleteLines(buf, baseOffset) {
+  const spans = [];
+  let lineStart = 0;
+  let consumed = 0;
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] !== 10) continue;
+    const lineLen = i - lineStart;
+    if (lineLen > 0) {
+      spans.push({
+        byteOffset: baseOffset + lineStart,
+        byteLength: lineLen,
+        text: buf.toString("utf-8", lineStart, i)
+      });
+    }
+    lineStart = i + 1;
+    consumed = lineStart;
+  }
+  return { spans, consumed };
+}
+
 // src/conversation-cache.ts
 function shortProjectName(fullPath) {
   const parts = fullPath.split(/[/\\]/).filter(Boolean);
@@ -3325,6 +3355,14 @@ var ConversationCache = class _ConversationCache {
   tailSize;
   fileIndex = /* @__PURE__ */ new Map();
   fileIndexLoaded = false;
+  // Per-file scanner parse state for the incremental offset-index writer. The
+  // reducer is stateful across lines (pending tool_uses, latest timestamp), so
+  // it must persist between watcher reads of the same file. Cleared on
+  // truncation/backfill.
+  indexParseState = /* @__PURE__ */ new Map();
+  // Single-flight guard for backfillIndex — concurrent detail requests for the
+  // same cold file await one walk, not N. Entry dropped on settle.
+  backfillInFlight = /* @__PURE__ */ new Map();
   // Monotonically increasing counter for tail updated_at — guarantees strict
   // ordering even when multiple updateFromLine() calls land within the same ms.
   tailSeq = Date.now();
@@ -3488,6 +3526,42 @@ var ConversationCache = class _ConversationCache {
          GROUP BY project_path
          ORDER BY cnt DESC
          LIMIT ?`
+      ),
+      getFileState: db.prepare("SELECT * FROM conversation_file_state WHERE path = ?"),
+      upsertFileState: db.prepare(
+        `INSERT INTO conversation_file_state
+           (path, identity, size, mtime_ms, byte_offset, last_message_index)
+         VALUES (@path, @identity, @size, @mtime_ms, @byte_offset, @last_message_index)
+         ON CONFLICT(path) DO UPDATE SET
+           identity           = excluded.identity,
+           size               = excluded.size,
+           mtime_ms           = excluded.mtime_ms,
+           byte_offset        = excluded.byte_offset,
+           last_message_index = excluded.last_message_index`
+      ),
+      deleteFileState: db.prepare("DELETE FROM conversation_file_state WHERE path = ?"),
+      deleteMessageIndex: db.prepare(
+        "DELETE FROM conversation_message_index WHERE conversation_id = ?"
+      ),
+      insertMessageIndexRow: db.prepare(
+        `INSERT INTO conversation_message_index
+           (conversation_id, message_index, byte_offset, byte_length, uuid, role, ts)
+         VALUES (@conversation_id, @message_index, @byte_offset, @byte_length, @uuid, @role, @ts)
+         ON CONFLICT(conversation_id, message_index) DO UPDATE SET
+           byte_offset = excluded.byte_offset,
+           byte_length = excluded.byte_length,
+           uuid        = excluded.uuid,
+           role        = excluded.role,
+           ts          = excluded.ts`
+      ),
+      getMessageIndexWindow: db.prepare(
+        `SELECT message_index, byte_offset, byte_length, uuid, role, ts
+         FROM conversation_message_index
+         WHERE conversation_id = ? AND message_index >= ? AND message_index < ?
+         ORDER BY message_index ASC`
+      ),
+      getIndexedMessageCount: db.prepare(
+        "SELECT COUNT(*) as cnt FROM conversation_message_index WHERE conversation_id = ?"
       )
     };
   }
@@ -3498,6 +3572,258 @@ var ConversationCache = class _ConversationCache {
   getDatabase() {
     return this.db;
   }
+  // ── Offset index (design 1b) ────────────────────────────────────────────
+  // conversation_file_state + conversation_message_index back the windowed
+  // detail read path (SQL window select + pread of byte ranges). All methods
+  // are thin wrappers over the prepared statements above.
+  getFileState(path) {
+    return this.stmts.getFileState.get(path) ?? null;
+  }
+  upsertFileState(row) {
+    this.stmts.upsertFileState.run(row);
+  }
+  /** Drop a file's index rows + file_state (truncation / identity change). */
+  deleteFileIndex(path, conversationId) {
+    const tx = this.db.transaction(() => {
+      this.stmts.deleteMessageIndex.run(conversationId);
+      this.stmts.deleteFileState.run(path);
+    });
+    tx();
+    this.indexParseState.delete(path);
+  }
+  /** Append/replace index rows in one transaction. */
+  appendMessageIndexRows(rows) {
+    const tx = this.db.transaction((batch) => {
+      for (const r of batch) this.stmts.insertMessageIndexRow.run(r);
+    });
+    tx(rows);
+  }
+  /** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
+  getMessageIndexWindow(conversationId, fromIndex, toIndex) {
+    return this.stmts.getMessageIndexWindow.all(
+      conversationId,
+      fromIndex,
+      toIndex
+    );
+  }
+  getIndexedMessageCount(conversationId) {
+    return this.stmts.getIndexedMessageCount.get(conversationId).cnt;
+  }
+  /**
+   * Conversation id for a JSONL path — the filename stem (matches the pseudo-id
+   * updateFromLine derives and the uuid the detail read path resolves). The
+   * offset index keys on this so the window select and the cursor agree.
+   */
+  static conversationIdForFile(filePath) {
+    return filePath.split(/[/\\]/).pop()?.replace(/\.jsonl$/, "") ?? filePath;
+  }
+  /**
+   * Incremental offset-index writer: extend the index for a burst of appended
+   * lines (one watcher read) using their byte spans. Each line is classified
+   * with the scanner's parseJsonlLine (a running per-file reducer state), so the
+   * message ordering can never drift from the scanner's. Message lines get an
+   * index row at the next message_index; non-message lines (summary/sidecar)
+   * get no row but still advance byte_offset. file_state is updated to the end
+   * of the last consumed span.
+   *
+   * Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
+   * path can detect truncation/replacement.
+   *
+   * `readFrom` is the absolute byte offset the watcher read started at, and
+   * `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
+   * entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
+   * left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
+   * doesn't — the watcher attached at EOF after the server was down, or an
+   * append raced an in-flight backfill — extending would assign wrong
+   * message_index values over a hole. In that case this writes nothing and
+   * returns null so the caller drops the index and backfills.
+   *
+   * On success returns the message_index assigned to each input span (null for a
+   * non-message line) so the caller can stamp WS `seq`. Empty array when spans
+   * is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
+   * offset and file_state.byte_offset are the same number by construction.
+   */
+  extendMessageIndex(filePath, spans, stat3, readFrom, endOffset) {
+    const existing = this.getFileState(filePath);
+    const expectedStart = existing?.byte_offset ?? 0;
+    if (readFrom !== expectedStart) return null;
+    if (spans.length === 0) return [];
+    const convId = _ConversationCache.conversationIdForFile(filePath);
+    let state = this.indexParseState.get(filePath);
+    if (!state) {
+      state = (0, import_scanner2.createJsonlParseState)();
+      this.indexParseState.set(filePath, state);
+    }
+    let nextIndex = existing ? existing.last_message_index + 1 : 0;
+    const rows = [];
+    const seqs = [];
+    for (const span of spans) {
+      const msg = (0, import_scanner2.parseJsonlLine)(span.text, state);
+      if (!msg) {
+        seqs.push(null);
+        continue;
+      }
+      rows.push({
+        conversation_id: convId,
+        message_index: nextIndex,
+        byte_offset: span.byteOffset,
+        byte_length: span.byteLength,
+        uuid: msg.uuid ?? null,
+        role: msg.role ?? null,
+        ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
+      });
+      seqs.push(nextIndex);
+      nextIndex++;
+    }
+    const tx = this.db.transaction(() => {
+      for (const r of rows) this.stmts.insertMessageIndexRow.run(r);
+      this.stmts.upsertFileState.run({
+        path: filePath,
+        identity: fileIdentity(stat3),
+        size: stat3.size,
+        mtime_ms: Math.round(stat3.mtimeMs),
+        // Store the watcher's end offset verbatim — same number as entry.offset,
+        // so the next read's contiguity check compares like-for-like (never
+        // false-positive on a read that ended in trailing empty lines).
+        byte_offset: endOffset,
+        last_message_index: nextIndex - 1
+      });
+    });
+    tx();
+    return seqs;
+  }
+  clearIndexParseState(filePath) {
+    this.indexParseState.delete(filePath);
+  }
+  /**
+   * On-demand full backfill of the offset index for a file with no/stale
+   * file_state (cold conversation, or after a truncation/replacement). Rebuilds
+   * from byte 0: drops any existing rows, walks the whole file in chunks with a
+   * running parse state, yields to the event loop every ~1000 lines so a large
+   * file never blocks, and writes index rows + file_state.
+   *
+   * Single-flighted per path: concurrent callers await the same walk. The
+   * triggering detail request is served by the scanner fallback while this runs.
+   */
+  backfillIndex(filePath) {
+    const inFlight = this.backfillInFlight.get(filePath);
+    if (inFlight) return inFlight;
+    const walk = this.runBackfill(filePath).finally(() => {
+      this.backfillInFlight.delete(filePath);
+    });
+    this.backfillInFlight.set(filePath, walk);
+    return walk;
+  }
+  async runBackfill(filePath) {
+    const convId = _ConversationCache.conversationIdForFile(filePath);
+    this.deleteFileIndex(filePath, convId);
+    this.indexParseState.delete(filePath);
+    const CHUNK = 256 * 1024;
+    const YIELD_EVERY = 1e3;
+    const state = (0, import_scanner2.createJsonlParseState)();
+    const fh = await (0, import_promises3.open)(filePath, "r");
+    let fileOffset = 0;
+    let carry = Buffer.alloc(0);
+    let nextIndex = 0;
+    let linesSinceYield = 0;
+    let lastConsumedEnd = 0;
+    let stat3;
+    try {
+      stat3 = await fh.stat();
+      const buf = Buffer.alloc(CHUNK);
+      for (; ; ) {
+        const { bytesRead } = await fh.read(buf, 0, CHUNK, null);
+        if (bytesRead === 0) break;
+        const combined = carry.length > 0 ? Buffer.concat([carry, buf.subarray(0, bytesRead)]) : buf.subarray(0, bytesRead);
+        const { spans, consumed } = splitCompleteLines(combined, fileOffset);
+        const rows = [];
+        for (const span of spans) {
+          const msg = (0, import_scanner2.parseJsonlLine)(span.text, state);
+          linesSinceYield++;
+          if (msg) {
+            rows.push({
+              conversation_id: convId,
+              message_index: nextIndex,
+              byte_offset: span.byteOffset,
+              byte_length: span.byteLength,
+              uuid: msg.uuid ?? null,
+              role: msg.role ?? null,
+              ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
+            });
+            nextIndex++;
+          }
+          if (linesSinceYield >= YIELD_EVERY) {
+            linesSinceYield = 0;
+            await (0, import_promises4.setImmediate)();
+          }
+        }
+        if (rows.length > 0) this.appendMessageIndexRows(rows);
+        lastConsumedEnd = fileOffset + consumed;
+        carry = Buffer.from(combined.subarray(consumed));
+        fileOffset += consumed;
+      }
+    } finally {
+      await fh.close();
+    }
+    this.upsertFileState({
+      path: filePath,
+      identity: fileIdentity(stat3),
+      size: stat3.size,
+      mtime_ms: Math.round(stat3.mtimeMs),
+      byte_offset: lastConsumedEnd,
+      last_message_index: nextIndex - 1
+    });
+    this.indexParseState.set(filePath, state);
+  }
+  /**
+   * Windowed detail read straight from the offset index — the hot path.
+   * Returns the parsed messages for message_index in [fromIndex, toIndex) plus
+   * the total indexed count, or null when the index can't serve this file (no
+   * file_state, identity/size mismatch = truncation/replacement, or cold index)
+   * so the caller falls back to the scanner and enqueues a backfill.
+   *
+   * On a match it SQL-selects the window's byte ranges and preads exactly those
+   * ranges from the JSONL (never the whole file), parsing only the sliced lines.
+   * Returns messages in the same ConversationMessage shape parseJsonlLine
+   * produces during a scan, so the payload is identical to the scanner path.
+   */
+  readMessageWindow(filePath, fromIndex, toIndex) {
+    const fileState = this.getFileState(filePath);
+    if (!fileState) return null;
+    let stat3;
+    try {
+      stat3 = (0, import_fs8.statSync)(filePath);
+    } catch {
+      return null;
+    }
+    if (fileIdentity(stat3) !== fileState.identity || stat3.size !== fileState.byte_offset) {
+      return null;
+    }
+    const total = fileState.last_message_index + 1;
+    const from = Math.max(0, fromIndex);
+    const to = Math.min(toIndex, total);
+    if (to <= from) return { messages: [], total, fromIndex: from };
+    const rows = this.getMessageIndexWindow(
+      _ConversationCache.conversationIdForFile(filePath),
+      from,
+      to
+    );
+    if (rows.length === 0) return { messages: [], total, fromIndex: from };
+    const messages = [];
+    const fd = (0, import_fs8.openSync)(filePath, "r");
+    try {
+      const state = (0, import_scanner2.createJsonlParseState)();
+      for (const row of rows) {
+        const buf = Buffer.alloc(row.byte_length);
+        (0, import_fs8.readSync)(fd, buf, 0, row.byte_length, row.byte_offset);
+        const msg = (0, import_scanner2.parseJsonlLine)(buf.toString("utf-8"), state);
+        if (msg) messages.push(msg);
+      }
+    } finally {
+      (0, import_fs8.closeSync)(fd);
+    }
+    return { messages, total, fromIndex: from };
+  }
   agentEntrypointsKey() {
     return [...this.agentEntrypoints].sort().join(",");
   }
@@ -4131,7 +4457,7 @@ var ConversationsRepository = class {
 };
 
 // src/db/repositories/projects.repository.ts
-var import_crypto5 = require("crypto");
+var import_crypto6 = require("crypto");
 
 // src/utils/canonicalizeProjectPath.ts
 function canonicalizeProjectPath(projectPath) {
@@ -4224,7 +4550,7 @@ var ProjectsRepository = class {
       });
       return rowToProject(this.getById.get(existing.id));
     }
-    const id = (0, import_crypto5.randomUUID)();
+    const id = (0, import_crypto6.randomUUID)();
     this.insert.run({
       id,
       path,
@@ -4315,7 +4641,7 @@ function handleListProjects(url, res) {
 }
 
 // src/pair-store.ts
-var import_crypto6 = require("crypto");
+var import_crypto7 = require("crypto");
 var DEFAULT_TTL_SECONDS = 180;
 var SWEEP_INTERVAL_MS = 6e4;
 var PairTokenStore = class {
@@ -4330,7 +4656,7 @@ var PairTokenStore = class {
     }
   }
   mint() {
-    const token = `pt_${(0, import_crypto6.randomBytes)(16).toString("hex")}`;
+    const token = `pt_${(0, import_crypto7.randomBytes)(16).toString("hex")}`;
     const expiresAt = Date.now() + this.ttlMs;
     this.current = { token, expiresAt, used: false };
     return {
@@ -4392,18 +4718,20 @@ function seal(plaintext, recipientPublicKeyBase64) {
 // src/services/conversations/conversationWatcher.ts
 var import_chokidar = __toESM(require("chokidar"), 1);
 var import_fs10 = require("fs");
-var import_promises3 = require("fs/promises");
+var import_promises5 = require("fs/promises");
 var ConversationWatcher = class {
   files = /* @__PURE__ */ new Map();
   directories = /* @__PURE__ */ new Map();
   onNewLine;
   onNewLines;
+  onNewLineSpans;
   onConversationChanged;
   onFileDeleted;
   onError;
   constructor(events = {}) {
     this.onNewLine = events.onNewLine;
     this.onNewLines = events.onNewLines;
+    this.onNewLineSpans = events.onNewLineSpans;
     this.onConversationChanged = events.onConversationChanged;
     this.onFileDeleted = events.onFileDeleted;
     this.onError = events.onError;
@@ -4496,20 +4824,24 @@ var ConversationWatcher = class {
     entry.reading = true;
     try {
       for (; ; ) {
-        const st = await (0, import_promises3.stat)(filePath);
+        const st = await (0, import_promises5.stat)(filePath);
         if (st.size <= entry.offset) break;
         const readFrom = entry.offset;
         const bytesToRead = st.size - readFrom;
         const buf = Buffer.alloc(bytesToRead);
-        const fh = await (0, import_promises3.open)(filePath, "r");
+        const fh = await (0, import_promises5.open)(filePath, "r");
         try {
           await fh.read(buf, 0, bytesToRead, readFrom);
         } finally {
           await fh.close();
         }
-        entry.offset = readFrom + bytesToRead;
+        const { spans, consumed } = splitCompleteLines(buf, readFrom);
+        entry.offset = readFrom + consumed;
         if (!this.files.has(filePath)) return;
-        const lines = buf.toString("utf-8").split("\n").filter(Boolean);
+        const lines = spans.map((s) => s.text);
+        if (spans.length > 0) {
+          this.onNewLineSpans?.(filePath, spans, readFrom, entry.offset);
+        }
         if (this.onNewLines) {
           this.onNewLines(filePath, lines);
         } else {
@@ -4923,8 +5255,8 @@ function discoveredToResponse(d, conversationId) {
 }
 
 // src/uploads.ts
-var import_crypto7 = require("crypto");
-var import_promises4 = require("fs/promises");
+var import_crypto8 = require("crypto");
+var import_promises6 = require("fs/promises");
 var import_heic_convert = __toESM(require("heic-convert"), 1);
 var import_path12 = require("path");
 var UPLOAD_DIR_NAME = ".threadbase-uploads";
@@ -4957,12 +5289,12 @@ async function saveUploadFile(input) {
     mimeType = "image/jpeg";
     originalName = originalName.replace(/\.(heic|heif)$/i, ".jpg");
   }
-  const id = `up_${(0, import_crypto7.randomBytes)(8).toString("hex")}`;
+  const id = `up_${(0, import_crypto8.randomBytes)(8).toString("hex")}`;
   const safeName = sanitizeFilename(originalName) || `file${MIME_TO_EXT[mimeType] ?? ""}`;
   const dir = (0, import_path12.join)(input.projectPath, UPLOAD_DIR_NAME, input.sessionId);
-  await (0, import_promises4.mkdir)(dir, { recursive: true });
+  await (0, import_promises6.mkdir)(dir, { recursive: true });
   const filePath = (0, import_path12.join)(dir, `${Date.now()}-${id}-${safeName}`);
-  await (0, import_promises4.writeFile)(filePath, buffer);
+  await (0, import_promises6.writeFile)(filePath, buffer);
   return {
     id,
     filePath,
@@ -5149,6 +5481,10 @@ var StreamerServer = class {
   fileWatcher;
   sessionFileMap = /* @__PURE__ */ new Map();
   // sessionId → JSONL filePath
+  // Per-file seq assignments from the most recent onNewLineSpans (offset index),
+  // handed to the immediately-following onNewLines so it can stamp WS `seq` on
+  // the matching conversation_events entries. Same read → same lines order.
+  pendingLineSeqs = /* @__PURE__ */ new Map();
   pendingQuestions = /* @__PURE__ */ new Map();
   // Content key of the AskUserQuestion currently broadcast for a session (from
   // either the rendered screen or JSONL), used to de-dupe the two paths: when
@@ -5272,7 +5608,7 @@ var StreamerServer = class {
     this.agentEntrypoints = parseAgentEntrypointsEnv(process.env.THREADBASE_AGENT_ENTRYPOINTS);
     const rawRoot = process.env.THREADBASE_BROWSE_ROOT ?? loadBrowseRoot() ?? config.browseRoot;
     if (rawRoot) {
-      (0, import_promises5.realpath)(rawRoot).then((resolved) => {
+      (0, import_promises7.realpath)(rawRoot).then((resolved) => {
         this.browseRoot = resolved;
         if (this.verbose) this.log.info(`Browse root: ${resolved}`, { browseRoot: resolved });
       }).catch(() => {
@@ -5294,6 +5630,42 @@ var StreamerServer = class {
     this.sessionStore = new SessionStore();
     this.wsHub = new WSHub();
     this.fileWatcher = new ConversationWatcher({
+      onNewLineSpans: (filePath, spans, readFrom, endOffset) => {
+        if (!this.cache) return;
+        const cache = this.cache;
+        this.pendingLineSeqs.delete(filePath);
+        try {
+          const seqs = cache.extendMessageIndex(
+            filePath,
+            spans,
+            (0, import_fs12.statSync)(filePath),
+            readFrom,
+            endOffset
+          );
+          if (seqs === null) {
+            cache.deleteFileIndex(filePath, ConversationCache.conversationIdForFile(filePath));
+            cache.clearIndexParseState(filePath);
+            this.trackCacheWrite(
+              cache.backfillIndex(filePath).catch((err) => {
+                this.log.warn("offset-index.backfill_failed", {
+                  event: "offset_index.backfill_failed",
+                  filePath,
+                  trigger: "noncontiguous-append",
+                  err
+                });
+              })
+            );
+            return;
+          }
+          this.pendingLineSeqs.set(filePath, seqs);
+        } catch (err) {
+          this.log.warn("offset-index.extend_failed", {
+            event: "offset_index.extend_failed",
+            filePath,
+            err
+          });
+        }
+      },
       onNewLines: (filePath, lines) => {
         this.cache?.updateFromLines(filePath, lines);
         for (const [sessionId, watchedPath] of this.sessionFileMap) {
@@ -5320,13 +5692,20 @@ var StreamerServer = class {
               this.pendingQuestionKey.set(sessionId, key);
               if (broadcast) this.wsHub.broadcast(m);
             }
-            this.wsHub.broadcast({ type: "conversation_events", sessionId, lines });
+            const seqs = this.pendingLineSeqs.get(filePath);
+            this.wsHub.broadcast({
+              type: "conversation_events",
+              sessionId,
+              lines,
+              ...seqs && seqs.length === lines.length ? { seqs } : {}
+            });
             for (const line of lines) {
               this.wsHub.broadcast({ type: "conversation_event", sessionId, line });
             }
             break;
           }
         }
+        this.pendingLineSeqs.delete(filePath);
       },
       onConversationChanged: (filePath) => {
         this.fileWatcher.poke(filePath);
@@ -6049,13 +6428,13 @@ var StreamerServer = class {
     }
     const scanner = await this.getScanner();
     let metas = [...scanner.getMetadataCache().values()];
-    metas = (0, import_scanner2.applyIncludeFilter)(metas, "conversations");
-    if (project) metas = (0, import_scanner2.applyProjectFilter)(metas, project);
+    metas = (0, import_scanner3.applyIncludeFilter)(metas, "conversations");
+    if (project) metas = (0, import_scanner3.applyProjectFilter)(metas, project);
     if (providerFilter)
       metas = metas.filter((m) => (m.provider ?? CLAUDE_CODE_PROVIDER) === providerFilter);
-    metas = (0, import_scanner2.applySort)(metas, sort);
+    metas = (0, import_scanner3.applySort)(metas, sort);
     const total = metas.length;
-    const page = (0, import_scanner2.applyPagination)(metas, limit, offset);
+    const page = (0, import_scanner3.applyPagination)(metas, limit, offset);
     const adapted = page.items.map((c) => {
       const id = c.sessionId || c.id.split("/").pop()?.replace(/\.jsonl$/, "") || c.id;
       return {
@@ -6099,8 +6478,8 @@ var StreamerServer = class {
     }
     const scanner = await this.getScanner(true);
     let metas = [...scanner.getMetadataCache().values()];
-    metas = (0, import_scanner2.applyIncludeFilter)(metas, "conversations");
-    if (project) metas = (0, import_scanner2.applyProjectFilter)(metas, project);
+    metas = (0, import_scanner3.applyIncludeFilter)(metas, "conversations");
+    if (project) metas = (0, import_scanner3.applyProjectFilter)(metas, project);
     if (providerFilter)
       metas = metas.filter((m) => (m.provider ?? CLAUDE_CODE_PROVIDER) === providerFilter);
     json(res, 200, { total: metas.length });
@@ -6197,7 +6576,7 @@ var StreamerServer = class {
   // reconciles the one conversation being requested, so paying a full-tree
   // rescan just because some OTHER file changed is the stall this avoids.
   newScanner(options) {
-    return new import_scanner2.ConversationScanner(
+    return new import_scanner3.ConversationScanner(
       options ?? (this.scannerPersistenceDisabled ? { persistent: false } : void 0)
     );
   }
@@ -6251,7 +6630,7 @@ var StreamerServer = class {
     if (this.scannerReady) await this.scannerReady;
     this.scannerStale = false;
     if (!this.scanner) {
-      this.scanner = new import_scanner2.ConversationScanner();
+      this.scanner = new import_scanner3.ConversationScanner();
       this.allScanners.add(this.scanner);
     }
     const scanner = this.scanner;
@@ -6410,9 +6789,13 @@ var StreamerServer = class {
       return;
     }
     const etagSource = conversation;
+    const indexedCount = etagSource.filePath && this.cache ? this.cache.getIndexedMessageCount(
+      ConversationCache.conversationIdForFile(etagSource.filePath)
+    ) : 0;
+    const etagMessageCount = Math.max(etagSource.messageCount, indexedCount);
     const etag = computeConversationEtag({
       filePath: etagSource.filePath,
-      messageCount: etagSource.messageCount,
+      messageCount: etagMessageCount,
       timestamp: etagSource.timestamp
     });
     const isFirstPage = !url.searchParams.has("before_index") && !url.searchParams.has("anchor_index") && !url.searchParams.has("after_index");
@@ -6429,12 +6812,14 @@ var StreamerServer = class {
     let slice = filtered;
     let fromIdx = 0;
     let messagePagination;
+    let indexTotal = null;
     if (usePaging) {
       const limit = Math.min(Math.max(intParam(url, "msg_limit", 80), 1), 500);
       let beforeIndex = total;
       let scanLimit = limit;
       let anchorIndex = null;
       let newerPaging = false;
+      let usedAfterIndex = false;
       if (url.searchParams.has("before_index")) {
         beforeIndex = intParam(url, "before_index", total);
         beforeIndex = Math.min(Math.max(beforeIndex, 0), total);
@@ -6443,6 +6828,7 @@ var StreamerServer = class {
         beforeIndex = Math.min(total, from + limit);
         scanLimit = beforeIndex - from;
         newerPaging = true;
+        usedAfterIndex = true;
       } else if (hasAnchor) {
         anchorIndex = Math.min(
           Math.max(intParam(url, "anchor_index", 0), 0),
@@ -6452,9 +6838,34 @@ var StreamerServer = class {
         beforeIndex = Math.min(total, from + limit);
         newerPaging = true;
       }
-      const pagedScanner = this.scannerReady ? await this.getScanner(true) : null;
-      const page = scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null;
-      const start = page?.fromIndex ?? Math.max(0, beforeIndex - scanLimit);
+      const isTailRequest = !url.searchParams.has("before_index") && !hasAfter && !hasAnchor;
+      const indexFilePath = conversation.filePath;
+      if (isTailRequest && indexFilePath && this.cache) {
+        const indexed = this.cache.getIndexedMessageCount(
+          ConversationCache.conversationIdForFile(indexFilePath)
+        );
+        if (indexed > beforeIndex) {
+          beforeIndex = indexed;
+        }
+      }
+      const windowStart = Math.max(0, beforeIndex - scanLimit);
+      const indexWindow = scanLimit > 0 && !hasAnchor && indexFilePath && this.cache ? this.cache.readMessageWindow(indexFilePath, windowStart, beforeIndex) : null;
+      if (!indexWindow && indexFilePath && this.cache && !hasAnchor) {
+        this.trackCacheWrite(
+          this.cache.backfillIndex(indexFilePath).catch((err) => {
+            this.log.warn("offset-index.backfill_failed", {
+              event: "offset_index.backfill_failed",
+              conversationId: id,
+              filePath: indexFilePath,
+              err
+            });
+          })
+        );
+      }
+      const pagedScanner = !indexWindow && this.scannerReady ? await this.getScanner(true) : null;
+      const page = indexWindow ?? (scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null);
+      if (indexWindow) indexTotal = indexWindow.total;
+      const start = page?.fromIndex ?? windowStart;
       slice = page?.messages ?? filtered.slice(start, beforeIndex);
       fromIdx = start;
       const effectiveTotal = page?.total ?? total;
@@ -6470,6 +6881,9 @@ var StreamerServer = class {
         messagePagination.has_more_newer = beforeIndex < effectiveTotal;
         messagePagination.next_after_index = beforeIndex < effectiveTotal ? beforeIndex : null;
       }
+      if (usedAfterIndex) {
+        messagePagination.etag = etag;
+      }
     }
     const messagesPayload = slice.map((m, localIdx) => {
       const content = [];
@@ -6511,6 +6925,8 @@ var StreamerServer = class {
     const cachedConvMeta = this.cache?.getMetaById(id);
     const convProvider = coerceProviderForRunner(conv.provider ?? cachedConvMeta?.provider);
     const availability = classifyResumability(conv.projectPath);
+    const metaMessageCount = indexTotal != null && indexTotal > conv.messageCount ? indexTotal : conv.messageCount;
+    const metaLastUpdatedAt = indexTotal != null && indexTotal > conv.messageCount ? slice.at(-1)?.timestamp ?? conv.timestamp : conv.timestamp;
     const body = {
       meta: {
         id,
@@ -6518,8 +6934,8 @@ var StreamerServer = class {
         project_name: conv.projectName,
         project_path: conv.projectPath,
         file_path: conv.filePath,
-        last_updated_at: conv.timestamp,
-        message_count: conv.messageCount,
+        last_updated_at: metaLastUpdatedAt,
+        message_count: metaMessageCount,
         last_prompt: conv.lastPrompt ?? void 0,
         provider: convProvider,
         resumable: isProviderResumable(convProvider, availability.resumable),
@@ -6611,7 +7027,7 @@ var StreamerServer = class {
     }
     const limit = intParam(url, "limit", 50);
     const scanner = await this.getScanner();
-    const results = await (0, import_scanner2.search)(
+    const results = await (0, import_scanner3.search)(
       q,
       {
         limit,