@threadbase-sh/streamer 1.27.2 → 1.28.0

package/dist/index.js

@@ -3124,9 +3124,15 @@ async function createDirectory(parentAbsolutePath, name) {
 }
 
 // src/conversation-cache.ts
+import {
+  createJsonlParseState,
+  parseJsonlLine
+} from "@threadbase-sh/scanner";
 import Database from "better-sqlite3";
 import { closeSync as closeSync2, existsSync as existsSync5, mkdirSync as mkdirSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
+import { open as openAsync } from "fs/promises";
 import { dirname as dirname6 } from "path";
+import { setImmediate as yieldToEventLoop } from "timers/promises";
 
 // src/db/sqlite-migrate.ts
 import { readdirSync as readdirSync2, readFileSync as readFileSync5 } from "fs";
@@ -3238,6 +3244,33 @@ function parseAgentEntrypointsEnv(raw) {
   return new Set(parts);
 }
 
+// src/utils/fileIdentity.ts
+import { createHash } from "crypto";
+function fileIdentity(stat3, headBytes) {
+  if (stat3.ino && stat3.ino > 0) return `inode:${stat3.dev}:${stat3.ino}`;
+  const head = headBytes ?? Buffer.alloc(0);
+  return `fp:${createHash("sha1").update(head).digest("hex")}`;
+}
+function splitCompleteLines(buf, baseOffset) {
+  const spans = [];
+  let lineStart = 0;
+  let consumed = 0;
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] !== 10) continue;
+    const lineLen = i - lineStart;
+    if (lineLen > 0) {
+      spans.push({
+        byteOffset: baseOffset + lineStart,
+        byteLength: lineLen,
+        text: buf.toString("utf-8", lineStart, i)
+      });
+    }
+    lineStart = i + 1;
+    consumed = lineStart;
+  }
+  return { spans, consumed };
+}
+
 // src/conversation-cache.ts
 function shortProjectName(fullPath) {
   const parts = fullPath.split(/[/\\]/).filter(Boolean);
@@ -3286,6 +3319,14 @@ var ConversationCache = class _ConversationCache {
   tailSize;
   fileIndex = /* @__PURE__ */ new Map();
   fileIndexLoaded = false;
+  // Per-file scanner parse state for the incremental offset-index writer. The
+  // reducer is stateful across lines (pending tool_uses, latest timestamp), so
+  // it must persist between watcher reads of the same file. Cleared on
+  // truncation/backfill.
+  indexParseState = /* @__PURE__ */ new Map();
+  // Single-flight guard for backfillIndex — concurrent detail requests for the
+  // same cold file await one walk, not N. Entry dropped on settle.
+  backfillInFlight = /* @__PURE__ */ new Map();
   // Monotonically increasing counter for tail updated_at — guarantees strict
   // ordering even when multiple updateFromLine() calls land within the same ms.
   tailSeq = Date.now();
@@ -3449,6 +3490,42 @@ var ConversationCache = class _ConversationCache {
          GROUP BY project_path
          ORDER BY cnt DESC
          LIMIT ?`
+      ),
+      getFileState: db.prepare("SELECT * FROM conversation_file_state WHERE path = ?"),
+      upsertFileState: db.prepare(
+        `INSERT INTO conversation_file_state
+           (path, identity, size, mtime_ms, byte_offset, last_message_index)
+         VALUES (@path, @identity, @size, @mtime_ms, @byte_offset, @last_message_index)
+         ON CONFLICT(path) DO UPDATE SET
+           identity           = excluded.identity,
+           size               = excluded.size,
+           mtime_ms           = excluded.mtime_ms,
+           byte_offset        = excluded.byte_offset,
+           last_message_index = excluded.last_message_index`
+      ),
+      deleteFileState: db.prepare("DELETE FROM conversation_file_state WHERE path = ?"),
+      deleteMessageIndex: db.prepare(
+        "DELETE FROM conversation_message_index WHERE conversation_id = ?"
+      ),
+      insertMessageIndexRow: db.prepare(
+        `INSERT INTO conversation_message_index
+           (conversation_id, message_index, byte_offset, byte_length, uuid, role, ts)
+         VALUES (@conversation_id, @message_index, @byte_offset, @byte_length, @uuid, @role, @ts)
+         ON CONFLICT(conversation_id, message_index) DO UPDATE SET
+           byte_offset = excluded.byte_offset,
+           byte_length = excluded.byte_length,
+           uuid        = excluded.uuid,
+           role        = excluded.role,
+           ts          = excluded.ts`
+      ),
+      getMessageIndexWindow: db.prepare(
+        `SELECT message_index, byte_offset, byte_length, uuid, role, ts
+         FROM conversation_message_index
+         WHERE conversation_id = ? AND message_index >= ? AND message_index < ?
+         ORDER BY message_index ASC`
+      ),
+      getIndexedMessageCount: db.prepare(
+        "SELECT COUNT(*) as cnt FROM conversation_message_index WHERE conversation_id = ?"
       )
     };
   }
@@ -3459,6 +3536,258 @@ var ConversationCache = class _ConversationCache {
   getDatabase() {
     return this.db;
   }
+  // ── Offset index (design 1b) ────────────────────────────────────────────
+  // conversation_file_state + conversation_message_index back the windowed
+  // detail read path (SQL window select + pread of byte ranges). All methods
+  // are thin wrappers over the prepared statements above.
+  getFileState(path) {
+    return this.stmts.getFileState.get(path) ?? null;
+  }
+  upsertFileState(row) {
+    this.stmts.upsertFileState.run(row);
+  }
+  /** Drop a file's index rows + file_state (truncation / identity change). */
+  deleteFileIndex(path, conversationId) {
+    const tx = this.db.transaction(() => {
+      this.stmts.deleteMessageIndex.run(conversationId);
+      this.stmts.deleteFileState.run(path);
+    });
+    tx();
+    this.indexParseState.delete(path);
+  }
+  /** Append/replace index rows in one transaction. */
+  appendMessageIndexRows(rows) {
+    const tx = this.db.transaction((batch) => {
+      for (const r of batch) this.stmts.insertMessageIndexRow.run(r);
+    });
+    tx(rows);
+  }
+  /** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
+  getMessageIndexWindow(conversationId, fromIndex, toIndex) {
+    return this.stmts.getMessageIndexWindow.all(
+      conversationId,
+      fromIndex,
+      toIndex
+    );
+  }
+  getIndexedMessageCount(conversationId) {
+    return this.stmts.getIndexedMessageCount.get(conversationId).cnt;
+  }
+  /**
+   * Conversation id for a JSONL path — the filename stem (matches the pseudo-id
+   * updateFromLine derives and the uuid the detail read path resolves). The
+   * offset index keys on this so the window select and the cursor agree.
+   */
+  static conversationIdForFile(filePath) {
+    return filePath.split(/[/\\]/).pop()?.replace(/\.jsonl$/, "") ?? filePath;
+  }
+  /**
+   * Incremental offset-index writer: extend the index for a burst of appended
+   * lines (one watcher read) using their byte spans. Each line is classified
+   * with the scanner's parseJsonlLine (a running per-file reducer state), so the
+   * message ordering can never drift from the scanner's. Message lines get an
+   * index row at the next message_index; non-message lines (summary/sidecar)
+   * get no row but still advance byte_offset. file_state is updated to the end
+   * of the last consumed span.
+   *
+   * Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
+   * path can detect truncation/replacement.
+   *
+   * `readFrom` is the absolute byte offset the watcher read started at, and
+   * `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
+   * entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
+   * left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
+   * doesn't — the watcher attached at EOF after the server was down, or an
+   * append raced an in-flight backfill — extending would assign wrong
+   * message_index values over a hole. In that case this writes nothing and
+   * returns null so the caller drops the index and backfills.
+   *
+   * On success returns the message_index assigned to each input span (null for a
+   * non-message line) so the caller can stamp WS `seq`. Empty array when spans
+   * is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
+   * offset and file_state.byte_offset are the same number by construction.
+   */
+  extendMessageIndex(filePath, spans, stat3, readFrom, endOffset) {
+    const existing = this.getFileState(filePath);
+    const expectedStart = existing?.byte_offset ?? 0;
+    if (readFrom !== expectedStart) return null;
+    if (spans.length === 0) return [];
+    const convId = _ConversationCache.conversationIdForFile(filePath);
+    let state = this.indexParseState.get(filePath);
+    if (!state) {
+      state = createJsonlParseState();
+      this.indexParseState.set(filePath, state);
+    }
+    let nextIndex = existing ? existing.last_message_index + 1 : 0;
+    const rows = [];
+    const seqs = [];
+    for (const span of spans) {
+      const msg = parseJsonlLine(span.text, state);
+      if (!msg) {
+        seqs.push(null);
+        continue;
+      }
+      rows.push({
+        conversation_id: convId,
+        message_index: nextIndex,
+        byte_offset: span.byteOffset,
+        byte_length: span.byteLength,
+        uuid: msg.uuid ?? null,
+        role: msg.role ?? null,
+        ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
+      });
+      seqs.push(nextIndex);
+      nextIndex++;
+    }
+    const tx = this.db.transaction(() => {
+      for (const r of rows) this.stmts.insertMessageIndexRow.run(r);
+      this.stmts.upsertFileState.run({
+        path: filePath,
+        identity: fileIdentity(stat3),
+        size: stat3.size,
+        mtime_ms: Math.round(stat3.mtimeMs),
+        // Store the watcher's end offset verbatim — same number as entry.offset,
+        // so the next read's contiguity check compares like-for-like (never
+        // false-positive on a read that ended in trailing empty lines).
+        byte_offset: endOffset,
+        last_message_index: nextIndex - 1
+      });
+    });
+    tx();
+    return seqs;
+  }
+  clearIndexParseState(filePath) {
+    this.indexParseState.delete(filePath);
+  }
+  /**
+   * On-demand full backfill of the offset index for a file with no/stale
+   * file_state (cold conversation, or after a truncation/replacement). Rebuilds
+   * from byte 0: drops any existing rows, walks the whole file in chunks with a
+   * running parse state, yields to the event loop every ~1000 lines so a large
+   * file never blocks, and writes index rows + file_state.
+   *
+   * Single-flighted per path: concurrent callers await the same walk. The
+   * triggering detail request is served by the scanner fallback while this runs.
+   */
+  backfillIndex(filePath) {
+    const inFlight = this.backfillInFlight.get(filePath);
+    if (inFlight) return inFlight;
+    const walk = this.runBackfill(filePath).finally(() => {
+      this.backfillInFlight.delete(filePath);
+    });
+    this.backfillInFlight.set(filePath, walk);
+    return walk;
+  }
+  async runBackfill(filePath) {
+    const convId = _ConversationCache.conversationIdForFile(filePath);
+    this.deleteFileIndex(filePath, convId);
+    this.indexParseState.delete(filePath);
+    const CHUNK = 256 * 1024;
+    const YIELD_EVERY = 1e3;
+    const state = createJsonlParseState();
+    const fh = await openAsync(filePath, "r");
+    let fileOffset = 0;
+    let carry = Buffer.alloc(0);
+    let nextIndex = 0;
+    let linesSinceYield = 0;
+    let lastConsumedEnd = 0;
+    let stat3;
+    try {
+      stat3 = await fh.stat();
+      const buf = Buffer.alloc(CHUNK);
+      for (; ; ) {
+        const { bytesRead } = await fh.read(buf, 0, CHUNK, null);
+        if (bytesRead === 0) break;
+        const combined = carry.length > 0 ? Buffer.concat([carry, buf.subarray(0, bytesRead)]) : buf.subarray(0, bytesRead);
+        const { spans, consumed } = splitCompleteLines(combined, fileOffset);
+        const rows = [];
+        for (const span of spans) {
+          const msg = parseJsonlLine(span.text, state);
+          linesSinceYield++;
+          if (msg) {
+            rows.push({
+              conversation_id: convId,
+              message_index: nextIndex,
+              byte_offset: span.byteOffset,
+              byte_length: span.byteLength,
+              uuid: msg.uuid ?? null,
+              role: msg.role ?? null,
+              ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
+            });
+            nextIndex++;
+          }
+          if (linesSinceYield >= YIELD_EVERY) {
+            linesSinceYield = 0;
+            await yieldToEventLoop();
+          }
+        }
+        if (rows.length > 0) this.appendMessageIndexRows(rows);
+        lastConsumedEnd = fileOffset + consumed;
+        carry = Buffer.from(combined.subarray(consumed));
+        fileOffset += consumed;
+      }
+    } finally {
+      await fh.close();
+    }
+    this.upsertFileState({
+      path: filePath,
+      identity: fileIdentity(stat3),
+      size: stat3.size,
+      mtime_ms: Math.round(stat3.mtimeMs),
+      byte_offset: lastConsumedEnd,
+      last_message_index: nextIndex - 1
+    });
+    this.indexParseState.set(filePath, state);
+  }
+  /**
+   * Windowed detail read straight from the offset index — the hot path.
+   * Returns the parsed messages for message_index in [fromIndex, toIndex) plus
+   * the total indexed count, or null when the index can't serve this file (no
+   * file_state, identity/size mismatch = truncation/replacement, or cold index)
+   * so the caller falls back to the scanner and enqueues a backfill.
+   *
+   * On a match it SQL-selects the window's byte ranges and preads exactly those
+   * ranges from the JSONL (never the whole file), parsing only the sliced lines.
+   * Returns messages in the same ConversationMessage shape parseJsonlLine
+   * produces during a scan, so the payload is identical to the scanner path.
+   */
+  readMessageWindow(filePath, fromIndex, toIndex) {
+    const fileState = this.getFileState(filePath);
+    if (!fileState) return null;
+    let stat3;
+    try {
+      stat3 = statSync2(filePath);
+    } catch {
+      return null;
+    }
+    if (fileIdentity(stat3) !== fileState.identity || stat3.size !== fileState.byte_offset) {
+      return null;
+    }
+    const total = fileState.last_message_index + 1;
+    const from = Math.max(0, fromIndex);
+    const to = Math.min(toIndex, total);
+    if (to <= from) return { messages: [], total, fromIndex: from };
+    const rows = this.getMessageIndexWindow(
+      _ConversationCache.conversationIdForFile(filePath),
+      from,
+      to
+    );
+    if (rows.length === 0) return { messages: [], total, fromIndex: from };
+    const messages = [];
+    const fd = openSync2(filePath, "r");
+    try {
+      const state = createJsonlParseState();
+      for (const row of rows) {
+        const buf = Buffer.alloc(row.byte_length);
+        readSync2(fd, buf, 0, row.byte_length, row.byte_offset);
+        const msg = parseJsonlLine(buf.toString("utf-8"), state);
+        if (msg) messages.push(msg);
+      }
+    } finally {
+      closeSync2(fd);
+    }
+    return { messages, total, fromIndex: from };
+  }
   agentEntrypointsKey() {
     return [...this.agentEntrypoints].sort().join(",");
   }
@@ -4359,12 +4688,14 @@ var ConversationWatcher = class {
   directories = /* @__PURE__ */ new Map();
   onNewLine;
   onNewLines;
+  onNewLineSpans;
   onConversationChanged;
   onFileDeleted;
   onError;
   constructor(events = {}) {
     this.onNewLine = events.onNewLine;
     this.onNewLines = events.onNewLines;
+    this.onNewLineSpans = events.onNewLineSpans;
     this.onConversationChanged = events.onConversationChanged;
     this.onFileDeleted = events.onFileDeleted;
     this.onError = events.onError;
@@ -4468,9 +4799,13 @@ var ConversationWatcher = class {
         } finally {
           await fh.close();
         }
-        entry.offset = readFrom + bytesToRead;
+        const { spans, consumed } = splitCompleteLines(buf, readFrom);
+        entry.offset = readFrom + consumed;
         if (!this.files.has(filePath)) return;
-        const lines = buf.toString("utf-8").split("\n").filter(Boolean);
+        const lines = spans.map((s) => s.text);
+        if (spans.length > 0) {
+          this.onNewLineSpans?.(filePath, spans, readFrom, entry.offset);
+        }
         if (this.onNewLines) {
           this.onNewLines(filePath, lines);
         } else {
@@ -4939,13 +5274,13 @@ function sanitizeFilename(name) {
 }
 
 // src/utils/conversationEtag.ts
-import { createHash } from "crypto";
+import { createHash as createHash2 } from "crypto";
 function computeConversationEtag({
   filePath,
   messageCount,
   timestamp
 }) {
-  const digest = createHash("sha1").update(`${filePath}:${messageCount}:${timestamp}`).digest("hex").slice(0, 16);
+  const digest = createHash2("sha1").update(`${filePath}:${messageCount}:${timestamp}`).digest("hex").slice(0, 16);
   return `"${digest}"`;
 }
 
@@ -5095,6 +5430,7 @@ var WSHub = class {
 var BROWSE_SYSTEM_PROMPT = (browseRoot) => `You are working within the project boundary: ${browseRoot}. Do not read, write, or execute commands that access files or directories outside this boundary.`;
 var DEFAULT_SYSTEM_PROMPT = "When presenting options or choices to the user, limit the options to at most 3.";
 var DEFAULT_PTY_GRACE_PERIOD_MS = 27e4;
+var REFRESH_TTL_MS = 2e3;
 var START_READY_TIMEOUT_MS = 15e3;
 function parseIncludeAgentsEnv(raw) {
   if (raw === void 0) return false;
@@ -5109,6 +5445,10 @@ var StreamerServer = class {
   fileWatcher;
   sessionFileMap = /* @__PURE__ */ new Map();
   // sessionId → JSONL filePath
+  // Per-file seq assignments from the most recent onNewLineSpans (offset index),
+  // handed to the immediately-following onNewLines so it can stamp WS `seq` on
+  // the matching conversation_events entries. Same read → same lines order.
+  pendingLineSeqs = /* @__PURE__ */ new Map();
   pendingQuestions = /* @__PURE__ */ new Map();
   // Content key of the AskUserQuestion currently broadcast for a session (from
   // either the rendered screen or JSONL), used to de-dupe the two paths: when
@@ -5134,6 +5474,12 @@ var StreamerServer = class {
   // Set by onConversationChanged while a scan is in-flight; getScanner() does
   // a single rescan after the current one completes instead of restarting it.
   scannerStale = false;
+  // Single-flight + TTL guard around scanner.refreshFile (see refreshFileGuarded).
+  // A live file's mtime is always newer than the snapshot, so an unguarded
+  // refresh fires on every request and re-parses the whole file from byte 0.
+  // Keyed by filePath; entries drop on settle + TTL expiry, so the map stays
+  // bounded by the active-file set.
+  refreshInFlight = /* @__PURE__ */ new Map();
   // True only while bindWithRetry is actively retrying. The persistent
   // listener-level 'error' handler demotes EADDRINUSE to debug during this
   // window so the self-healing kickstart-relaunch race doesn't spam warn.
@@ -5248,6 +5594,42 @@ var StreamerServer = class {
     this.sessionStore = new SessionStore();
     this.wsHub = new WSHub();
     this.fileWatcher = new ConversationWatcher({
+      onNewLineSpans: (filePath, spans, readFrom, endOffset) => {
+        if (!this.cache) return;
+        const cache = this.cache;
+        this.pendingLineSeqs.delete(filePath);
+        try {
+          const seqs = cache.extendMessageIndex(
+            filePath,
+            spans,
+            statSync5(filePath),
+            readFrom,
+            endOffset
+          );
+          if (seqs === null) {
+            cache.deleteFileIndex(filePath, ConversationCache.conversationIdForFile(filePath));
+            cache.clearIndexParseState(filePath);
+            this.trackCacheWrite(
+              cache.backfillIndex(filePath).catch((err) => {
+                this.log.warn("offset-index.backfill_failed", {
+                  event: "offset_index.backfill_failed",
+                  filePath,
+                  trigger: "noncontiguous-append",
+                  err
+                });
+              })
+            );
+            return;
+          }
+          this.pendingLineSeqs.set(filePath, seqs);
+        } catch (err) {
+          this.log.warn("offset-index.extend_failed", {
+            event: "offset_index.extend_failed",
+            filePath,
+            err
+          });
+        }
+      },
       onNewLines: (filePath, lines) => {
         this.cache?.updateFromLines(filePath, lines);
         for (const [sessionId, watchedPath] of this.sessionFileMap) {
@@ -5274,13 +5656,20 @@ var StreamerServer = class {
               this.pendingQuestionKey.set(sessionId, key);
               if (broadcast) this.wsHub.broadcast(m);
             }
-            this.wsHub.broadcast({ type: "conversation_events", sessionId, lines });
+            const seqs = this.pendingLineSeqs.get(filePath);
+            this.wsHub.broadcast({
+              type: "conversation_events",
+              sessionId,
+              lines,
+              ...seqs && seqs.length === lines.length ? { seqs } : {}
+            });
             for (const line of lines) {
               this.wsHub.broadcast({ type: "conversation_event", sessionId, line });
             }
             break;
           }
         }
+        this.pendingLineSeqs.delete(filePath);
       },
       onConversationChanged: (filePath) => {
         this.fileWatcher.poke(filePath);
@@ -5332,7 +5721,7 @@ var StreamerServer = class {
         if (session.status === "waiting_input" || session.status === "idle") {
           const filePath = this.sessionFileMap.get(session.id);
           if (filePath) {
-            this.getScanner().then((scanner) => scanner.refreshFile(filePath)).then((meta) => {
+            this.getScanner().then((scanner) => this.refreshFileGuarded(scanner, filePath)).then((meta) => {
               this.log.info("scanner.refreshFile: ok", {
                 event: "scanner.refresh",
                 sessionId: session.id,
@@ -5781,6 +6170,34 @@ var StreamerServer = class {
       this.inFlightCacheWrites.delete(guarded);
     });
   }
+  // Single-flight + TTL wrapper for scanner.refreshFile. Both call sites (the
+  // detail-stale branch and the per-turn refresh) route through here so that
+  // N stacked retries on a live, actively-appended file cost one parse, not N:
+  //  - a refresh already in flight for the path → await the same promise;
+  //  - a refresh that settled within REFRESH_TTL_MS → skip, return null (the
+  //    caller keeps serving the current snapshot);
+  //  - otherwise start one, cache the promise, and stamp completedAt on settle.
+  // The map is bounded by the active-file set (entries only live while a
+  // refresh is in flight or within its TTL and are overwritten on the next
+  // refresh of the same path).
+  refreshFileGuarded(scanner, filePath) {
+    const existing = this.refreshInFlight.get(filePath);
+    if (existing) {
+      const settled = existing.completedAt > 0;
+      if (!settled) {
+        return existing.promise;
+      }
+      if (Date.now() - existing.completedAt < REFRESH_TTL_MS) {
+        return Promise.resolve(null);
+      }
+    }
+    const entry = { promise: Promise.resolve(null), completedAt: 0 };
+    entry.promise = scanner.refreshFile(filePath).finally(() => {
+      entry.completedAt = Date.now();
+    });
+    this.refreshInFlight.set(filePath, entry);
+    return entry.promise;
+  }
   async close() {
     for (const timer of this.ptyGraceTimers.values()) clearTimeout(timer);
     this.ptyGraceTimers.clear();
@@ -6245,10 +6662,22 @@ var StreamerServer = class {
     const scanner = await this.getScanner(true);
     const fromIndex = await scanner.getConversation(uuid);
     if (fromIndex) {
+      if (this.ptyManager.hasSession(uuid)) {
+        return fromIndex;
+      }
       if (fromIndex.filePath && this.isConversationSnapshotStale(fromIndex)) {
-        const refreshedMeta = await scanner.refreshFile(fromIndex.filePath);
-        if (!refreshedMeta) return null;
-        return await scanner.getConversation(uuid) ?? fromIndex;
+        const filePath2 = fromIndex.filePath;
+        this.trackCacheWrite(
+          this.refreshFileGuarded(scanner, filePath2).catch((err) => {
+            this.log.warn("scanner.refreshFile: failed", {
+              event: "scanner.refresh_failed",
+              conversationId: uuid,
+              filePath: filePath2,
+              trigger: "detail-swr",
+              err
+            });
+          })
+        );
       }
       return fromIndex;
     }
@@ -6324,9 +6753,13 @@ var StreamerServer = class {
       return;
     }
     const etagSource = conversation;
+    const indexedCount = etagSource.filePath && this.cache ? this.cache.getIndexedMessageCount(
+      ConversationCache.conversationIdForFile(etagSource.filePath)
+    ) : 0;
+    const etagMessageCount = Math.max(etagSource.messageCount, indexedCount);
     const etag = computeConversationEtag({
       filePath: etagSource.filePath,
-      messageCount: etagSource.messageCount,
+      messageCount: etagMessageCount,
       timestamp: etagSource.timestamp
     });
     const isFirstPage = !url.searchParams.has("before_index") && !url.searchParams.has("anchor_index") && !url.searchParams.has("after_index");
@@ -6343,12 +6776,14 @@ var StreamerServer = class {
     let slice = filtered;
     let fromIdx = 0;
     let messagePagination;
+    let indexTotal = null;
     if (usePaging) {
       const limit = Math.min(Math.max(intParam(url, "msg_limit", 80), 1), 500);
       let beforeIndex = total;
       let scanLimit = limit;
       let anchorIndex = null;
       let newerPaging = false;
+      let usedAfterIndex = false;
       if (url.searchParams.has("before_index")) {
         beforeIndex = intParam(url, "before_index", total);
         beforeIndex = Math.min(Math.max(beforeIndex, 0), total);
@@ -6357,6 +6792,7 @@ var StreamerServer = class {
         beforeIndex = Math.min(total, from + limit);
         scanLimit = beforeIndex - from;
         newerPaging = true;
+        usedAfterIndex = true;
       } else if (hasAnchor) {
         anchorIndex = Math.min(
           Math.max(intParam(url, "anchor_index", 0), 0),
@@ -6366,9 +6802,34 @@ var StreamerServer = class {
         beforeIndex = Math.min(total, from + limit);
         newerPaging = true;
       }
-      const pagedScanner = this.scannerReady ? await this.getScanner(true) : null;
-      const page = scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null;
-      const start = page?.fromIndex ?? Math.max(0, beforeIndex - scanLimit);
+      const isTailRequest = !url.searchParams.has("before_index") && !hasAfter && !hasAnchor;
+      const indexFilePath = conversation.filePath;
+      if (isTailRequest && indexFilePath && this.cache) {
+        const indexed = this.cache.getIndexedMessageCount(
+          ConversationCache.conversationIdForFile(indexFilePath)
+        );
+        if (indexed > beforeIndex) {
+          beforeIndex = indexed;
+        }
+      }
+      const windowStart = Math.max(0, beforeIndex - scanLimit);
+      const indexWindow = scanLimit > 0 && !hasAnchor && indexFilePath && this.cache ? this.cache.readMessageWindow(indexFilePath, windowStart, beforeIndex) : null;
+      if (!indexWindow && indexFilePath && this.cache && !hasAnchor) {
+        this.trackCacheWrite(
+          this.cache.backfillIndex(indexFilePath).catch((err) => {
+            this.log.warn("offset-index.backfill_failed", {
+              event: "offset_index.backfill_failed",
+              conversationId: id,
+              filePath: indexFilePath,
+              err
+            });
+          })
+        );
+      }
+      const pagedScanner = !indexWindow && this.scannerReady ? await this.getScanner(true) : null;
+      const page = indexWindow ?? (scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null);
+      if (indexWindow) indexTotal = indexWindow.total;
+      const start = page?.fromIndex ?? windowStart;
       slice = page?.messages ?? filtered.slice(start, beforeIndex);
       fromIdx = start;
       const effectiveTotal = page?.total ?? total;
@@ -6384,6 +6845,9 @@ var StreamerServer = class {
         messagePagination.has_more_newer = beforeIndex < effectiveTotal;
         messagePagination.next_after_index = beforeIndex < effectiveTotal ? beforeIndex : null;
       }
+      if (usedAfterIndex) {
+        messagePagination.etag = etag;
+      }
     }
     const messagesPayload = slice.map((m, localIdx) => {
       const content = [];
@@ -6425,6 +6889,8 @@ var StreamerServer = class {
     const cachedConvMeta = this.cache?.getMetaById(id);
     const convProvider = coerceProviderForRunner(conv.provider ?? cachedConvMeta?.provider);
     const availability = classifyResumability(conv.projectPath);
+    const metaMessageCount = indexTotal != null && indexTotal > conv.messageCount ? indexTotal : conv.messageCount;
+    const metaLastUpdatedAt = indexTotal != null && indexTotal > conv.messageCount ? slice.at(-1)?.timestamp ?? conv.timestamp : conv.timestamp;
     const body = {
       meta: {
         id,
@@ -6432,8 +6898,8 @@ var StreamerServer = class {
         project_name: conv.projectName,
         project_path: conv.projectPath,
         file_path: conv.filePath,
-        last_updated_at: conv.timestamp,
-        message_count: conv.messageCount,
+        last_updated_at: metaLastUpdatedAt,
+        message_count: metaMessageCount,
         last_prompt: conv.lastPrompt ?? void 0,
         provider: convProvider,
         resumable: isProviderResumable(convProvider, availability.resumable),