@threadbase-sh/streamer 1.27.3 → 1.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3124,9 +3124,15 @@ async function createDirectory(parentAbsolutePath, name) {
3124
3124
  }
3125
3125
 
3126
3126
  // src/conversation-cache.ts
3127
+ import {
3128
+ createJsonlParseState,
3129
+ parseJsonlLine
3130
+ } from "@threadbase-sh/scanner";
3127
3131
  import Database from "better-sqlite3";
3128
3132
  import { closeSync as closeSync2, existsSync as existsSync5, mkdirSync as mkdirSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
3133
+ import { open as openAsync } from "fs/promises";
3129
3134
  import { dirname as dirname6 } from "path";
3135
+ import { setImmediate as yieldToEventLoop } from "timers/promises";
3130
3136
 
3131
3137
  // src/db/sqlite-migrate.ts
3132
3138
  import { readdirSync as readdirSync2, readFileSync as readFileSync5 } from "fs";
@@ -3238,6 +3244,33 @@ function parseAgentEntrypointsEnv(raw) {
3238
3244
  return new Set(parts);
3239
3245
  }
3240
3246
 
3247
+ // src/utils/fileIdentity.ts
3248
+ import { createHash } from "crypto";
3249
+ function fileIdentity(stat3, headBytes) {
3250
+ if (stat3.ino && stat3.ino > 0) return `inode:${stat3.dev}:${stat3.ino}`;
3251
+ const head = headBytes ?? Buffer.alloc(0);
3252
+ return `fp:${createHash("sha1").update(head).digest("hex")}`;
3253
+ }
3254
+ function splitCompleteLines(buf, baseOffset) {
3255
+ const spans = [];
3256
+ let lineStart = 0;
3257
+ let consumed = 0;
3258
+ for (let i = 0; i < buf.length; i++) {
3259
+ if (buf[i] !== 10) continue;
3260
+ const lineLen = i - lineStart;
3261
+ if (lineLen > 0) {
3262
+ spans.push({
3263
+ byteOffset: baseOffset + lineStart,
3264
+ byteLength: lineLen,
3265
+ text: buf.toString("utf-8", lineStart, i)
3266
+ });
3267
+ }
3268
+ lineStart = i + 1;
3269
+ consumed = lineStart;
3270
+ }
3271
+ return { spans, consumed };
3272
+ }
3273
+
3241
3274
  // src/conversation-cache.ts
3242
3275
  function shortProjectName(fullPath) {
3243
3276
  const parts = fullPath.split(/[/\\]/).filter(Boolean);
@@ -3286,6 +3319,14 @@ var ConversationCache = class _ConversationCache {
3286
3319
  tailSize;
3287
3320
  fileIndex = /* @__PURE__ */ new Map();
3288
3321
  fileIndexLoaded = false;
3322
+ // Per-file scanner parse state for the incremental offset-index writer. The
3323
+ // reducer is stateful across lines (pending tool_uses, latest timestamp), so
3324
+ // it must persist between watcher reads of the same file. Cleared on
3325
+ // truncation/backfill.
3326
+ indexParseState = /* @__PURE__ */ new Map();
3327
+ // Single-flight guard for backfillIndex — concurrent detail requests for the
3328
+ // same cold file await one walk, not N. Entry dropped on settle.
3329
+ backfillInFlight = /* @__PURE__ */ new Map();
3289
3330
  // Monotonically increasing counter for tail updated_at — guarantees strict
3290
3331
  // ordering even when multiple updateFromLine() calls land within the same ms.
3291
3332
  tailSeq = Date.now();
@@ -3449,6 +3490,42 @@ var ConversationCache = class _ConversationCache {
3449
3490
  GROUP BY project_path
3450
3491
  ORDER BY cnt DESC
3451
3492
  LIMIT ?`
3493
+ ),
3494
+ getFileState: db.prepare("SELECT * FROM conversation_file_state WHERE path = ?"),
3495
+ upsertFileState: db.prepare(
3496
+ `INSERT INTO conversation_file_state
3497
+ (path, identity, size, mtime_ms, byte_offset, last_message_index)
3498
+ VALUES (@path, @identity, @size, @mtime_ms, @byte_offset, @last_message_index)
3499
+ ON CONFLICT(path) DO UPDATE SET
3500
+ identity = excluded.identity,
3501
+ size = excluded.size,
3502
+ mtime_ms = excluded.mtime_ms,
3503
+ byte_offset = excluded.byte_offset,
3504
+ last_message_index = excluded.last_message_index`
3505
+ ),
3506
+ deleteFileState: db.prepare("DELETE FROM conversation_file_state WHERE path = ?"),
3507
+ deleteMessageIndex: db.prepare(
3508
+ "DELETE FROM conversation_message_index WHERE conversation_id = ?"
3509
+ ),
3510
+ insertMessageIndexRow: db.prepare(
3511
+ `INSERT INTO conversation_message_index
3512
+ (conversation_id, message_index, byte_offset, byte_length, uuid, role, ts)
3513
+ VALUES (@conversation_id, @message_index, @byte_offset, @byte_length, @uuid, @role, @ts)
3514
+ ON CONFLICT(conversation_id, message_index) DO UPDATE SET
3515
+ byte_offset = excluded.byte_offset,
3516
+ byte_length = excluded.byte_length,
3517
+ uuid = excluded.uuid,
3518
+ role = excluded.role,
3519
+ ts = excluded.ts`
3520
+ ),
3521
+ getMessageIndexWindow: db.prepare(
3522
+ `SELECT message_index, byte_offset, byte_length, uuid, role, ts
3523
+ FROM conversation_message_index
3524
+ WHERE conversation_id = ? AND message_index >= ? AND message_index < ?
3525
+ ORDER BY message_index ASC`
3526
+ ),
3527
+ getIndexedMessageCount: db.prepare(
3528
+ "SELECT COUNT(*) as cnt FROM conversation_message_index WHERE conversation_id = ?"
3452
3529
  )
3453
3530
  };
3454
3531
  }
@@ -3459,6 +3536,258 @@ var ConversationCache = class _ConversationCache {
3459
3536
  getDatabase() {
3460
3537
  return this.db;
3461
3538
  }
3539
+ // ── Offset index (design 1b) ────────────────────────────────────────────
3540
+ // conversation_file_state + conversation_message_index back the windowed
3541
+ // detail read path (SQL window select + pread of byte ranges). All methods
3542
+ // are thin wrappers over the prepared statements above.
3543
+ getFileState(path) {
3544
+ return this.stmts.getFileState.get(path) ?? null;
3545
+ }
3546
+ upsertFileState(row) {
3547
+ this.stmts.upsertFileState.run(row);
3548
+ }
3549
+ /** Drop a file's index rows + file_state (truncation / identity change). */
3550
+ deleteFileIndex(path, conversationId) {
3551
+ const tx = this.db.transaction(() => {
3552
+ this.stmts.deleteMessageIndex.run(conversationId);
3553
+ this.stmts.deleteFileState.run(path);
3554
+ });
3555
+ tx();
3556
+ this.indexParseState.delete(path);
3557
+ }
3558
+ /** Append/replace index rows in one transaction. */
3559
+ appendMessageIndexRows(rows) {
3560
+ const tx = this.db.transaction((batch) => {
3561
+ for (const r of batch) this.stmts.insertMessageIndexRow.run(r);
3562
+ });
3563
+ tx(rows);
3564
+ }
3565
+ /** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
3566
+ getMessageIndexWindow(conversationId, fromIndex, toIndex) {
3567
+ return this.stmts.getMessageIndexWindow.all(
3568
+ conversationId,
3569
+ fromIndex,
3570
+ toIndex
3571
+ );
3572
+ }
3573
+ getIndexedMessageCount(conversationId) {
3574
+ return this.stmts.getIndexedMessageCount.get(conversationId).cnt;
3575
+ }
3576
+ /**
3577
+ * Conversation id for a JSONL path — the filename stem (matches the pseudo-id
3578
+ * updateFromLine derives and the uuid the detail read path resolves). The
3579
+ * offset index keys on this so the window select and the cursor agree.
3580
+ */
3581
+ static conversationIdForFile(filePath) {
3582
+ return filePath.split(/[/\\]/).pop()?.replace(/\.jsonl$/, "") ?? filePath;
3583
+ }
3584
+ /**
3585
+ * Incremental offset-index writer: extend the index for a burst of appended
3586
+ * lines (one watcher read) using their byte spans. Each line is classified
3587
+ * with the scanner's parseJsonlLine (a running per-file reducer state), so the
3588
+ * message ordering can never drift from the scanner's. Message lines get an
3589
+ * index row at the next message_index; non-message lines (summary/sidecar)
3590
+ * get no row but still advance byte_offset. file_state is updated to the end
3591
+ * of the last consumed span.
3592
+ *
3593
+ * Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
3594
+ * path can detect truncation/replacement.
3595
+ *
3596
+ * `readFrom` is the absolute byte offset the watcher read started at, and
3597
+ * `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
3598
+ * entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
3599
+ * left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
3600
+ * doesn't — the watcher attached at EOF after the server was down, or an
3601
+ * append raced an in-flight backfill — extending would assign wrong
3602
+ * message_index values over a hole. In that case this writes nothing and
3603
+ * returns null so the caller drops the index and backfills.
3604
+ *
3605
+ * On success returns the message_index assigned to each input span (null for a
3606
+ * non-message line) so the caller can stamp WS `seq`. Empty array when spans
3607
+ * is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
3608
+ * offset and file_state.byte_offset are the same number by construction.
3609
+ */
3610
+ extendMessageIndex(filePath, spans, stat3, readFrom, endOffset) {
3611
+ const existing = this.getFileState(filePath);
3612
+ const expectedStart = existing?.byte_offset ?? 0;
3613
+ if (readFrom !== expectedStart) return null;
3614
+ if (spans.length === 0) return [];
3615
+ const convId = _ConversationCache.conversationIdForFile(filePath);
3616
+ let state = this.indexParseState.get(filePath);
3617
+ if (!state) {
3618
+ state = createJsonlParseState();
3619
+ this.indexParseState.set(filePath, state);
3620
+ }
3621
+ let nextIndex = existing ? existing.last_message_index + 1 : 0;
3622
+ const rows = [];
3623
+ const seqs = [];
3624
+ for (const span of spans) {
3625
+ const msg = parseJsonlLine(span.text, state);
3626
+ if (!msg) {
3627
+ seqs.push(null);
3628
+ continue;
3629
+ }
3630
+ rows.push({
3631
+ conversation_id: convId,
3632
+ message_index: nextIndex,
3633
+ byte_offset: span.byteOffset,
3634
+ byte_length: span.byteLength,
3635
+ uuid: msg.uuid ?? null,
3636
+ role: msg.role ?? null,
3637
+ ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
3638
+ });
3639
+ seqs.push(nextIndex);
3640
+ nextIndex++;
3641
+ }
3642
+ const tx = this.db.transaction(() => {
3643
+ for (const r of rows) this.stmts.insertMessageIndexRow.run(r);
3644
+ this.stmts.upsertFileState.run({
3645
+ path: filePath,
3646
+ identity: fileIdentity(stat3),
3647
+ size: stat3.size,
3648
+ mtime_ms: Math.round(stat3.mtimeMs),
3649
+ // Store the watcher's end offset verbatim — same number as entry.offset,
3650
+ // so the next read's contiguity check compares like-for-like (never
3651
+ // false-positive on a read that ended in trailing empty lines).
3652
+ byte_offset: endOffset,
3653
+ last_message_index: nextIndex - 1
3654
+ });
3655
+ });
3656
+ tx();
3657
+ return seqs;
3658
+ }
3659
+ clearIndexParseState(filePath) {
3660
+ this.indexParseState.delete(filePath);
3661
+ }
3662
+ /**
3663
+ * On-demand full backfill of the offset index for a file with no/stale
3664
+ * file_state (cold conversation, or after a truncation/replacement). Rebuilds
3665
+ * from byte 0: drops any existing rows, walks the whole file in chunks with a
3666
+ * running parse state, yields to the event loop every ~1000 lines so a large
3667
+ * file never blocks, and writes index rows + file_state.
3668
+ *
3669
+ * Single-flighted per path: concurrent callers await the same walk. The
3670
+ * triggering detail request is served by the scanner fallback while this runs.
3671
+ */
3672
+ backfillIndex(filePath) {
3673
+ const inFlight = this.backfillInFlight.get(filePath);
3674
+ if (inFlight) return inFlight;
3675
+ const walk = this.runBackfill(filePath).finally(() => {
3676
+ this.backfillInFlight.delete(filePath);
3677
+ });
3678
+ this.backfillInFlight.set(filePath, walk);
3679
+ return walk;
3680
+ }
3681
+ async runBackfill(filePath) {
3682
+ const convId = _ConversationCache.conversationIdForFile(filePath);
3683
+ this.deleteFileIndex(filePath, convId);
3684
+ this.indexParseState.delete(filePath);
3685
+ const CHUNK = 256 * 1024;
3686
+ const YIELD_EVERY = 1e3;
3687
+ const state = createJsonlParseState();
3688
+ const fh = await openAsync(filePath, "r");
3689
+ let fileOffset = 0;
3690
+ let carry = Buffer.alloc(0);
3691
+ let nextIndex = 0;
3692
+ let linesSinceYield = 0;
3693
+ let lastConsumedEnd = 0;
3694
+ let stat3;
3695
+ try {
3696
+ stat3 = await fh.stat();
3697
+ const buf = Buffer.alloc(CHUNK);
3698
+ for (; ; ) {
3699
+ const { bytesRead } = await fh.read(buf, 0, CHUNK, null);
3700
+ if (bytesRead === 0) break;
3701
+ const combined = carry.length > 0 ? Buffer.concat([carry, buf.subarray(0, bytesRead)]) : buf.subarray(0, bytesRead);
3702
+ const { spans, consumed } = splitCompleteLines(combined, fileOffset);
3703
+ const rows = [];
3704
+ for (const span of spans) {
3705
+ const msg = parseJsonlLine(span.text, state);
3706
+ linesSinceYield++;
3707
+ if (msg) {
3708
+ rows.push({
3709
+ conversation_id: convId,
3710
+ message_index: nextIndex,
3711
+ byte_offset: span.byteOffset,
3712
+ byte_length: span.byteLength,
3713
+ uuid: msg.uuid ?? null,
3714
+ role: msg.role ?? null,
3715
+ ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
3716
+ });
3717
+ nextIndex++;
3718
+ }
3719
+ if (linesSinceYield >= YIELD_EVERY) {
3720
+ linesSinceYield = 0;
3721
+ await yieldToEventLoop();
3722
+ }
3723
+ }
3724
+ if (rows.length > 0) this.appendMessageIndexRows(rows);
3725
+ lastConsumedEnd = fileOffset + consumed;
3726
+ carry = Buffer.from(combined.subarray(consumed));
3727
+ fileOffset += consumed;
3728
+ }
3729
+ } finally {
3730
+ await fh.close();
3731
+ }
3732
+ this.upsertFileState({
3733
+ path: filePath,
3734
+ identity: fileIdentity(stat3),
3735
+ size: stat3.size,
3736
+ mtime_ms: Math.round(stat3.mtimeMs),
3737
+ byte_offset: lastConsumedEnd,
3738
+ last_message_index: nextIndex - 1
3739
+ });
3740
+ this.indexParseState.set(filePath, state);
3741
+ }
3742
+ /**
3743
+ * Windowed detail read straight from the offset index — the hot path.
3744
+ * Returns the parsed messages for message_index in [fromIndex, toIndex) plus
3745
+ * the total indexed count, or null when the index can't serve this file (no
3746
+ * file_state, identity/size mismatch = truncation/replacement, or cold index)
3747
+ * so the caller falls back to the scanner and enqueues a backfill.
3748
+ *
3749
+ * On a match it SQL-selects the window's byte ranges and preads exactly those
3750
+ * ranges from the JSONL (never the whole file), parsing only the sliced lines.
3751
+ * Returns messages in the same ConversationMessage shape parseJsonlLine
3752
+ * produces during a scan, so the payload is identical to the scanner path.
3753
+ */
3754
+ readMessageWindow(filePath, fromIndex, toIndex) {
3755
+ const fileState = this.getFileState(filePath);
3756
+ if (!fileState) return null;
3757
+ let stat3;
3758
+ try {
3759
+ stat3 = statSync2(filePath);
3760
+ } catch {
3761
+ return null;
3762
+ }
3763
+ if (fileIdentity(stat3) !== fileState.identity || stat3.size !== fileState.byte_offset) {
3764
+ return null;
3765
+ }
3766
+ const total = fileState.last_message_index + 1;
3767
+ const from = Math.max(0, fromIndex);
3768
+ const to = Math.min(toIndex, total);
3769
+ if (to <= from) return { messages: [], total, fromIndex: from };
3770
+ const rows = this.getMessageIndexWindow(
3771
+ _ConversationCache.conversationIdForFile(filePath),
3772
+ from,
3773
+ to
3774
+ );
3775
+ if (rows.length === 0) return { messages: [], total, fromIndex: from };
3776
+ const messages = [];
3777
+ const fd = openSync2(filePath, "r");
3778
+ try {
3779
+ const state = createJsonlParseState();
3780
+ for (const row of rows) {
3781
+ const buf = Buffer.alloc(row.byte_length);
3782
+ readSync2(fd, buf, 0, row.byte_length, row.byte_offset);
3783
+ const msg = parseJsonlLine(buf.toString("utf-8"), state);
3784
+ if (msg) messages.push(msg);
3785
+ }
3786
+ } finally {
3787
+ closeSync2(fd);
3788
+ }
3789
+ return { messages, total, fromIndex: from };
3790
+ }
3462
3791
  agentEntrypointsKey() {
3463
3792
  return [...this.agentEntrypoints].sort().join(",");
3464
3793
  }
@@ -4359,12 +4688,14 @@ var ConversationWatcher = class {
4359
4688
  directories = /* @__PURE__ */ new Map();
4360
4689
  onNewLine;
4361
4690
  onNewLines;
4691
+ onNewLineSpans;
4362
4692
  onConversationChanged;
4363
4693
  onFileDeleted;
4364
4694
  onError;
4365
4695
  constructor(events = {}) {
4366
4696
  this.onNewLine = events.onNewLine;
4367
4697
  this.onNewLines = events.onNewLines;
4698
+ this.onNewLineSpans = events.onNewLineSpans;
4368
4699
  this.onConversationChanged = events.onConversationChanged;
4369
4700
  this.onFileDeleted = events.onFileDeleted;
4370
4701
  this.onError = events.onError;
@@ -4468,9 +4799,13 @@ var ConversationWatcher = class {
4468
4799
  } finally {
4469
4800
  await fh.close();
4470
4801
  }
4471
- entry.offset = readFrom + bytesToRead;
4802
+ const { spans, consumed } = splitCompleteLines(buf, readFrom);
4803
+ entry.offset = readFrom + consumed;
4472
4804
  if (!this.files.has(filePath)) return;
4473
- const lines = buf.toString("utf-8").split("\n").filter(Boolean);
4805
+ const lines = spans.map((s) => s.text);
4806
+ if (spans.length > 0) {
4807
+ this.onNewLineSpans?.(filePath, spans, readFrom, entry.offset);
4808
+ }
4474
4809
  if (this.onNewLines) {
4475
4810
  this.onNewLines(filePath, lines);
4476
4811
  } else {
@@ -4939,13 +5274,13 @@ function sanitizeFilename(name) {
4939
5274
  }
4940
5275
 
4941
5276
  // src/utils/conversationEtag.ts
4942
- import { createHash } from "crypto";
5277
+ import { createHash as createHash2 } from "crypto";
4943
5278
  function computeConversationEtag({
4944
5279
  filePath,
4945
5280
  messageCount,
4946
5281
  timestamp
4947
5282
  }) {
4948
- const digest = createHash("sha1").update(`${filePath}:${messageCount}:${timestamp}`).digest("hex").slice(0, 16);
5283
+ const digest = createHash2("sha1").update(`${filePath}:${messageCount}:${timestamp}`).digest("hex").slice(0, 16);
4949
5284
  return `"${digest}"`;
4950
5285
  }
4951
5286
 
@@ -5110,6 +5445,10 @@ var StreamerServer = class {
5110
5445
  fileWatcher;
5111
5446
  sessionFileMap = /* @__PURE__ */ new Map();
5112
5447
  // sessionId → JSONL filePath
5448
+ // Per-file seq assignments from the most recent onNewLineSpans (offset index),
5449
+ // handed to the immediately-following onNewLines so it can stamp WS `seq` on
5450
+ // the matching conversation_events entries. Same read → same lines order.
5451
+ pendingLineSeqs = /* @__PURE__ */ new Map();
5113
5452
  pendingQuestions = /* @__PURE__ */ new Map();
5114
5453
  // Content key of the AskUserQuestion currently broadcast for a session (from
5115
5454
  // either the rendered screen or JSONL), used to de-dupe the two paths: when
@@ -5255,6 +5594,42 @@ var StreamerServer = class {
5255
5594
  this.sessionStore = new SessionStore();
5256
5595
  this.wsHub = new WSHub();
5257
5596
  this.fileWatcher = new ConversationWatcher({
5597
+ onNewLineSpans: (filePath, spans, readFrom, endOffset) => {
5598
+ if (!this.cache) return;
5599
+ const cache = this.cache;
5600
+ this.pendingLineSeqs.delete(filePath);
5601
+ try {
5602
+ const seqs = cache.extendMessageIndex(
5603
+ filePath,
5604
+ spans,
5605
+ statSync5(filePath),
5606
+ readFrom,
5607
+ endOffset
5608
+ );
5609
+ if (seqs === null) {
5610
+ cache.deleteFileIndex(filePath, ConversationCache.conversationIdForFile(filePath));
5611
+ cache.clearIndexParseState(filePath);
5612
+ this.trackCacheWrite(
5613
+ cache.backfillIndex(filePath).catch((err) => {
5614
+ this.log.warn("offset-index.backfill_failed", {
5615
+ event: "offset_index.backfill_failed",
5616
+ filePath,
5617
+ trigger: "noncontiguous-append",
5618
+ err
5619
+ });
5620
+ })
5621
+ );
5622
+ return;
5623
+ }
5624
+ this.pendingLineSeqs.set(filePath, seqs);
5625
+ } catch (err) {
5626
+ this.log.warn("offset-index.extend_failed", {
5627
+ event: "offset_index.extend_failed",
5628
+ filePath,
5629
+ err
5630
+ });
5631
+ }
5632
+ },
5258
5633
  onNewLines: (filePath, lines) => {
5259
5634
  this.cache?.updateFromLines(filePath, lines);
5260
5635
  for (const [sessionId, watchedPath] of this.sessionFileMap) {
@@ -5281,13 +5656,20 @@ var StreamerServer = class {
5281
5656
  this.pendingQuestionKey.set(sessionId, key);
5282
5657
  if (broadcast) this.wsHub.broadcast(m);
5283
5658
  }
5284
- this.wsHub.broadcast({ type: "conversation_events", sessionId, lines });
5659
+ const seqs = this.pendingLineSeqs.get(filePath);
5660
+ this.wsHub.broadcast({
5661
+ type: "conversation_events",
5662
+ sessionId,
5663
+ lines,
5664
+ ...seqs && seqs.length === lines.length ? { seqs } : {}
5665
+ });
5285
5666
  for (const line of lines) {
5286
5667
  this.wsHub.broadcast({ type: "conversation_event", sessionId, line });
5287
5668
  }
5288
5669
  break;
5289
5670
  }
5290
5671
  }
5672
+ this.pendingLineSeqs.delete(filePath);
5291
5673
  },
5292
5674
  onConversationChanged: (filePath) => {
5293
5675
  this.fileWatcher.poke(filePath);
@@ -6371,9 +6753,13 @@ var StreamerServer = class {
6371
6753
  return;
6372
6754
  }
6373
6755
  const etagSource = conversation;
6756
+ const indexedCount = etagSource.filePath && this.cache ? this.cache.getIndexedMessageCount(
6757
+ ConversationCache.conversationIdForFile(etagSource.filePath)
6758
+ ) : 0;
6759
+ const etagMessageCount = Math.max(etagSource.messageCount, indexedCount);
6374
6760
  const etag = computeConversationEtag({
6375
6761
  filePath: etagSource.filePath,
6376
- messageCount: etagSource.messageCount,
6762
+ messageCount: etagMessageCount,
6377
6763
  timestamp: etagSource.timestamp
6378
6764
  });
6379
6765
  const isFirstPage = !url.searchParams.has("before_index") && !url.searchParams.has("anchor_index") && !url.searchParams.has("after_index");
@@ -6390,12 +6776,14 @@ var StreamerServer = class {
6390
6776
  let slice = filtered;
6391
6777
  let fromIdx = 0;
6392
6778
  let messagePagination;
6779
+ let indexTotal = null;
6393
6780
  if (usePaging) {
6394
6781
  const limit = Math.min(Math.max(intParam(url, "msg_limit", 80), 1), 500);
6395
6782
  let beforeIndex = total;
6396
6783
  let scanLimit = limit;
6397
6784
  let anchorIndex = null;
6398
6785
  let newerPaging = false;
6786
+ let usedAfterIndex = false;
6399
6787
  if (url.searchParams.has("before_index")) {
6400
6788
  beforeIndex = intParam(url, "before_index", total);
6401
6789
  beforeIndex = Math.min(Math.max(beforeIndex, 0), total);
@@ -6404,6 +6792,7 @@ var StreamerServer = class {
6404
6792
  beforeIndex = Math.min(total, from + limit);
6405
6793
  scanLimit = beforeIndex - from;
6406
6794
  newerPaging = true;
6795
+ usedAfterIndex = true;
6407
6796
  } else if (hasAnchor) {
6408
6797
  anchorIndex = Math.min(
6409
6798
  Math.max(intParam(url, "anchor_index", 0), 0),
@@ -6413,9 +6802,34 @@ var StreamerServer = class {
6413
6802
  beforeIndex = Math.min(total, from + limit);
6414
6803
  newerPaging = true;
6415
6804
  }
6416
- const pagedScanner = this.scannerReady ? await this.getScanner(true) : null;
6417
- const page = scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null;
6418
- const start = page?.fromIndex ?? Math.max(0, beforeIndex - scanLimit);
6805
+ const isTailRequest = !url.searchParams.has("before_index") && !hasAfter && !hasAnchor;
6806
+ const indexFilePath = conversation.filePath;
6807
+ if (isTailRequest && indexFilePath && this.cache) {
6808
+ const indexed = this.cache.getIndexedMessageCount(
6809
+ ConversationCache.conversationIdForFile(indexFilePath)
6810
+ );
6811
+ if (indexed > beforeIndex) {
6812
+ beforeIndex = indexed;
6813
+ }
6814
+ }
6815
+ const windowStart = Math.max(0, beforeIndex - scanLimit);
6816
+ const indexWindow = scanLimit > 0 && !hasAnchor && indexFilePath && this.cache ? this.cache.readMessageWindow(indexFilePath, windowStart, beforeIndex) : null;
6817
+ if (!indexWindow && indexFilePath && this.cache && !hasAnchor) {
6818
+ this.trackCacheWrite(
6819
+ this.cache.backfillIndex(indexFilePath).catch((err) => {
6820
+ this.log.warn("offset-index.backfill_failed", {
6821
+ event: "offset_index.backfill_failed",
6822
+ conversationId: id,
6823
+ filePath: indexFilePath,
6824
+ err
6825
+ });
6826
+ })
6827
+ );
6828
+ }
6829
+ const pagedScanner = !indexWindow && this.scannerReady ? await this.getScanner(true) : null;
6830
+ const page = indexWindow ?? (scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null);
6831
+ if (indexWindow) indexTotal = indexWindow.total;
6832
+ const start = page?.fromIndex ?? windowStart;
6419
6833
  slice = page?.messages ?? filtered.slice(start, beforeIndex);
6420
6834
  fromIdx = start;
6421
6835
  const effectiveTotal = page?.total ?? total;
@@ -6431,6 +6845,9 @@ var StreamerServer = class {
6431
6845
  messagePagination.has_more_newer = beforeIndex < effectiveTotal;
6432
6846
  messagePagination.next_after_index = beforeIndex < effectiveTotal ? beforeIndex : null;
6433
6847
  }
6848
+ if (usedAfterIndex) {
6849
+ messagePagination.etag = etag;
6850
+ }
6434
6851
  }
6435
6852
  const messagesPayload = slice.map((m, localIdx) => {
6436
6853
  const content = [];
@@ -6472,6 +6889,8 @@ var StreamerServer = class {
6472
6889
  const cachedConvMeta = this.cache?.getMetaById(id);
6473
6890
  const convProvider = coerceProviderForRunner(conv.provider ?? cachedConvMeta?.provider);
6474
6891
  const availability = classifyResumability(conv.projectPath);
6892
+ const metaMessageCount = indexTotal != null && indexTotal > conv.messageCount ? indexTotal : conv.messageCount;
6893
+ const metaLastUpdatedAt = indexTotal != null && indexTotal > conv.messageCount ? slice.at(-1)?.timestamp ?? conv.timestamp : conv.timestamp;
6475
6894
  const body = {
6476
6895
  meta: {
6477
6896
  id,
@@ -6479,8 +6898,8 @@ var StreamerServer = class {
6479
6898
  project_name: conv.projectName,
6480
6899
  project_path: conv.projectPath,
6481
6900
  file_path: conv.filePath,
6482
- last_updated_at: conv.timestamp,
6483
- message_count: conv.messageCount,
6901
+ last_updated_at: metaLastUpdatedAt,
6902
+ message_count: metaMessageCount,
6484
6903
  last_prompt: conv.lastPrompt ?? void 0,
6485
6904
  provider: convProvider,
6486
6905
  resumable: isProviderResumable(convProvider, availability.resumable),