@threadbase-sh/streamer 1.27.3 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +480 -54
- package/dist/cli.cjs.map +1 -1
- package/dist/index.cjs +449 -33
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +116 -1
- package/dist/index.d.ts +116 -1
- package/dist/index.js +430 -11
- package/dist/index.js.map +1 -1
- package/dist/migrations/009_create_offset_index.sql +28 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3124,9 +3124,15 @@ async function createDirectory(parentAbsolutePath, name) {
|
|
|
3124
3124
|
}
|
|
3125
3125
|
|
|
3126
3126
|
// src/conversation-cache.ts
|
|
3127
|
+
import {
|
|
3128
|
+
createJsonlParseState,
|
|
3129
|
+
parseJsonlLine
|
|
3130
|
+
} from "@threadbase-sh/scanner";
|
|
3127
3131
|
import Database from "better-sqlite3";
|
|
3128
3132
|
import { closeSync as closeSync2, existsSync as existsSync5, mkdirSync as mkdirSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "fs";
|
|
3133
|
+
import { open as openAsync } from "fs/promises";
|
|
3129
3134
|
import { dirname as dirname6 } from "path";
|
|
3135
|
+
import { setImmediate as yieldToEventLoop } from "timers/promises";
|
|
3130
3136
|
|
|
3131
3137
|
// src/db/sqlite-migrate.ts
|
|
3132
3138
|
import { readdirSync as readdirSync2, readFileSync as readFileSync5 } from "fs";
|
|
@@ -3238,6 +3244,33 @@ function parseAgentEntrypointsEnv(raw) {
|
|
|
3238
3244
|
return new Set(parts);
|
|
3239
3245
|
}
|
|
3240
3246
|
|
|
3247
|
+
// src/utils/fileIdentity.ts
|
|
3248
|
+
import { createHash } from "crypto";
|
|
3249
|
+
function fileIdentity(stat3, headBytes) {
|
|
3250
|
+
if (stat3.ino && stat3.ino > 0) return `inode:${stat3.dev}:${stat3.ino}`;
|
|
3251
|
+
const head = headBytes ?? Buffer.alloc(0);
|
|
3252
|
+
return `fp:${createHash("sha1").update(head).digest("hex")}`;
|
|
3253
|
+
}
|
|
3254
|
+
function splitCompleteLines(buf, baseOffset) {
|
|
3255
|
+
const spans = [];
|
|
3256
|
+
let lineStart = 0;
|
|
3257
|
+
let consumed = 0;
|
|
3258
|
+
for (let i = 0; i < buf.length; i++) {
|
|
3259
|
+
if (buf[i] !== 10) continue;
|
|
3260
|
+
const lineLen = i - lineStart;
|
|
3261
|
+
if (lineLen > 0) {
|
|
3262
|
+
spans.push({
|
|
3263
|
+
byteOffset: baseOffset + lineStart,
|
|
3264
|
+
byteLength: lineLen,
|
|
3265
|
+
text: buf.toString("utf-8", lineStart, i)
|
|
3266
|
+
});
|
|
3267
|
+
}
|
|
3268
|
+
lineStart = i + 1;
|
|
3269
|
+
consumed = lineStart;
|
|
3270
|
+
}
|
|
3271
|
+
return { spans, consumed };
|
|
3272
|
+
}
|
|
3273
|
+
|
|
3241
3274
|
// src/conversation-cache.ts
|
|
3242
3275
|
function shortProjectName(fullPath) {
|
|
3243
3276
|
const parts = fullPath.split(/[/\\]/).filter(Boolean);
|
|
@@ -3286,6 +3319,14 @@ var ConversationCache = class _ConversationCache {
|
|
|
3286
3319
|
tailSize;
|
|
3287
3320
|
fileIndex = /* @__PURE__ */ new Map();
|
|
3288
3321
|
fileIndexLoaded = false;
|
|
3322
|
+
// Per-file scanner parse state for the incremental offset-index writer. The
|
|
3323
|
+
// reducer is stateful across lines (pending tool_uses, latest timestamp), so
|
|
3324
|
+
// it must persist between watcher reads of the same file. Cleared on
|
|
3325
|
+
// truncation/backfill.
|
|
3326
|
+
indexParseState = /* @__PURE__ */ new Map();
|
|
3327
|
+
// Single-flight guard for backfillIndex — concurrent detail requests for the
|
|
3328
|
+
// same cold file await one walk, not N. Entry dropped on settle.
|
|
3329
|
+
backfillInFlight = /* @__PURE__ */ new Map();
|
|
3289
3330
|
// Monotonically increasing counter for tail updated_at — guarantees strict
|
|
3290
3331
|
// ordering even when multiple updateFromLine() calls land within the same ms.
|
|
3291
3332
|
tailSeq = Date.now();
|
|
@@ -3449,6 +3490,42 @@ var ConversationCache = class _ConversationCache {
|
|
|
3449
3490
|
GROUP BY project_path
|
|
3450
3491
|
ORDER BY cnt DESC
|
|
3451
3492
|
LIMIT ?`
|
|
3493
|
+
),
|
|
3494
|
+
getFileState: db.prepare("SELECT * FROM conversation_file_state WHERE path = ?"),
|
|
3495
|
+
upsertFileState: db.prepare(
|
|
3496
|
+
`INSERT INTO conversation_file_state
|
|
3497
|
+
(path, identity, size, mtime_ms, byte_offset, last_message_index)
|
|
3498
|
+
VALUES (@path, @identity, @size, @mtime_ms, @byte_offset, @last_message_index)
|
|
3499
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
3500
|
+
identity = excluded.identity,
|
|
3501
|
+
size = excluded.size,
|
|
3502
|
+
mtime_ms = excluded.mtime_ms,
|
|
3503
|
+
byte_offset = excluded.byte_offset,
|
|
3504
|
+
last_message_index = excluded.last_message_index`
|
|
3505
|
+
),
|
|
3506
|
+
deleteFileState: db.prepare("DELETE FROM conversation_file_state WHERE path = ?"),
|
|
3507
|
+
deleteMessageIndex: db.prepare(
|
|
3508
|
+
"DELETE FROM conversation_message_index WHERE conversation_id = ?"
|
|
3509
|
+
),
|
|
3510
|
+
insertMessageIndexRow: db.prepare(
|
|
3511
|
+
`INSERT INTO conversation_message_index
|
|
3512
|
+
(conversation_id, message_index, byte_offset, byte_length, uuid, role, ts)
|
|
3513
|
+
VALUES (@conversation_id, @message_index, @byte_offset, @byte_length, @uuid, @role, @ts)
|
|
3514
|
+
ON CONFLICT(conversation_id, message_index) DO UPDATE SET
|
|
3515
|
+
byte_offset = excluded.byte_offset,
|
|
3516
|
+
byte_length = excluded.byte_length,
|
|
3517
|
+
uuid = excluded.uuid,
|
|
3518
|
+
role = excluded.role,
|
|
3519
|
+
ts = excluded.ts`
|
|
3520
|
+
),
|
|
3521
|
+
getMessageIndexWindow: db.prepare(
|
|
3522
|
+
`SELECT message_index, byte_offset, byte_length, uuid, role, ts
|
|
3523
|
+
FROM conversation_message_index
|
|
3524
|
+
WHERE conversation_id = ? AND message_index >= ? AND message_index < ?
|
|
3525
|
+
ORDER BY message_index ASC`
|
|
3526
|
+
),
|
|
3527
|
+
getIndexedMessageCount: db.prepare(
|
|
3528
|
+
"SELECT COUNT(*) as cnt FROM conversation_message_index WHERE conversation_id = ?"
|
|
3452
3529
|
)
|
|
3453
3530
|
};
|
|
3454
3531
|
}
|
|
@@ -3459,6 +3536,258 @@ var ConversationCache = class _ConversationCache {
|
|
|
3459
3536
|
getDatabase() {
|
|
3460
3537
|
return this.db;
|
|
3461
3538
|
}
|
|
3539
|
+
// ── Offset index (design 1b) ────────────────────────────────────────────
|
|
3540
|
+
// conversation_file_state + conversation_message_index back the windowed
|
|
3541
|
+
// detail read path (SQL window select + pread of byte ranges). All methods
|
|
3542
|
+
// are thin wrappers over the prepared statements above.
|
|
3543
|
+
getFileState(path) {
|
|
3544
|
+
return this.stmts.getFileState.get(path) ?? null;
|
|
3545
|
+
}
|
|
3546
|
+
upsertFileState(row) {
|
|
3547
|
+
this.stmts.upsertFileState.run(row);
|
|
3548
|
+
}
|
|
3549
|
+
/** Drop a file's index rows + file_state (truncation / identity change). */
|
|
3550
|
+
deleteFileIndex(path, conversationId) {
|
|
3551
|
+
const tx = this.db.transaction(() => {
|
|
3552
|
+
this.stmts.deleteMessageIndex.run(conversationId);
|
|
3553
|
+
this.stmts.deleteFileState.run(path);
|
|
3554
|
+
});
|
|
3555
|
+
tx();
|
|
3556
|
+
this.indexParseState.delete(path);
|
|
3557
|
+
}
|
|
3558
|
+
/** Append/replace index rows in one transaction. */
|
|
3559
|
+
appendMessageIndexRows(rows) {
|
|
3560
|
+
const tx = this.db.transaction((batch) => {
|
|
3561
|
+
for (const r of batch) this.stmts.insertMessageIndexRow.run(r);
|
|
3562
|
+
});
|
|
3563
|
+
tx(rows);
|
|
3564
|
+
}
|
|
3565
|
+
/** Rows for message_index in [fromIndex, toIndex), ordered ascending. */
|
|
3566
|
+
getMessageIndexWindow(conversationId, fromIndex, toIndex) {
|
|
3567
|
+
return this.stmts.getMessageIndexWindow.all(
|
|
3568
|
+
conversationId,
|
|
3569
|
+
fromIndex,
|
|
3570
|
+
toIndex
|
|
3571
|
+
);
|
|
3572
|
+
}
|
|
3573
|
+
getIndexedMessageCount(conversationId) {
|
|
3574
|
+
return this.stmts.getIndexedMessageCount.get(conversationId).cnt;
|
|
3575
|
+
}
|
|
3576
|
+
/**
|
|
3577
|
+
* Conversation id for a JSONL path — the filename stem (matches the pseudo-id
|
|
3578
|
+
* updateFromLine derives and the uuid the detail read path resolves). The
|
|
3579
|
+
* offset index keys on this so the window select and the cursor agree.
|
|
3580
|
+
*/
|
|
3581
|
+
static conversationIdForFile(filePath) {
|
|
3582
|
+
return filePath.split(/[/\\]/).pop()?.replace(/\.jsonl$/, "") ?? filePath;
|
|
3583
|
+
}
|
|
3584
|
+
/**
|
|
3585
|
+
* Incremental offset-index writer: extend the index for a burst of appended
|
|
3586
|
+
* lines (one watcher read) using their byte spans. Each line is classified
|
|
3587
|
+
* with the scanner's parseJsonlLine (a running per-file reducer state), so the
|
|
3588
|
+
* message ordering can never drift from the scanner's. Message lines get an
|
|
3589
|
+
* index row at the next message_index; non-message lines (summary/sidecar)
|
|
3590
|
+
* get no row but still advance byte_offset. file_state is updated to the end
|
|
3591
|
+
* of the last consumed span.
|
|
3592
|
+
*
|
|
3593
|
+
* Requires an up-to-date `stat` (identity/size/mtime) for the file so the read
|
|
3594
|
+
* path can detect truncation/replacement.
|
|
3595
|
+
*
|
|
3596
|
+
* `readFrom` is the absolute byte offset the watcher read started at, and
|
|
3597
|
+
* `endOffset` is where it ended (readFrom + consumed, i.e. the watcher's new
|
|
3598
|
+
* entry.offset). CONTIGUITY GUARD: the read must begin exactly where the index
|
|
3599
|
+
* left off (`readFrom === existing.byte_offset`, or 0 with no state). If it
|
|
3600
|
+
* doesn't — the watcher attached at EOF after the server was down, or an
|
|
3601
|
+
* append raced an in-flight backfill — extending would assign wrong
|
|
3602
|
+
* message_index values over a hole. In that case this writes nothing and
|
|
3603
|
+
* returns null so the caller drops the index and backfills.
|
|
3604
|
+
*
|
|
3605
|
+
* On success returns the message_index assigned to each input span (null for a
|
|
3606
|
+
* non-message line) so the caller can stamp WS `seq`. Empty array when spans
|
|
3607
|
+
* is empty. `endOffset` is stored verbatim as byte_offset so the watcher's
|
|
3608
|
+
* offset and file_state.byte_offset are the same number by construction.
|
|
3609
|
+
*/
|
|
3610
|
+
extendMessageIndex(filePath, spans, stat3, readFrom, endOffset) {
|
|
3611
|
+
const existing = this.getFileState(filePath);
|
|
3612
|
+
const expectedStart = existing?.byte_offset ?? 0;
|
|
3613
|
+
if (readFrom !== expectedStart) return null;
|
|
3614
|
+
if (spans.length === 0) return [];
|
|
3615
|
+
const convId = _ConversationCache.conversationIdForFile(filePath);
|
|
3616
|
+
let state = this.indexParseState.get(filePath);
|
|
3617
|
+
if (!state) {
|
|
3618
|
+
state = createJsonlParseState();
|
|
3619
|
+
this.indexParseState.set(filePath, state);
|
|
3620
|
+
}
|
|
3621
|
+
let nextIndex = existing ? existing.last_message_index + 1 : 0;
|
|
3622
|
+
const rows = [];
|
|
3623
|
+
const seqs = [];
|
|
3624
|
+
for (const span of spans) {
|
|
3625
|
+
const msg = parseJsonlLine(span.text, state);
|
|
3626
|
+
if (!msg) {
|
|
3627
|
+
seqs.push(null);
|
|
3628
|
+
continue;
|
|
3629
|
+
}
|
|
3630
|
+
rows.push({
|
|
3631
|
+
conversation_id: convId,
|
|
3632
|
+
message_index: nextIndex,
|
|
3633
|
+
byte_offset: span.byteOffset,
|
|
3634
|
+
byte_length: span.byteLength,
|
|
3635
|
+
uuid: msg.uuid ?? null,
|
|
3636
|
+
role: msg.role ?? null,
|
|
3637
|
+
ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
|
|
3638
|
+
});
|
|
3639
|
+
seqs.push(nextIndex);
|
|
3640
|
+
nextIndex++;
|
|
3641
|
+
}
|
|
3642
|
+
const tx = this.db.transaction(() => {
|
|
3643
|
+
for (const r of rows) this.stmts.insertMessageIndexRow.run(r);
|
|
3644
|
+
this.stmts.upsertFileState.run({
|
|
3645
|
+
path: filePath,
|
|
3646
|
+
identity: fileIdentity(stat3),
|
|
3647
|
+
size: stat3.size,
|
|
3648
|
+
mtime_ms: Math.round(stat3.mtimeMs),
|
|
3649
|
+
// Store the watcher's end offset verbatim — same number as entry.offset,
|
|
3650
|
+
// so the next read's contiguity check compares like-for-like (never
|
|
3651
|
+
// false-positive on a read that ended in trailing empty lines).
|
|
3652
|
+
byte_offset: endOffset,
|
|
3653
|
+
last_message_index: nextIndex - 1
|
|
3654
|
+
});
|
|
3655
|
+
});
|
|
3656
|
+
tx();
|
|
3657
|
+
return seqs;
|
|
3658
|
+
}
|
|
3659
|
+
clearIndexParseState(filePath) {
|
|
3660
|
+
this.indexParseState.delete(filePath);
|
|
3661
|
+
}
|
|
3662
|
+
/**
|
|
3663
|
+
* On-demand full backfill of the offset index for a file with no/stale
|
|
3664
|
+
* file_state (cold conversation, or after a truncation/replacement). Rebuilds
|
|
3665
|
+
* from byte 0: drops any existing rows, walks the whole file in chunks with a
|
|
3666
|
+
* running parse state, yields to the event loop every ~1000 lines so a large
|
|
3667
|
+
* file never blocks, and writes index rows + file_state.
|
|
3668
|
+
*
|
|
3669
|
+
* Single-flighted per path: concurrent callers await the same walk. The
|
|
3670
|
+
* triggering detail request is served by the scanner fallback while this runs.
|
|
3671
|
+
*/
|
|
3672
|
+
backfillIndex(filePath) {
|
|
3673
|
+
const inFlight = this.backfillInFlight.get(filePath);
|
|
3674
|
+
if (inFlight) return inFlight;
|
|
3675
|
+
const walk = this.runBackfill(filePath).finally(() => {
|
|
3676
|
+
this.backfillInFlight.delete(filePath);
|
|
3677
|
+
});
|
|
3678
|
+
this.backfillInFlight.set(filePath, walk);
|
|
3679
|
+
return walk;
|
|
3680
|
+
}
|
|
3681
|
+
async runBackfill(filePath) {
|
|
3682
|
+
const convId = _ConversationCache.conversationIdForFile(filePath);
|
|
3683
|
+
this.deleteFileIndex(filePath, convId);
|
|
3684
|
+
this.indexParseState.delete(filePath);
|
|
3685
|
+
const CHUNK = 256 * 1024;
|
|
3686
|
+
const YIELD_EVERY = 1e3;
|
|
3687
|
+
const state = createJsonlParseState();
|
|
3688
|
+
const fh = await openAsync(filePath, "r");
|
|
3689
|
+
let fileOffset = 0;
|
|
3690
|
+
let carry = Buffer.alloc(0);
|
|
3691
|
+
let nextIndex = 0;
|
|
3692
|
+
let linesSinceYield = 0;
|
|
3693
|
+
let lastConsumedEnd = 0;
|
|
3694
|
+
let stat3;
|
|
3695
|
+
try {
|
|
3696
|
+
stat3 = await fh.stat();
|
|
3697
|
+
const buf = Buffer.alloc(CHUNK);
|
|
3698
|
+
for (; ; ) {
|
|
3699
|
+
const { bytesRead } = await fh.read(buf, 0, CHUNK, null);
|
|
3700
|
+
if (bytesRead === 0) break;
|
|
3701
|
+
const combined = carry.length > 0 ? Buffer.concat([carry, buf.subarray(0, bytesRead)]) : buf.subarray(0, bytesRead);
|
|
3702
|
+
const { spans, consumed } = splitCompleteLines(combined, fileOffset);
|
|
3703
|
+
const rows = [];
|
|
3704
|
+
for (const span of spans) {
|
|
3705
|
+
const msg = parseJsonlLine(span.text, state);
|
|
3706
|
+
linesSinceYield++;
|
|
3707
|
+
if (msg) {
|
|
3708
|
+
rows.push({
|
|
3709
|
+
conversation_id: convId,
|
|
3710
|
+
message_index: nextIndex,
|
|
3711
|
+
byte_offset: span.byteOffset,
|
|
3712
|
+
byte_length: span.byteLength,
|
|
3713
|
+
uuid: msg.uuid ?? null,
|
|
3714
|
+
role: msg.role ?? null,
|
|
3715
|
+
ts: msg.timestamp ? Date.parse(msg.timestamp) || null : null
|
|
3716
|
+
});
|
|
3717
|
+
nextIndex++;
|
|
3718
|
+
}
|
|
3719
|
+
if (linesSinceYield >= YIELD_EVERY) {
|
|
3720
|
+
linesSinceYield = 0;
|
|
3721
|
+
await yieldToEventLoop();
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
3724
|
+
if (rows.length > 0) this.appendMessageIndexRows(rows);
|
|
3725
|
+
lastConsumedEnd = fileOffset + consumed;
|
|
3726
|
+
carry = Buffer.from(combined.subarray(consumed));
|
|
3727
|
+
fileOffset += consumed;
|
|
3728
|
+
}
|
|
3729
|
+
} finally {
|
|
3730
|
+
await fh.close();
|
|
3731
|
+
}
|
|
3732
|
+
this.upsertFileState({
|
|
3733
|
+
path: filePath,
|
|
3734
|
+
identity: fileIdentity(stat3),
|
|
3735
|
+
size: stat3.size,
|
|
3736
|
+
mtime_ms: Math.round(stat3.mtimeMs),
|
|
3737
|
+
byte_offset: lastConsumedEnd,
|
|
3738
|
+
last_message_index: nextIndex - 1
|
|
3739
|
+
});
|
|
3740
|
+
this.indexParseState.set(filePath, state);
|
|
3741
|
+
}
|
|
3742
|
+
/**
|
|
3743
|
+
* Windowed detail read straight from the offset index — the hot path.
|
|
3744
|
+
* Returns the parsed messages for message_index in [fromIndex, toIndex) plus
|
|
3745
|
+
* the total indexed count, or null when the index can't serve this file (no
|
|
3746
|
+
* file_state, identity/size mismatch = truncation/replacement, or cold index)
|
|
3747
|
+
* so the caller falls back to the scanner and enqueues a backfill.
|
|
3748
|
+
*
|
|
3749
|
+
* On a match it SQL-selects the window's byte ranges and preads exactly those
|
|
3750
|
+
* ranges from the JSONL (never the whole file), parsing only the sliced lines.
|
|
3751
|
+
* Returns messages in the same ConversationMessage shape parseJsonlLine
|
|
3752
|
+
* produces during a scan, so the payload is identical to the scanner path.
|
|
3753
|
+
*/
|
|
3754
|
+
readMessageWindow(filePath, fromIndex, toIndex) {
|
|
3755
|
+
const fileState = this.getFileState(filePath);
|
|
3756
|
+
if (!fileState) return null;
|
|
3757
|
+
let stat3;
|
|
3758
|
+
try {
|
|
3759
|
+
stat3 = statSync2(filePath);
|
|
3760
|
+
} catch {
|
|
3761
|
+
return null;
|
|
3762
|
+
}
|
|
3763
|
+
if (fileIdentity(stat3) !== fileState.identity || stat3.size !== fileState.byte_offset) {
|
|
3764
|
+
return null;
|
|
3765
|
+
}
|
|
3766
|
+
const total = fileState.last_message_index + 1;
|
|
3767
|
+
const from = Math.max(0, fromIndex);
|
|
3768
|
+
const to = Math.min(toIndex, total);
|
|
3769
|
+
if (to <= from) return { messages: [], total, fromIndex: from };
|
|
3770
|
+
const rows = this.getMessageIndexWindow(
|
|
3771
|
+
_ConversationCache.conversationIdForFile(filePath),
|
|
3772
|
+
from,
|
|
3773
|
+
to
|
|
3774
|
+
);
|
|
3775
|
+
if (rows.length === 0) return { messages: [], total, fromIndex: from };
|
|
3776
|
+
const messages = [];
|
|
3777
|
+
const fd = openSync2(filePath, "r");
|
|
3778
|
+
try {
|
|
3779
|
+
const state = createJsonlParseState();
|
|
3780
|
+
for (const row of rows) {
|
|
3781
|
+
const buf = Buffer.alloc(row.byte_length);
|
|
3782
|
+
readSync2(fd, buf, 0, row.byte_length, row.byte_offset);
|
|
3783
|
+
const msg = parseJsonlLine(buf.toString("utf-8"), state);
|
|
3784
|
+
if (msg) messages.push(msg);
|
|
3785
|
+
}
|
|
3786
|
+
} finally {
|
|
3787
|
+
closeSync2(fd);
|
|
3788
|
+
}
|
|
3789
|
+
return { messages, total, fromIndex: from };
|
|
3790
|
+
}
|
|
3462
3791
|
agentEntrypointsKey() {
|
|
3463
3792
|
return [...this.agentEntrypoints].sort().join(",");
|
|
3464
3793
|
}
|
|
@@ -4359,12 +4688,14 @@ var ConversationWatcher = class {
|
|
|
4359
4688
|
directories = /* @__PURE__ */ new Map();
|
|
4360
4689
|
onNewLine;
|
|
4361
4690
|
onNewLines;
|
|
4691
|
+
onNewLineSpans;
|
|
4362
4692
|
onConversationChanged;
|
|
4363
4693
|
onFileDeleted;
|
|
4364
4694
|
onError;
|
|
4365
4695
|
constructor(events = {}) {
|
|
4366
4696
|
this.onNewLine = events.onNewLine;
|
|
4367
4697
|
this.onNewLines = events.onNewLines;
|
|
4698
|
+
this.onNewLineSpans = events.onNewLineSpans;
|
|
4368
4699
|
this.onConversationChanged = events.onConversationChanged;
|
|
4369
4700
|
this.onFileDeleted = events.onFileDeleted;
|
|
4370
4701
|
this.onError = events.onError;
|
|
@@ -4468,9 +4799,13 @@ var ConversationWatcher = class {
|
|
|
4468
4799
|
} finally {
|
|
4469
4800
|
await fh.close();
|
|
4470
4801
|
}
|
|
4471
|
-
|
|
4802
|
+
const { spans, consumed } = splitCompleteLines(buf, readFrom);
|
|
4803
|
+
entry.offset = readFrom + consumed;
|
|
4472
4804
|
if (!this.files.has(filePath)) return;
|
|
4473
|
-
const lines =
|
|
4805
|
+
const lines = spans.map((s) => s.text);
|
|
4806
|
+
if (spans.length > 0) {
|
|
4807
|
+
this.onNewLineSpans?.(filePath, spans, readFrom, entry.offset);
|
|
4808
|
+
}
|
|
4474
4809
|
if (this.onNewLines) {
|
|
4475
4810
|
this.onNewLines(filePath, lines);
|
|
4476
4811
|
} else {
|
|
@@ -4939,13 +5274,13 @@ function sanitizeFilename(name) {
|
|
|
4939
5274
|
}
|
|
4940
5275
|
|
|
4941
5276
|
// src/utils/conversationEtag.ts
|
|
4942
|
-
import { createHash } from "crypto";
|
|
5277
|
+
import { createHash as createHash2 } from "crypto";
|
|
4943
5278
|
function computeConversationEtag({
|
|
4944
5279
|
filePath,
|
|
4945
5280
|
messageCount,
|
|
4946
5281
|
timestamp
|
|
4947
5282
|
}) {
|
|
4948
|
-
const digest =
|
|
5283
|
+
const digest = createHash2("sha1").update(`${filePath}:${messageCount}:${timestamp}`).digest("hex").slice(0, 16);
|
|
4949
5284
|
return `"${digest}"`;
|
|
4950
5285
|
}
|
|
4951
5286
|
|
|
@@ -5110,6 +5445,10 @@ var StreamerServer = class {
|
|
|
5110
5445
|
fileWatcher;
|
|
5111
5446
|
sessionFileMap = /* @__PURE__ */ new Map();
|
|
5112
5447
|
// sessionId → JSONL filePath
|
|
5448
|
+
// Per-file seq assignments from the most recent onNewLineSpans (offset index),
|
|
5449
|
+
// handed to the immediately-following onNewLines so it can stamp WS `seq` on
|
|
5450
|
+
// the matching conversation_events entries. Same read → same lines order.
|
|
5451
|
+
pendingLineSeqs = /* @__PURE__ */ new Map();
|
|
5113
5452
|
pendingQuestions = /* @__PURE__ */ new Map();
|
|
5114
5453
|
// Content key of the AskUserQuestion currently broadcast for a session (from
|
|
5115
5454
|
// either the rendered screen or JSONL), used to de-dupe the two paths: when
|
|
@@ -5255,6 +5594,42 @@ var StreamerServer = class {
|
|
|
5255
5594
|
this.sessionStore = new SessionStore();
|
|
5256
5595
|
this.wsHub = new WSHub();
|
|
5257
5596
|
this.fileWatcher = new ConversationWatcher({
|
|
5597
|
+
onNewLineSpans: (filePath, spans, readFrom, endOffset) => {
|
|
5598
|
+
if (!this.cache) return;
|
|
5599
|
+
const cache = this.cache;
|
|
5600
|
+
this.pendingLineSeqs.delete(filePath);
|
|
5601
|
+
try {
|
|
5602
|
+
const seqs = cache.extendMessageIndex(
|
|
5603
|
+
filePath,
|
|
5604
|
+
spans,
|
|
5605
|
+
statSync5(filePath),
|
|
5606
|
+
readFrom,
|
|
5607
|
+
endOffset
|
|
5608
|
+
);
|
|
5609
|
+
if (seqs === null) {
|
|
5610
|
+
cache.deleteFileIndex(filePath, ConversationCache.conversationIdForFile(filePath));
|
|
5611
|
+
cache.clearIndexParseState(filePath);
|
|
5612
|
+
this.trackCacheWrite(
|
|
5613
|
+
cache.backfillIndex(filePath).catch((err) => {
|
|
5614
|
+
this.log.warn("offset-index.backfill_failed", {
|
|
5615
|
+
event: "offset_index.backfill_failed",
|
|
5616
|
+
filePath,
|
|
5617
|
+
trigger: "noncontiguous-append",
|
|
5618
|
+
err
|
|
5619
|
+
});
|
|
5620
|
+
})
|
|
5621
|
+
);
|
|
5622
|
+
return;
|
|
5623
|
+
}
|
|
5624
|
+
this.pendingLineSeqs.set(filePath, seqs);
|
|
5625
|
+
} catch (err) {
|
|
5626
|
+
this.log.warn("offset-index.extend_failed", {
|
|
5627
|
+
event: "offset_index.extend_failed",
|
|
5628
|
+
filePath,
|
|
5629
|
+
err
|
|
5630
|
+
});
|
|
5631
|
+
}
|
|
5632
|
+
},
|
|
5258
5633
|
onNewLines: (filePath, lines) => {
|
|
5259
5634
|
this.cache?.updateFromLines(filePath, lines);
|
|
5260
5635
|
for (const [sessionId, watchedPath] of this.sessionFileMap) {
|
|
@@ -5281,13 +5656,20 @@ var StreamerServer = class {
|
|
|
5281
5656
|
this.pendingQuestionKey.set(sessionId, key);
|
|
5282
5657
|
if (broadcast) this.wsHub.broadcast(m);
|
|
5283
5658
|
}
|
|
5284
|
-
this.
|
|
5659
|
+
const seqs = this.pendingLineSeqs.get(filePath);
|
|
5660
|
+
this.wsHub.broadcast({
|
|
5661
|
+
type: "conversation_events",
|
|
5662
|
+
sessionId,
|
|
5663
|
+
lines,
|
|
5664
|
+
...seqs && seqs.length === lines.length ? { seqs } : {}
|
|
5665
|
+
});
|
|
5285
5666
|
for (const line of lines) {
|
|
5286
5667
|
this.wsHub.broadcast({ type: "conversation_event", sessionId, line });
|
|
5287
5668
|
}
|
|
5288
5669
|
break;
|
|
5289
5670
|
}
|
|
5290
5671
|
}
|
|
5672
|
+
this.pendingLineSeqs.delete(filePath);
|
|
5291
5673
|
},
|
|
5292
5674
|
onConversationChanged: (filePath) => {
|
|
5293
5675
|
this.fileWatcher.poke(filePath);
|
|
@@ -6371,9 +6753,13 @@ var StreamerServer = class {
|
|
|
6371
6753
|
return;
|
|
6372
6754
|
}
|
|
6373
6755
|
const etagSource = conversation;
|
|
6756
|
+
const indexedCount = etagSource.filePath && this.cache ? this.cache.getIndexedMessageCount(
|
|
6757
|
+
ConversationCache.conversationIdForFile(etagSource.filePath)
|
|
6758
|
+
) : 0;
|
|
6759
|
+
const etagMessageCount = Math.max(etagSource.messageCount, indexedCount);
|
|
6374
6760
|
const etag = computeConversationEtag({
|
|
6375
6761
|
filePath: etagSource.filePath,
|
|
6376
|
-
messageCount:
|
|
6762
|
+
messageCount: etagMessageCount,
|
|
6377
6763
|
timestamp: etagSource.timestamp
|
|
6378
6764
|
});
|
|
6379
6765
|
const isFirstPage = !url.searchParams.has("before_index") && !url.searchParams.has("anchor_index") && !url.searchParams.has("after_index");
|
|
@@ -6390,12 +6776,14 @@ var StreamerServer = class {
|
|
|
6390
6776
|
let slice = filtered;
|
|
6391
6777
|
let fromIdx = 0;
|
|
6392
6778
|
let messagePagination;
|
|
6779
|
+
let indexTotal = null;
|
|
6393
6780
|
if (usePaging) {
|
|
6394
6781
|
const limit = Math.min(Math.max(intParam(url, "msg_limit", 80), 1), 500);
|
|
6395
6782
|
let beforeIndex = total;
|
|
6396
6783
|
let scanLimit = limit;
|
|
6397
6784
|
let anchorIndex = null;
|
|
6398
6785
|
let newerPaging = false;
|
|
6786
|
+
let usedAfterIndex = false;
|
|
6399
6787
|
if (url.searchParams.has("before_index")) {
|
|
6400
6788
|
beforeIndex = intParam(url, "before_index", total);
|
|
6401
6789
|
beforeIndex = Math.min(Math.max(beforeIndex, 0), total);
|
|
@@ -6404,6 +6792,7 @@ var StreamerServer = class {
|
|
|
6404
6792
|
beforeIndex = Math.min(total, from + limit);
|
|
6405
6793
|
scanLimit = beforeIndex - from;
|
|
6406
6794
|
newerPaging = true;
|
|
6795
|
+
usedAfterIndex = true;
|
|
6407
6796
|
} else if (hasAnchor) {
|
|
6408
6797
|
anchorIndex = Math.min(
|
|
6409
6798
|
Math.max(intParam(url, "anchor_index", 0), 0),
|
|
@@ -6413,9 +6802,34 @@ var StreamerServer = class {
|
|
|
6413
6802
|
beforeIndex = Math.min(total, from + limit);
|
|
6414
6803
|
newerPaging = true;
|
|
6415
6804
|
}
|
|
6416
|
-
const
|
|
6417
|
-
const
|
|
6418
|
-
|
|
6805
|
+
const isTailRequest = !url.searchParams.has("before_index") && !hasAfter && !hasAnchor;
|
|
6806
|
+
const indexFilePath = conversation.filePath;
|
|
6807
|
+
if (isTailRequest && indexFilePath && this.cache) {
|
|
6808
|
+
const indexed = this.cache.getIndexedMessageCount(
|
|
6809
|
+
ConversationCache.conversationIdForFile(indexFilePath)
|
|
6810
|
+
);
|
|
6811
|
+
if (indexed > beforeIndex) {
|
|
6812
|
+
beforeIndex = indexed;
|
|
6813
|
+
}
|
|
6814
|
+
}
|
|
6815
|
+
const windowStart = Math.max(0, beforeIndex - scanLimit);
|
|
6816
|
+
const indexWindow = scanLimit > 0 && !hasAnchor && indexFilePath && this.cache ? this.cache.readMessageWindow(indexFilePath, windowStart, beforeIndex) : null;
|
|
6817
|
+
if (!indexWindow && indexFilePath && this.cache && !hasAnchor) {
|
|
6818
|
+
this.trackCacheWrite(
|
|
6819
|
+
this.cache.backfillIndex(indexFilePath).catch((err) => {
|
|
6820
|
+
this.log.warn("offset-index.backfill_failed", {
|
|
6821
|
+
event: "offset_index.backfill_failed",
|
|
6822
|
+
conversationId: id,
|
|
6823
|
+
filePath: indexFilePath,
|
|
6824
|
+
err
|
|
6825
|
+
});
|
|
6826
|
+
})
|
|
6827
|
+
);
|
|
6828
|
+
}
|
|
6829
|
+
const pagedScanner = !indexWindow && this.scannerReady ? await this.getScanner(true) : null;
|
|
6830
|
+
const page = indexWindow ?? (scanLimit > 0 && pagedScanner && typeof pagedScanner.getConversationPage === "function" ? await pagedScanner.getConversationPage(id, { beforeIndex, limit: scanLimit }) : null);
|
|
6831
|
+
if (indexWindow) indexTotal = indexWindow.total;
|
|
6832
|
+
const start = page?.fromIndex ?? windowStart;
|
|
6419
6833
|
slice = page?.messages ?? filtered.slice(start, beforeIndex);
|
|
6420
6834
|
fromIdx = start;
|
|
6421
6835
|
const effectiveTotal = page?.total ?? total;
|
|
@@ -6431,6 +6845,9 @@ var StreamerServer = class {
|
|
|
6431
6845
|
messagePagination.has_more_newer = beforeIndex < effectiveTotal;
|
|
6432
6846
|
messagePagination.next_after_index = beforeIndex < effectiveTotal ? beforeIndex : null;
|
|
6433
6847
|
}
|
|
6848
|
+
if (usedAfterIndex) {
|
|
6849
|
+
messagePagination.etag = etag;
|
|
6850
|
+
}
|
|
6434
6851
|
}
|
|
6435
6852
|
const messagesPayload = slice.map((m, localIdx) => {
|
|
6436
6853
|
const content = [];
|
|
@@ -6472,6 +6889,8 @@ var StreamerServer = class {
|
|
|
6472
6889
|
const cachedConvMeta = this.cache?.getMetaById(id);
|
|
6473
6890
|
const convProvider = coerceProviderForRunner(conv.provider ?? cachedConvMeta?.provider);
|
|
6474
6891
|
const availability = classifyResumability(conv.projectPath);
|
|
6892
|
+
const metaMessageCount = indexTotal != null && indexTotal > conv.messageCount ? indexTotal : conv.messageCount;
|
|
6893
|
+
const metaLastUpdatedAt = indexTotal != null && indexTotal > conv.messageCount ? slice.at(-1)?.timestamp ?? conv.timestamp : conv.timestamp;
|
|
6475
6894
|
const body = {
|
|
6476
6895
|
meta: {
|
|
6477
6896
|
id,
|
|
@@ -6479,8 +6898,8 @@ var StreamerServer = class {
|
|
|
6479
6898
|
project_name: conv.projectName,
|
|
6480
6899
|
project_path: conv.projectPath,
|
|
6481
6900
|
file_path: conv.filePath,
|
|
6482
|
-
last_updated_at:
|
|
6483
|
-
message_count:
|
|
6901
|
+
last_updated_at: metaLastUpdatedAt,
|
|
6902
|
+
message_count: metaMessageCount,
|
|
6484
6903
|
last_prompt: conv.lastPrompt ?? void 0,
|
|
6485
6904
|
provider: convProvider,
|
|
6486
6905
|
resumable: isProviderResumable(convProvider, availability.resumable),
|