@threadbase-sh/scanner 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +158 -45
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +157 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +157 -44
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -1
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -1
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -751,7 +751,8 @@ function initialReducerState() {
|
|
|
751
751
|
snippetParts: [],
|
|
752
752
|
previewLength: 0,
|
|
753
753
|
snippetLength: 0,
|
|
754
|
-
badJsonLines: 0
|
|
754
|
+
badJsonLines: 0,
|
|
755
|
+
turnDurations: []
|
|
755
756
|
};
|
|
756
757
|
}
|
|
757
758
|
function reduceLine(state, entry, tier) {
|
|
@@ -765,6 +766,8 @@ function reduceLine(state, entry, tier) {
|
|
|
765
766
|
const ts = entry.timestamp;
|
|
766
767
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
767
768
|
}
|
|
769
|
+
const turnDuration = turnDurationOf(entry);
|
|
770
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
768
771
|
const type = entry.type;
|
|
769
772
|
if (type === "last-prompt") {
|
|
770
773
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -863,6 +866,16 @@ function getShortProjectName(fullPath) {
|
|
|
863
866
|
}
|
|
864
867
|
|
|
865
868
|
// src/parser.ts
|
|
869
|
+
function turnDurationOf(entry) {
|
|
870
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
871
|
+
return null;
|
|
872
|
+
}
|
|
873
|
+
return {
|
|
874
|
+
durationMs: entry.durationMs,
|
|
875
|
+
messageCount: entry.messageCount || 0,
|
|
876
|
+
uuid: entry.uuid
|
|
877
|
+
};
|
|
878
|
+
}
|
|
866
879
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
867
880
|
const log = getLogger();
|
|
868
881
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -916,12 +929,9 @@ async function parseConversation(filePath, account) {
|
|
|
916
929
|
badJsonLines++;
|
|
917
930
|
continue;
|
|
918
931
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
messageCount: entry.messageCount || 0,
|
|
923
|
-
uuid: entry.uuid
|
|
924
|
-
});
|
|
932
|
+
const turnDuration = turnDurationOf(entry);
|
|
933
|
+
if (turnDuration) {
|
|
934
|
+
turnDurations.push(turnDuration);
|
|
925
935
|
continue;
|
|
926
936
|
}
|
|
927
937
|
const message = reduceConvLine(state, entry);
|
|
@@ -2608,7 +2618,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2608
2618
|
}
|
|
2609
2619
|
return { offset, line };
|
|
2610
2620
|
}
|
|
2611
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
2621
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
2612
2622
|
const checkpoints = [];
|
|
2613
2623
|
const state = from ? from.state : initialConvState();
|
|
2614
2624
|
let index = from ? from.messageIndex : 0;
|
|
@@ -2628,7 +2638,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
2628
2638
|
});
|
|
2629
2639
|
}
|
|
2630
2640
|
return false;
|
|
2631
|
-
}
|
|
2641
|
+
},
|
|
2642
|
+
onEntry
|
|
2632
2643
|
);
|
|
2633
2644
|
return checkpoints;
|
|
2634
2645
|
}
|
|
@@ -2649,6 +2660,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
2649
2660
|
applyTeamInfo(window, state);
|
|
2650
2661
|
return { messages: window, total, fromIndex };
|
|
2651
2662
|
}
|
|
2663
|
+
function isInjectedTurn(message, isInjected) {
|
|
2664
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
2665
|
+
}
|
|
2666
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
2667
|
+
let skip = 0;
|
|
2668
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
2669
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
2670
|
+
skip += 1;
|
|
2671
|
+
return false;
|
|
2672
|
+
});
|
|
2673
|
+
return skip;
|
|
2674
|
+
}
|
|
2675
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
2676
|
+
const total = Math.max(0, rawTotal - skip);
|
|
2677
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2678
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2679
|
+
return {
|
|
2680
|
+
total,
|
|
2681
|
+
fromIndex,
|
|
2682
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
2683
|
+
};
|
|
2684
|
+
}
|
|
2685
|
+
function pageOfConversation(conversation, options) {
|
|
2686
|
+
const { messages } = conversation;
|
|
2687
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
2688
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
2689
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
2690
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
2691
|
+
return {
|
|
2692
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
2693
|
+
total,
|
|
2694
|
+
fromIndex,
|
|
2695
|
+
turnDurations: conversation.turnDurations
|
|
2696
|
+
};
|
|
2697
|
+
}
|
|
2652
2698
|
|
|
2653
2699
|
// src/persistent/conversation-stream.ts
|
|
2654
2700
|
async function foldTail(filePath, resume) {
|
|
@@ -4178,6 +4224,7 @@ var PersistentEngine = class {
|
|
|
4178
4224
|
const tier = resolveTier(tierName, customTiers);
|
|
4179
4225
|
const filePath = canonicalPath(rawFilePath);
|
|
4180
4226
|
const existing = this.files.getByPath(filePath);
|
|
4227
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4181
4228
|
const classified = classify(filePath, existing);
|
|
4182
4229
|
const stat6 = classified.stat;
|
|
4183
4230
|
let change = classified.change;
|
|
@@ -4185,7 +4232,7 @@ var PersistentEngine = class {
|
|
|
4185
4232
|
this.markDeleted(filePath);
|
|
4186
4233
|
return { meta: null, change: "vanished" };
|
|
4187
4234
|
}
|
|
4188
|
-
if (change === "unchanged" && !
|
|
4235
|
+
if (change === "unchanged" && !refold) {
|
|
4189
4236
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4190
4237
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4191
4238
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4202,7 +4249,7 @@ var PersistentEngine = class {
|
|
|
4202
4249
|
);
|
|
4203
4250
|
return { meta: meta2, change };
|
|
4204
4251
|
}
|
|
4205
|
-
const resume = change === "appended" && !
|
|
4252
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4206
4253
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4207
4254
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4208
4255
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4363,6 +4410,10 @@ var PersistentEngine = class {
|
|
|
4363
4410
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4364
4411
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4365
4412
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4413
|
+
//
|
|
4414
|
+
// The window is over the conversation with its leading injected-context run
|
|
4415
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4416
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4366
4417
|
async getPage(id, options) {
|
|
4367
4418
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4368
4419
|
if (!meta) return null;
|
|
@@ -4370,24 +4421,28 @@ var PersistentEngine = class {
|
|
|
4370
4421
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4371
4422
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4372
4423
|
if (!conversation) return null;
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4381
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4382
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4383
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4424
|
+
return pageOfConversation(conversation, options);
|
|
4425
|
+
}
|
|
4426
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4427
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4428
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4429
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4430
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4384
4431
|
if (floor) {
|
|
4385
4432
|
try {
|
|
4386
4433
|
if (floor.byteOffset > statSync5(filePath).size) floor = null;
|
|
4387
4434
|
} catch {
|
|
4388
4435
|
}
|
|
4389
4436
|
}
|
|
4390
|
-
|
|
4437
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4438
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4439
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4440
|
+
return {
|
|
4441
|
+
messages: page.messages,
|
|
4442
|
+
total,
|
|
4443
|
+
fromIndex,
|
|
4444
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4445
|
+
};
|
|
4391
4446
|
}
|
|
4392
4447
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4393
4448
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4876,36 +4931,94 @@ var ConversationScanner = class {
|
|
|
4876
4931
|
}
|
|
4877
4932
|
}
|
|
4878
4933
|
// Return one bounded window of a conversation's messages plus the total
|
|
4879
|
-
// message count,
|
|
4880
|
-
// holding the whole
|
|
4934
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
4935
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
4936
|
+
// conversation itself.
|
|
4881
4937
|
//
|
|
4882
|
-
//
|
|
4938
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
4939
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
4940
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
4941
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4883
4942
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4884
4943
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4885
4944
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4886
4945
|
// same contract as getConversation.
|
|
4887
4946
|
//
|
|
4888
|
-
//
|
|
4889
|
-
//
|
|
4890
|
-
//
|
|
4891
|
-
//
|
|
4892
|
-
//
|
|
4893
|
-
//
|
|
4894
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4895
|
-
// which is identical by construction.
|
|
4947
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
4948
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
4949
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
4950
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
4951
|
+
// whole per call.
|
|
4896
4952
|
async getConversationPage(id, options) {
|
|
4953
|
+
const cid = canonicalPath(id);
|
|
4954
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
4955
|
+
if (!known) return null;
|
|
4956
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
4957
|
+
if (!this.persistent && isClaude) {
|
|
4958
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
4959
|
+
if (!index) return null;
|
|
4960
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
4961
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
4962
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
4963
|
+
let floor = null;
|
|
4964
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
4965
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
4966
|
+
return {
|
|
4967
|
+
messages: page.messages,
|
|
4968
|
+
total,
|
|
4969
|
+
fromIndex,
|
|
4970
|
+
meta: index.meta,
|
|
4971
|
+
turnDurations: index.turnDurations
|
|
4972
|
+
};
|
|
4973
|
+
}
|
|
4974
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
4975
|
+
if (!meta) return null;
|
|
4897
4976
|
if (this.persistent) {
|
|
4898
|
-
|
|
4977
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
4978
|
+
return page && { ...page, meta };
|
|
4979
|
+
}
|
|
4980
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
4981
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
4982
|
+
}
|
|
4983
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
4984
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
4985
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
4986
|
+
// file is never held, and the index LRU is capped.
|
|
4987
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
4988
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
4989
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
4990
|
+
pageIndexes = new LRUCache(32);
|
|
4991
|
+
async pageIndexFor(filePath, account) {
|
|
4992
|
+
let stat6;
|
|
4993
|
+
try {
|
|
4994
|
+
stat6 = statSync6(filePath);
|
|
4995
|
+
} catch {
|
|
4996
|
+
this.pageIndexes.delete(filePath);
|
|
4997
|
+
return null;
|
|
4899
4998
|
}
|
|
4900
|
-
const
|
|
4901
|
-
if (
|
|
4902
|
-
const
|
|
4903
|
-
const
|
|
4904
|
-
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
const
|
|
4908
|
-
|
|
4999
|
+
const cached = this.pageIndexes.get(filePath);
|
|
5000
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
5001
|
+
const state = initialReducerState();
|
|
5002
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
5003
|
+
reduceLine(state, entry, this.lastTier);
|
|
5004
|
+
return false;
|
|
5005
|
+
});
|
|
5006
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
5007
|
+
if (!meta) {
|
|
5008
|
+
this.pageIndexes.delete(filePath);
|
|
5009
|
+
return null;
|
|
5010
|
+
}
|
|
5011
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
5012
|
+
const index = {
|
|
5013
|
+
size: stat6.size,
|
|
5014
|
+
mtimeMs: stat6.mtimeMs,
|
|
5015
|
+
checkpoints,
|
|
5016
|
+
meta,
|
|
5017
|
+
rawTotal: state.pageMessageCount,
|
|
5018
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
5019
|
+
};
|
|
5020
|
+
this.pageIndexes.set(filePath, index);
|
|
5021
|
+
return index;
|
|
4909
5022
|
}
|
|
4910
5023
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
4911
5024
|
// scan() or metadata index. This is the cold-start fast path: a single
|