@threadbase-sh/scanner 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +161 -58
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +160 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +160 -57
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -2
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -3
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -740,8 +740,6 @@ function initialReducerState() {
|
|
|
740
740
|
lastMessageSender: "user",
|
|
741
741
|
agentId: "",
|
|
742
742
|
isSidechain: false,
|
|
743
|
-
isTeammate: false,
|
|
744
|
-
firstUserSeen: false,
|
|
745
743
|
firstMessage: null,
|
|
746
744
|
lastMessage: null,
|
|
747
745
|
lastPrompt: "",
|
|
@@ -751,7 +749,8 @@ function initialReducerState() {
|
|
|
751
749
|
snippetParts: [],
|
|
752
750
|
previewLength: 0,
|
|
753
751
|
snippetLength: 0,
|
|
754
|
-
badJsonLines: 0
|
|
752
|
+
badJsonLines: 0,
|
|
753
|
+
turnDurations: []
|
|
755
754
|
};
|
|
756
755
|
}
|
|
757
756
|
function reduceLine(state, entry, tier) {
|
|
@@ -765,6 +764,8 @@ function reduceLine(state, entry, tier) {
|
|
|
765
764
|
const ts = entry.timestamp;
|
|
766
765
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
767
766
|
}
|
|
767
|
+
const turnDuration = turnDurationOf(entry);
|
|
768
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
768
769
|
const type = entry.type;
|
|
769
770
|
if (type === "last-prompt") {
|
|
770
771
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -778,10 +779,6 @@ function reduceLine(state, entry, tier) {
|
|
|
778
779
|
return;
|
|
779
780
|
}
|
|
780
781
|
if (state.model === null && msg?.model) state.model = msg.model;
|
|
781
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
782
|
-
state.firstUserSeen = true;
|
|
783
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
784
|
-
}
|
|
785
782
|
const content = extractTextContent(msg?.content);
|
|
786
783
|
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
787
784
|
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
@@ -848,7 +845,9 @@ function finalizeMeta(state, filePath, account, tier) {
|
|
|
848
845
|
// A sidechain line's `sessionId` IS the parent's id — that is exactly the
|
|
849
846
|
// conflation `subagentId` exists to undo, so it is the parent UUID here.
|
|
850
847
|
parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
|
|
851
|
-
|
|
848
|
+
// A teammate's records carry teamName; the lead's never do, yet the lead
|
|
849
|
+
// receives <teammate-message> tags too, so the tag is not the discriminator.
|
|
850
|
+
isTeammate: Boolean(state.teamName),
|
|
852
851
|
teamName: state.teamName || null,
|
|
853
852
|
toolNames: state.toolNames,
|
|
854
853
|
firstMessage: state.firstMessage,
|
|
@@ -863,6 +862,16 @@ function getShortProjectName(fullPath) {
|
|
|
863
862
|
}
|
|
864
863
|
|
|
865
864
|
// src/parser.ts
|
|
865
|
+
function turnDurationOf(entry) {
|
|
866
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
867
|
+
return null;
|
|
868
|
+
}
|
|
869
|
+
return {
|
|
870
|
+
durationMs: entry.durationMs,
|
|
871
|
+
messageCount: entry.messageCount || 0,
|
|
872
|
+
uuid: entry.uuid
|
|
873
|
+
};
|
|
874
|
+
}
|
|
866
875
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
867
876
|
const log = getLogger();
|
|
868
877
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -916,12 +925,9 @@ async function parseConversation(filePath, account) {
|
|
|
916
925
|
badJsonLines++;
|
|
917
926
|
continue;
|
|
918
927
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
messageCount: entry.messageCount || 0,
|
|
923
|
-
uuid: entry.uuid
|
|
924
|
-
});
|
|
928
|
+
const turnDuration = turnDurationOf(entry);
|
|
929
|
+
if (turnDuration) {
|
|
930
|
+
turnDurations.push(turnDuration);
|
|
925
931
|
continue;
|
|
926
932
|
}
|
|
927
933
|
const message = reduceConvLine(state, entry);
|
|
@@ -1019,12 +1025,6 @@ function isOnlyToolResultContent(content) {
|
|
|
1019
1025
|
if (!Array.isArray(content)) return false;
|
|
1020
1026
|
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
1021
1027
|
}
|
|
1022
|
-
function isTeammateContent(content) {
|
|
1023
|
-
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
1024
|
-
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
1025
|
-
).join("") : "";
|
|
1026
|
-
return raw.includes("<teammate-message");
|
|
1027
|
-
}
|
|
1028
1028
|
function extractThinking(content) {
|
|
1029
1029
|
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
1030
1030
|
const blocks = content.filter((item) => item?.type === "thinking");
|
|
@@ -2608,7 +2608,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2608
2608
|
}
|
|
2609
2609
|
return { offset, line };
|
|
2610
2610
|
}
|
|
2611
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
2611
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
2612
2612
|
const checkpoints = [];
|
|
2613
2613
|
const state = from ? from.state : initialConvState();
|
|
2614
2614
|
let index = from ? from.messageIndex : 0;
|
|
@@ -2628,7 +2628,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
2628
2628
|
});
|
|
2629
2629
|
}
|
|
2630
2630
|
return false;
|
|
2631
|
-
}
|
|
2631
|
+
},
|
|
2632
|
+
onEntry
|
|
2632
2633
|
);
|
|
2633
2634
|
return checkpoints;
|
|
2634
2635
|
}
|
|
@@ -2649,6 +2650,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
2649
2650
|
applyTeamInfo(window, state);
|
|
2650
2651
|
return { messages: window, total, fromIndex };
|
|
2651
2652
|
}
|
|
2653
|
+
function isInjectedTurn(message, isInjected) {
|
|
2654
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
2655
|
+
}
|
|
2656
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
2657
|
+
let skip = 0;
|
|
2658
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
2659
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
2660
|
+
skip += 1;
|
|
2661
|
+
return false;
|
|
2662
|
+
});
|
|
2663
|
+
return skip;
|
|
2664
|
+
}
|
|
2665
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
2666
|
+
const total = Math.max(0, rawTotal - skip);
|
|
2667
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2668
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2669
|
+
return {
|
|
2670
|
+
total,
|
|
2671
|
+
fromIndex,
|
|
2672
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
2673
|
+
};
|
|
2674
|
+
}
|
|
2675
|
+
function pageOfConversation(conversation, options) {
|
|
2676
|
+
const { messages } = conversation;
|
|
2677
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
2678
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
2679
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
2680
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
2681
|
+
return {
|
|
2682
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
2683
|
+
total,
|
|
2684
|
+
fromIndex,
|
|
2685
|
+
turnDurations: conversation.turnDurations
|
|
2686
|
+
};
|
|
2687
|
+
}
|
|
2652
2688
|
|
|
2653
2689
|
// src/persistent/conversation-stream.ts
|
|
2654
2690
|
async function foldTail(filePath, resume) {
|
|
@@ -4178,6 +4214,7 @@ var PersistentEngine = class {
|
|
|
4178
4214
|
const tier = resolveTier(tierName, customTiers);
|
|
4179
4215
|
const filePath = canonicalPath(rawFilePath);
|
|
4180
4216
|
const existing = this.files.getByPath(filePath);
|
|
4217
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4181
4218
|
const classified = classify(filePath, existing);
|
|
4182
4219
|
const stat6 = classified.stat;
|
|
4183
4220
|
let change = classified.change;
|
|
@@ -4185,7 +4222,7 @@ var PersistentEngine = class {
|
|
|
4185
4222
|
this.markDeleted(filePath);
|
|
4186
4223
|
return { meta: null, change: "vanished" };
|
|
4187
4224
|
}
|
|
4188
|
-
if (change === "unchanged" && !
|
|
4225
|
+
if (change === "unchanged" && !refold) {
|
|
4189
4226
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4190
4227
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4191
4228
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4202,7 +4239,7 @@ var PersistentEngine = class {
|
|
|
4202
4239
|
);
|
|
4203
4240
|
return { meta: meta2, change };
|
|
4204
4241
|
}
|
|
4205
|
-
const resume = change === "appended" && !
|
|
4242
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4206
4243
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4207
4244
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4208
4245
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4363,6 +4400,10 @@ var PersistentEngine = class {
|
|
|
4363
4400
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4364
4401
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4365
4402
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4403
|
+
//
|
|
4404
|
+
// The window is over the conversation with its leading injected-context run
|
|
4405
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4406
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4366
4407
|
async getPage(id, options) {
|
|
4367
4408
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4368
4409
|
if (!meta) return null;
|
|
@@ -4370,24 +4411,28 @@ var PersistentEngine = class {
|
|
|
4370
4411
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4371
4412
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4372
4413
|
if (!conversation) return null;
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4381
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4382
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4383
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4414
|
+
return pageOfConversation(conversation, options);
|
|
4415
|
+
}
|
|
4416
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4417
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4418
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4419
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4420
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4384
4421
|
if (floor) {
|
|
4385
4422
|
try {
|
|
4386
4423
|
if (floor.byteOffset > statSync5(filePath).size) floor = null;
|
|
4387
4424
|
} catch {
|
|
4388
4425
|
}
|
|
4389
4426
|
}
|
|
4390
|
-
|
|
4427
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4428
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4429
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4430
|
+
return {
|
|
4431
|
+
messages: page.messages,
|
|
4432
|
+
total,
|
|
4433
|
+
fromIndex,
|
|
4434
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4435
|
+
};
|
|
4391
4436
|
}
|
|
4392
4437
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4393
4438
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4876,36 +4921,94 @@ var ConversationScanner = class {
|
|
|
4876
4921
|
}
|
|
4877
4922
|
}
|
|
4878
4923
|
// Return one bounded window of a conversation's messages plus the total
|
|
4879
|
-
// message count,
|
|
4880
|
-
// holding the whole
|
|
4924
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
4925
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
4926
|
+
// conversation itself.
|
|
4881
4927
|
//
|
|
4882
|
-
//
|
|
4928
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
4929
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
4930
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
4931
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4883
4932
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4884
4933
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4885
4934
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4886
4935
|
// same contract as getConversation.
|
|
4887
4936
|
//
|
|
4888
|
-
//
|
|
4889
|
-
//
|
|
4890
|
-
//
|
|
4891
|
-
//
|
|
4892
|
-
//
|
|
4893
|
-
//
|
|
4894
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4895
|
-
// which is identical by construction.
|
|
4937
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
4938
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
4939
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
4940
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
4941
|
+
// whole per call.
|
|
4896
4942
|
async getConversationPage(id, options) {
|
|
4943
|
+
const cid = canonicalPath(id);
|
|
4944
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
4945
|
+
if (!known) return null;
|
|
4946
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
4947
|
+
if (!this.persistent && isClaude) {
|
|
4948
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
4949
|
+
if (!index) return null;
|
|
4950
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
4951
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
4952
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
4953
|
+
let floor = null;
|
|
4954
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
4955
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
4956
|
+
return {
|
|
4957
|
+
messages: page.messages,
|
|
4958
|
+
total,
|
|
4959
|
+
fromIndex,
|
|
4960
|
+
meta: index.meta,
|
|
4961
|
+
turnDurations: index.turnDurations
|
|
4962
|
+
};
|
|
4963
|
+
}
|
|
4964
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
4965
|
+
if (!meta) return null;
|
|
4897
4966
|
if (this.persistent) {
|
|
4898
|
-
|
|
4967
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
4968
|
+
return page && { ...page, meta };
|
|
4969
|
+
}
|
|
4970
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
4971
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
4972
|
+
}
|
|
4973
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
4974
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
4975
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
4976
|
+
// file is never held, and the index LRU is capped.
|
|
4977
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
4978
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
4979
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
4980
|
+
pageIndexes = new LRUCache(32);
|
|
4981
|
+
async pageIndexFor(filePath, account) {
|
|
4982
|
+
let stat6;
|
|
4983
|
+
try {
|
|
4984
|
+
stat6 = statSync6(filePath);
|
|
4985
|
+
} catch {
|
|
4986
|
+
this.pageIndexes.delete(filePath);
|
|
4987
|
+
return null;
|
|
4899
4988
|
}
|
|
4900
|
-
const
|
|
4901
|
-
if (
|
|
4902
|
-
const
|
|
4903
|
-
const
|
|
4904
|
-
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
const
|
|
4908
|
-
|
|
4989
|
+
const cached = this.pageIndexes.get(filePath);
|
|
4990
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
4991
|
+
const state = initialReducerState();
|
|
4992
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
4993
|
+
reduceLine(state, entry, this.lastTier);
|
|
4994
|
+
return false;
|
|
4995
|
+
});
|
|
4996
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
4997
|
+
if (!meta) {
|
|
4998
|
+
this.pageIndexes.delete(filePath);
|
|
4999
|
+
return null;
|
|
5000
|
+
}
|
|
5001
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
5002
|
+
const index = {
|
|
5003
|
+
size: stat6.size,
|
|
5004
|
+
mtimeMs: stat6.mtimeMs,
|
|
5005
|
+
checkpoints,
|
|
5006
|
+
meta,
|
|
5007
|
+
rawTotal: state.pageMessageCount,
|
|
5008
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
5009
|
+
};
|
|
5010
|
+
this.pageIndexes.set(filePath, index);
|
|
5011
|
+
return index;
|
|
4909
5012
|
}
|
|
4910
5013
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
4911
5014
|
// scan() or metadata index. This is the cold-start fast path: a single
|