@threadbase-sh/scanner 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +161 -58
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +160 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +160 -57
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -2
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -3
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -830,8 +830,6 @@ function initialReducerState() {
|
|
|
830
830
|
lastMessageSender: "user",
|
|
831
831
|
agentId: "",
|
|
832
832
|
isSidechain: false,
|
|
833
|
-
isTeammate: false,
|
|
834
|
-
firstUserSeen: false,
|
|
835
833
|
firstMessage: null,
|
|
836
834
|
lastMessage: null,
|
|
837
835
|
lastPrompt: "",
|
|
@@ -841,7 +839,8 @@ function initialReducerState() {
|
|
|
841
839
|
snippetParts: [],
|
|
842
840
|
previewLength: 0,
|
|
843
841
|
snippetLength: 0,
|
|
844
|
-
badJsonLines: 0
|
|
842
|
+
badJsonLines: 0,
|
|
843
|
+
turnDurations: []
|
|
845
844
|
};
|
|
846
845
|
}
|
|
847
846
|
function reduceLine(state, entry, tier) {
|
|
@@ -855,6 +854,8 @@ function reduceLine(state, entry, tier) {
|
|
|
855
854
|
const ts = entry.timestamp;
|
|
856
855
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
857
856
|
}
|
|
857
|
+
const turnDuration = turnDurationOf(entry);
|
|
858
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
858
859
|
const type = entry.type;
|
|
859
860
|
if (type === "last-prompt") {
|
|
860
861
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -868,10 +869,6 @@ function reduceLine(state, entry, tier) {
|
|
|
868
869
|
return;
|
|
869
870
|
}
|
|
870
871
|
if (state.model === null && msg?.model) state.model = msg.model;
|
|
871
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
872
|
-
state.firstUserSeen = true;
|
|
873
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
874
|
-
}
|
|
875
872
|
const content = extractTextContent(msg?.content);
|
|
876
873
|
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
877
874
|
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
@@ -938,7 +935,9 @@ function finalizeMeta(state, filePath, account, tier) {
|
|
|
938
935
|
// A sidechain line's `sessionId` IS the parent's id — that is exactly the
|
|
939
936
|
// conflation `subagentId` exists to undo, so it is the parent UUID here.
|
|
940
937
|
parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
|
|
941
|
-
|
|
938
|
+
// A teammate's records carry teamName; the lead's never do, yet the lead
|
|
939
|
+
// receives <teammate-message> tags too, so the tag is not the discriminator.
|
|
940
|
+
isTeammate: Boolean(state.teamName),
|
|
942
941
|
teamName: state.teamName || null,
|
|
943
942
|
toolNames: state.toolNames,
|
|
944
943
|
firstMessage: state.firstMessage,
|
|
@@ -953,6 +952,16 @@ function getShortProjectName(fullPath) {
|
|
|
953
952
|
}
|
|
954
953
|
|
|
955
954
|
// src/parser.ts
|
|
955
|
+
function turnDurationOf(entry) {
|
|
956
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
957
|
+
return null;
|
|
958
|
+
}
|
|
959
|
+
return {
|
|
960
|
+
durationMs: entry.durationMs,
|
|
961
|
+
messageCount: entry.messageCount || 0,
|
|
962
|
+
uuid: entry.uuid
|
|
963
|
+
};
|
|
964
|
+
}
|
|
956
965
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
957
966
|
const log = getLogger();
|
|
958
967
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -1006,12 +1015,9 @@ async function parseConversation(filePath, account) {
|
|
|
1006
1015
|
badJsonLines++;
|
|
1007
1016
|
continue;
|
|
1008
1017
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
messageCount: entry.messageCount || 0,
|
|
1013
|
-
uuid: entry.uuid
|
|
1014
|
-
});
|
|
1018
|
+
const turnDuration = turnDurationOf(entry);
|
|
1019
|
+
if (turnDuration) {
|
|
1020
|
+
turnDurations.push(turnDuration);
|
|
1015
1021
|
continue;
|
|
1016
1022
|
}
|
|
1017
1023
|
const message = reduceConvLine(state, entry);
|
|
@@ -1109,12 +1115,6 @@ function isOnlyToolResultContent(content) {
|
|
|
1109
1115
|
if (!Array.isArray(content)) return false;
|
|
1110
1116
|
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
1111
1117
|
}
|
|
1112
|
-
function isTeammateContent(content) {
|
|
1113
|
-
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
1114
|
-
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
1115
|
-
).join("") : "";
|
|
1116
|
-
return raw.includes("<teammate-message");
|
|
1117
|
-
}
|
|
1118
1118
|
function extractThinking(content) {
|
|
1119
1119
|
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
1120
1120
|
const blocks = content.filter((item) => item?.type === "thinking");
|
|
@@ -2698,7 +2698,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2698
2698
|
}
|
|
2699
2699
|
return { offset, line };
|
|
2700
2700
|
}
|
|
2701
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
2701
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
2702
2702
|
const checkpoints = [];
|
|
2703
2703
|
const state = from ? from.state : initialConvState();
|
|
2704
2704
|
let index = from ? from.messageIndex : 0;
|
|
@@ -2718,7 +2718,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
2718
2718
|
});
|
|
2719
2719
|
}
|
|
2720
2720
|
return false;
|
|
2721
|
-
}
|
|
2721
|
+
},
|
|
2722
|
+
onEntry
|
|
2722
2723
|
);
|
|
2723
2724
|
return checkpoints;
|
|
2724
2725
|
}
|
|
@@ -2739,6 +2740,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
2739
2740
|
applyTeamInfo(window, state);
|
|
2740
2741
|
return { messages: window, total, fromIndex };
|
|
2741
2742
|
}
|
|
2743
|
+
function isInjectedTurn(message, isInjected) {
|
|
2744
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
2745
|
+
}
|
|
2746
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
2747
|
+
let skip = 0;
|
|
2748
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
2749
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
2750
|
+
skip += 1;
|
|
2751
|
+
return false;
|
|
2752
|
+
});
|
|
2753
|
+
return skip;
|
|
2754
|
+
}
|
|
2755
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
2756
|
+
const total = Math.max(0, rawTotal - skip);
|
|
2757
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2758
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2759
|
+
return {
|
|
2760
|
+
total,
|
|
2761
|
+
fromIndex,
|
|
2762
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
function pageOfConversation(conversation, options) {
|
|
2766
|
+
const { messages } = conversation;
|
|
2767
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
2768
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
2769
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
2770
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
2771
|
+
return {
|
|
2772
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
2773
|
+
total,
|
|
2774
|
+
fromIndex,
|
|
2775
|
+
turnDurations: conversation.turnDurations
|
|
2776
|
+
};
|
|
2777
|
+
}
|
|
2742
2778
|
|
|
2743
2779
|
// src/persistent/conversation-stream.ts
|
|
2744
2780
|
async function foldTail(filePath, resume) {
|
|
@@ -4268,6 +4304,7 @@ var PersistentEngine = class {
|
|
|
4268
4304
|
const tier = resolveTier(tierName, customTiers);
|
|
4269
4305
|
const filePath = canonicalPath(rawFilePath);
|
|
4270
4306
|
const existing = this.files.getByPath(filePath);
|
|
4307
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4271
4308
|
const classified = classify(filePath, existing);
|
|
4272
4309
|
const stat6 = classified.stat;
|
|
4273
4310
|
let change = classified.change;
|
|
@@ -4275,7 +4312,7 @@ var PersistentEngine = class {
|
|
|
4275
4312
|
this.markDeleted(filePath);
|
|
4276
4313
|
return { meta: null, change: "vanished" };
|
|
4277
4314
|
}
|
|
4278
|
-
if (change === "unchanged" && !
|
|
4315
|
+
if (change === "unchanged" && !refold) {
|
|
4279
4316
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4280
4317
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4281
4318
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4292,7 +4329,7 @@ var PersistentEngine = class {
|
|
|
4292
4329
|
);
|
|
4293
4330
|
return { meta: meta2, change };
|
|
4294
4331
|
}
|
|
4295
|
-
const resume = change === "appended" && !
|
|
4332
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4296
4333
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4297
4334
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4298
4335
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4453,6 +4490,10 @@ var PersistentEngine = class {
|
|
|
4453
4490
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4454
4491
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4455
4492
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4493
|
+
//
|
|
4494
|
+
// The window is over the conversation with its leading injected-context run
|
|
4495
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4496
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4456
4497
|
async getPage(id, options) {
|
|
4457
4498
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4458
4499
|
if (!meta) return null;
|
|
@@ -4460,24 +4501,28 @@ var PersistentEngine = class {
|
|
|
4460
4501
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4461
4502
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4462
4503
|
if (!conversation) return null;
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4471
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4472
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4473
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4504
|
+
return pageOfConversation(conversation, options);
|
|
4505
|
+
}
|
|
4506
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4507
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4508
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4509
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4510
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4474
4511
|
if (floor) {
|
|
4475
4512
|
try {
|
|
4476
4513
|
if (floor.byteOffset > (0, import_fs13.statSync)(filePath).size) floor = null;
|
|
4477
4514
|
} catch {
|
|
4478
4515
|
}
|
|
4479
4516
|
}
|
|
4480
|
-
|
|
4517
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4518
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4519
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4520
|
+
return {
|
|
4521
|
+
messages: page.messages,
|
|
4522
|
+
total,
|
|
4523
|
+
fromIndex,
|
|
4524
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4525
|
+
};
|
|
4481
4526
|
}
|
|
4482
4527
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4483
4528
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4966,36 +5011,94 @@ var ConversationScanner = class {
|
|
|
4966
5011
|
}
|
|
4967
5012
|
}
|
|
4968
5013
|
// Return one bounded window of a conversation's messages plus the total
|
|
4969
|
-
// message count,
|
|
4970
|
-
// holding the whole
|
|
5014
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
5015
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
5016
|
+
// conversation itself.
|
|
4971
5017
|
//
|
|
4972
|
-
//
|
|
5018
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
5019
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
5020
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
5021
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4973
5022
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4974
5023
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4975
5024
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4976
5025
|
// same contract as getConversation.
|
|
4977
5026
|
//
|
|
4978
|
-
//
|
|
4979
|
-
//
|
|
4980
|
-
//
|
|
4981
|
-
//
|
|
4982
|
-
//
|
|
4983
|
-
//
|
|
4984
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4985
|
-
// which is identical by construction.
|
|
5027
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
5028
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
5029
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
5030
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
5031
|
+
// whole per call.
|
|
4986
5032
|
async getConversationPage(id, options) {
|
|
5033
|
+
const cid = canonicalPath(id);
|
|
5034
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
5035
|
+
if (!known) return null;
|
|
5036
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
5037
|
+
if (!this.persistent && isClaude) {
|
|
5038
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
5039
|
+
if (!index) return null;
|
|
5040
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
5041
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
5042
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
5043
|
+
let floor = null;
|
|
5044
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
5045
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
5046
|
+
return {
|
|
5047
|
+
messages: page.messages,
|
|
5048
|
+
total,
|
|
5049
|
+
fromIndex,
|
|
5050
|
+
meta: index.meta,
|
|
5051
|
+
turnDurations: index.turnDurations
|
|
5052
|
+
};
|
|
5053
|
+
}
|
|
5054
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
5055
|
+
if (!meta) return null;
|
|
4987
5056
|
if (this.persistent) {
|
|
4988
|
-
|
|
5057
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
5058
|
+
return page && { ...page, meta };
|
|
5059
|
+
}
|
|
5060
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
5061
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
5062
|
+
}
|
|
5063
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
5064
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
5065
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
5066
|
+
// file is never held, and the index LRU is capped.
|
|
5067
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
5068
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
5069
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
5070
|
+
pageIndexes = new LRUCache(32);
|
|
5071
|
+
async pageIndexFor(filePath, account) {
|
|
5072
|
+
let stat6;
|
|
5073
|
+
try {
|
|
5074
|
+
stat6 = (0, import_fs14.statSync)(filePath);
|
|
5075
|
+
} catch {
|
|
5076
|
+
this.pageIndexes.delete(filePath);
|
|
5077
|
+
return null;
|
|
4989
5078
|
}
|
|
4990
|
-
const
|
|
4991
|
-
if (
|
|
4992
|
-
const
|
|
4993
|
-
const
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
const
|
|
4998
|
-
|
|
5079
|
+
const cached = this.pageIndexes.get(filePath);
|
|
5080
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
5081
|
+
const state = initialReducerState();
|
|
5082
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
5083
|
+
reduceLine(state, entry, this.lastTier);
|
|
5084
|
+
return false;
|
|
5085
|
+
});
|
|
5086
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
5087
|
+
if (!meta) {
|
|
5088
|
+
this.pageIndexes.delete(filePath);
|
|
5089
|
+
return null;
|
|
5090
|
+
}
|
|
5091
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
5092
|
+
const index = {
|
|
5093
|
+
size: stat6.size,
|
|
5094
|
+
mtimeMs: stat6.mtimeMs,
|
|
5095
|
+
checkpoints,
|
|
5096
|
+
meta,
|
|
5097
|
+
rawTotal: state.pageMessageCount,
|
|
5098
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
5099
|
+
};
|
|
5100
|
+
this.pageIndexes.set(filePath, index);
|
|
5101
|
+
return index;
|
|
4999
5102
|
}
|
|
5000
5103
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
5001
5104
|
// scan() or metadata index. This is the cold-start fast path: a single
|