@threadbase-sh/scanner 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +158 -45
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +157 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +157 -44
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -1
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -1
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -841,7 +841,8 @@ function initialReducerState() {
|
|
|
841
841
|
snippetParts: [],
|
|
842
842
|
previewLength: 0,
|
|
843
843
|
snippetLength: 0,
|
|
844
|
-
badJsonLines: 0
|
|
844
|
+
badJsonLines: 0,
|
|
845
|
+
turnDurations: []
|
|
845
846
|
};
|
|
846
847
|
}
|
|
847
848
|
function reduceLine(state, entry, tier) {
|
|
@@ -855,6 +856,8 @@ function reduceLine(state, entry, tier) {
|
|
|
855
856
|
const ts = entry.timestamp;
|
|
856
857
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
857
858
|
}
|
|
859
|
+
const turnDuration = turnDurationOf(entry);
|
|
860
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
858
861
|
const type = entry.type;
|
|
859
862
|
if (type === "last-prompt") {
|
|
860
863
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -953,6 +956,16 @@ function getShortProjectName(fullPath) {
|
|
|
953
956
|
}
|
|
954
957
|
|
|
955
958
|
// src/parser.ts
|
|
959
|
+
function turnDurationOf(entry) {
|
|
960
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
961
|
+
return null;
|
|
962
|
+
}
|
|
963
|
+
return {
|
|
964
|
+
durationMs: entry.durationMs,
|
|
965
|
+
messageCount: entry.messageCount || 0,
|
|
966
|
+
uuid: entry.uuid
|
|
967
|
+
};
|
|
968
|
+
}
|
|
956
969
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
957
970
|
const log = getLogger();
|
|
958
971
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -1006,12 +1019,9 @@ async function parseConversation(filePath, account) {
|
|
|
1006
1019
|
badJsonLines++;
|
|
1007
1020
|
continue;
|
|
1008
1021
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
messageCount: entry.messageCount || 0,
|
|
1013
|
-
uuid: entry.uuid
|
|
1014
|
-
});
|
|
1022
|
+
const turnDuration = turnDurationOf(entry);
|
|
1023
|
+
if (turnDuration) {
|
|
1024
|
+
turnDurations.push(turnDuration);
|
|
1015
1025
|
continue;
|
|
1016
1026
|
}
|
|
1017
1027
|
const message = reduceConvLine(state, entry);
|
|
@@ -2698,7 +2708,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
2698
2708
|
}
|
|
2699
2709
|
return { offset, line };
|
|
2700
2710
|
}
|
|
2701
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
2711
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
2702
2712
|
const checkpoints = [];
|
|
2703
2713
|
const state = from ? from.state : initialConvState();
|
|
2704
2714
|
let index = from ? from.messageIndex : 0;
|
|
@@ -2718,7 +2728,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
2718
2728
|
});
|
|
2719
2729
|
}
|
|
2720
2730
|
return false;
|
|
2721
|
-
}
|
|
2731
|
+
},
|
|
2732
|
+
onEntry
|
|
2722
2733
|
);
|
|
2723
2734
|
return checkpoints;
|
|
2724
2735
|
}
|
|
@@ -2739,6 +2750,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
2739
2750
|
applyTeamInfo(window, state);
|
|
2740
2751
|
return { messages: window, total, fromIndex };
|
|
2741
2752
|
}
|
|
2753
|
+
function isInjectedTurn(message, isInjected) {
|
|
2754
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
2755
|
+
}
|
|
2756
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
2757
|
+
let skip = 0;
|
|
2758
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
2759
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
2760
|
+
skip += 1;
|
|
2761
|
+
return false;
|
|
2762
|
+
});
|
|
2763
|
+
return skip;
|
|
2764
|
+
}
|
|
2765
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
2766
|
+
const total = Math.max(0, rawTotal - skip);
|
|
2767
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
2768
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
2769
|
+
return {
|
|
2770
|
+
total,
|
|
2771
|
+
fromIndex,
|
|
2772
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
2773
|
+
};
|
|
2774
|
+
}
|
|
2775
|
+
function pageOfConversation(conversation, options) {
|
|
2776
|
+
const { messages } = conversation;
|
|
2777
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
2778
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
2779
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
2780
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
2781
|
+
return {
|
|
2782
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
2783
|
+
total,
|
|
2784
|
+
fromIndex,
|
|
2785
|
+
turnDurations: conversation.turnDurations
|
|
2786
|
+
};
|
|
2787
|
+
}
|
|
2742
2788
|
|
|
2743
2789
|
// src/persistent/conversation-stream.ts
|
|
2744
2790
|
async function foldTail(filePath, resume) {
|
|
@@ -4268,6 +4314,7 @@ var PersistentEngine = class {
|
|
|
4268
4314
|
const tier = resolveTier(tierName, customTiers);
|
|
4269
4315
|
const filePath = canonicalPath(rawFilePath);
|
|
4270
4316
|
const existing = this.files.getByPath(filePath);
|
|
4317
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4271
4318
|
const classified = classify(filePath, existing);
|
|
4272
4319
|
const stat6 = classified.stat;
|
|
4273
4320
|
let change = classified.change;
|
|
@@ -4275,7 +4322,7 @@ var PersistentEngine = class {
|
|
|
4275
4322
|
this.markDeleted(filePath);
|
|
4276
4323
|
return { meta: null, change: "vanished" };
|
|
4277
4324
|
}
|
|
4278
|
-
if (change === "unchanged" && !
|
|
4325
|
+
if (change === "unchanged" && !refold) {
|
|
4279
4326
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4280
4327
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4281
4328
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4292,7 +4339,7 @@ var PersistentEngine = class {
|
|
|
4292
4339
|
);
|
|
4293
4340
|
return { meta: meta2, change };
|
|
4294
4341
|
}
|
|
4295
|
-
const resume = change === "appended" && !
|
|
4342
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4296
4343
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4297
4344
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4298
4345
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4453,6 +4500,10 @@ var PersistentEngine = class {
|
|
|
4453
4500
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4454
4501
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4455
4502
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4503
|
+
//
|
|
4504
|
+
// The window is over the conversation with its leading injected-context run
|
|
4505
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4506
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4456
4507
|
async getPage(id, options) {
|
|
4457
4508
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4458
4509
|
if (!meta) return null;
|
|
@@ -4460,24 +4511,28 @@ var PersistentEngine = class {
|
|
|
4460
4511
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4461
4512
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4462
4513
|
if (!conversation) return null;
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4471
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4472
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4473
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4514
|
+
return pageOfConversation(conversation, options);
|
|
4515
|
+
}
|
|
4516
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4517
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4518
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4519
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4520
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4474
4521
|
if (floor) {
|
|
4475
4522
|
try {
|
|
4476
4523
|
if (floor.byteOffset > (0, import_fs13.statSync)(filePath).size) floor = null;
|
|
4477
4524
|
} catch {
|
|
4478
4525
|
}
|
|
4479
4526
|
}
|
|
4480
|
-
|
|
4527
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4528
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4529
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4530
|
+
return {
|
|
4531
|
+
messages: page.messages,
|
|
4532
|
+
total,
|
|
4533
|
+
fromIndex,
|
|
4534
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4535
|
+
};
|
|
4481
4536
|
}
|
|
4482
4537
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4483
4538
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4966,36 +5021,94 @@ var ConversationScanner = class {
|
|
|
4966
5021
|
}
|
|
4967
5022
|
}
|
|
4968
5023
|
// Return one bounded window of a conversation's messages plus the total
|
|
4969
|
-
// message count,
|
|
4970
|
-
// holding the whole
|
|
5024
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
5025
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
5026
|
+
// conversation itself.
|
|
4971
5027
|
//
|
|
4972
|
-
//
|
|
5028
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
5029
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
5030
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
5031
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4973
5032
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4974
5033
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4975
5034
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4976
5035
|
// same contract as getConversation.
|
|
4977
5036
|
//
|
|
4978
|
-
//
|
|
4979
|
-
//
|
|
4980
|
-
//
|
|
4981
|
-
//
|
|
4982
|
-
//
|
|
4983
|
-
//
|
|
4984
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4985
|
-
// which is identical by construction.
|
|
5037
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
5038
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
5039
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
5040
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
5041
|
+
// whole per call.
|
|
4986
5042
|
async getConversationPage(id, options) {
|
|
5043
|
+
const cid = canonicalPath(id);
|
|
5044
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
5045
|
+
if (!known) return null;
|
|
5046
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
5047
|
+
if (!this.persistent && isClaude) {
|
|
5048
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
5049
|
+
if (!index) return null;
|
|
5050
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
5051
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
5052
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
5053
|
+
let floor = null;
|
|
5054
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
5055
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
5056
|
+
return {
|
|
5057
|
+
messages: page.messages,
|
|
5058
|
+
total,
|
|
5059
|
+
fromIndex,
|
|
5060
|
+
meta: index.meta,
|
|
5061
|
+
turnDurations: index.turnDurations
|
|
5062
|
+
};
|
|
5063
|
+
}
|
|
5064
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
5065
|
+
if (!meta) return null;
|
|
4987
5066
|
if (this.persistent) {
|
|
4988
|
-
|
|
5067
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
5068
|
+
return page && { ...page, meta };
|
|
5069
|
+
}
|
|
5070
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
5071
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
5072
|
+
}
|
|
5073
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
5074
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
5075
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
5076
|
+
// file is never held, and the index LRU is capped.
|
|
5077
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
5078
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
5079
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
5080
|
+
pageIndexes = new LRUCache(32);
|
|
5081
|
+
async pageIndexFor(filePath, account) {
|
|
5082
|
+
let stat6;
|
|
5083
|
+
try {
|
|
5084
|
+
stat6 = (0, import_fs14.statSync)(filePath);
|
|
5085
|
+
} catch {
|
|
5086
|
+
this.pageIndexes.delete(filePath);
|
|
5087
|
+
return null;
|
|
4989
5088
|
}
|
|
4990
|
-
const
|
|
4991
|
-
if (
|
|
4992
|
-
const
|
|
4993
|
-
const
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
const
|
|
4998
|
-
|
|
5089
|
+
const cached = this.pageIndexes.get(filePath);
|
|
5090
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
5091
|
+
const state = initialReducerState();
|
|
5092
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
5093
|
+
reduceLine(state, entry, this.lastTier);
|
|
5094
|
+
return false;
|
|
5095
|
+
});
|
|
5096
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
5097
|
+
if (!meta) {
|
|
5098
|
+
this.pageIndexes.delete(filePath);
|
|
5099
|
+
return null;
|
|
5100
|
+
}
|
|
5101
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
5102
|
+
const index = {
|
|
5103
|
+
size: stat6.size,
|
|
5104
|
+
mtimeMs: stat6.mtimeMs,
|
|
5105
|
+
checkpoints,
|
|
5106
|
+
meta,
|
|
5107
|
+
rawTotal: state.pageMessageCount,
|
|
5108
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
5109
|
+
};
|
|
5110
|
+
this.pageIndexes.set(filePath, index);
|
|
5111
|
+
return index;
|
|
4999
5112
|
}
|
|
5000
5113
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
5001
5114
|
// scan() or metadata index. This is the cold-start fast path: a single
|