@threadbase-sh/scanner 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +161 -58
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +160 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +160 -57
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -2
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -3
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -5,7 +5,7 @@ import { Command } from "commander";
|
|
|
5
5
|
import pino2 from "pino";
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.24.1";
|
|
9
9
|
|
|
10
10
|
// src/logger.ts
|
|
11
11
|
import pino from "pino";
|
|
@@ -966,8 +966,6 @@ function initialReducerState() {
|
|
|
966
966
|
lastMessageSender: "user",
|
|
967
967
|
agentId: "",
|
|
968
968
|
isSidechain: false,
|
|
969
|
-
isTeammate: false,
|
|
970
|
-
firstUserSeen: false,
|
|
971
969
|
firstMessage: null,
|
|
972
970
|
lastMessage: null,
|
|
973
971
|
lastPrompt: "",
|
|
@@ -977,7 +975,8 @@ function initialReducerState() {
|
|
|
977
975
|
snippetParts: [],
|
|
978
976
|
previewLength: 0,
|
|
979
977
|
snippetLength: 0,
|
|
980
|
-
badJsonLines: 0
|
|
978
|
+
badJsonLines: 0,
|
|
979
|
+
turnDurations: []
|
|
981
980
|
};
|
|
982
981
|
}
|
|
983
982
|
function reduceLine(state, entry, tier) {
|
|
@@ -991,6 +990,8 @@ function reduceLine(state, entry, tier) {
|
|
|
991
990
|
const ts = entry.timestamp;
|
|
992
991
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
993
992
|
}
|
|
993
|
+
const turnDuration = turnDurationOf(entry);
|
|
994
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
994
995
|
const type = entry.type;
|
|
995
996
|
if (type === "last-prompt") {
|
|
996
997
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -1004,10 +1005,6 @@ function reduceLine(state, entry, tier) {
|
|
|
1004
1005
|
return;
|
|
1005
1006
|
}
|
|
1006
1007
|
if (state.model === null && msg?.model) state.model = msg.model;
|
|
1007
|
-
if (type === "user" && !state.firstUserSeen) {
|
|
1008
|
-
state.firstUserSeen = true;
|
|
1009
|
-
if (isTeammateContent(msg?.content)) state.isTeammate = true;
|
|
1010
|
-
}
|
|
1011
1008
|
const content = extractTextContent(msg?.content);
|
|
1012
1009
|
const hasToolUseResult = type === "user" && entry.toolUseResult != null;
|
|
1013
1010
|
const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
|
|
@@ -1074,7 +1071,9 @@ function finalizeMeta(state, filePath, account, tier) {
|
|
|
1074
1071
|
// A sidechain line's `sessionId` IS the parent's id — that is exactly the
|
|
1075
1072
|
// conflation `subagentId` exists to undo, so it is the parent UUID here.
|
|
1076
1073
|
parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
|
|
1077
|
-
|
|
1074
|
+
// A teammate's records carry teamName; the lead's never do, yet the lead
|
|
1075
|
+
// receives <teammate-message> tags too, so the tag is not the discriminator.
|
|
1076
|
+
isTeammate: Boolean(state.teamName),
|
|
1078
1077
|
teamName: state.teamName || null,
|
|
1079
1078
|
toolNames: state.toolNames,
|
|
1080
1079
|
firstMessage: state.firstMessage,
|
|
@@ -1089,6 +1088,16 @@ function getShortProjectName(fullPath) {
|
|
|
1089
1088
|
}
|
|
1090
1089
|
|
|
1091
1090
|
// src/parser.ts
|
|
1091
|
+
function turnDurationOf(entry) {
|
|
1092
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
1093
|
+
return null;
|
|
1094
|
+
}
|
|
1095
|
+
return {
|
|
1096
|
+
durationMs: entry.durationMs,
|
|
1097
|
+
messageCount: entry.messageCount || 0,
|
|
1098
|
+
uuid: entry.uuid
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1092
1101
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
1093
1102
|
const log = getLogger();
|
|
1094
1103
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -1142,12 +1151,9 @@ async function parseConversation(filePath, account) {
|
|
|
1142
1151
|
badJsonLines++;
|
|
1143
1152
|
continue;
|
|
1144
1153
|
}
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
messageCount: entry.messageCount || 0,
|
|
1149
|
-
uuid: entry.uuid
|
|
1150
|
-
});
|
|
1154
|
+
const turnDuration = turnDurationOf(entry);
|
|
1155
|
+
if (turnDuration) {
|
|
1156
|
+
turnDurations.push(turnDuration);
|
|
1151
1157
|
continue;
|
|
1152
1158
|
}
|
|
1153
1159
|
const message = reduceConvLine(state, entry);
|
|
@@ -1245,12 +1251,6 @@ function isOnlyToolResultContent(content) {
|
|
|
1245
1251
|
if (!Array.isArray(content)) return false;
|
|
1246
1252
|
return content.length > 0 && content.every((item) => item?.type === "tool_result");
|
|
1247
1253
|
}
|
|
1248
|
-
function isTeammateContent(content) {
|
|
1249
|
-
const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
|
|
1250
|
-
(item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
|
|
1251
|
-
).join("") : "";
|
|
1252
|
-
return raw.includes("<teammate-message");
|
|
1253
|
-
}
|
|
1254
1254
|
function extractThinking(content) {
|
|
1255
1255
|
if (!Array.isArray(content)) return { content: "", signature: "" };
|
|
1256
1256
|
const blocks = content.filter((item) => item?.type === "thinking");
|
|
@@ -1364,7 +1364,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
1364
1364
|
}
|
|
1365
1365
|
return { offset, line };
|
|
1366
1366
|
}
|
|
1367
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
1367
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
1368
1368
|
const checkpoints = [];
|
|
1369
1369
|
const state = from ? from.state : initialConvState();
|
|
1370
1370
|
let index = from ? from.messageIndex : 0;
|
|
@@ -1384,7 +1384,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
1384
1384
|
});
|
|
1385
1385
|
}
|
|
1386
1386
|
return false;
|
|
1387
|
-
}
|
|
1387
|
+
},
|
|
1388
|
+
onEntry
|
|
1388
1389
|
);
|
|
1389
1390
|
return checkpoints;
|
|
1390
1391
|
}
|
|
@@ -1405,6 +1406,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
1405
1406
|
applyTeamInfo(window, state);
|
|
1406
1407
|
return { messages: window, total, fromIndex };
|
|
1407
1408
|
}
|
|
1409
|
+
function isInjectedTurn(message, isInjected) {
|
|
1410
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
1411
|
+
}
|
|
1412
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
1413
|
+
let skip = 0;
|
|
1414
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
1415
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
1416
|
+
skip += 1;
|
|
1417
|
+
return false;
|
|
1418
|
+
});
|
|
1419
|
+
return skip;
|
|
1420
|
+
}
|
|
1421
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
1422
|
+
const total = Math.max(0, rawTotal - skip);
|
|
1423
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1424
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1425
|
+
return {
|
|
1426
|
+
total,
|
|
1427
|
+
fromIndex,
|
|
1428
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
1429
|
+
};
|
|
1430
|
+
}
|
|
1431
|
+
function pageOfConversation(conversation, options) {
|
|
1432
|
+
const { messages } = conversation;
|
|
1433
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
1434
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
1435
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
1436
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
1437
|
+
return {
|
|
1438
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
1439
|
+
total,
|
|
1440
|
+
fromIndex,
|
|
1441
|
+
turnDurations: conversation.turnDurations
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1408
1444
|
|
|
1409
1445
|
// src/persistent/conversation-stream.ts
|
|
1410
1446
|
async function foldTail(filePath, resume) {
|
|
@@ -4082,6 +4118,7 @@ var PersistentEngine = class {
|
|
|
4082
4118
|
const tier = resolveTier(tierName, customTiers);
|
|
4083
4119
|
const filePath = canonicalPath(rawFilePath);
|
|
4084
4120
|
const existing = this.files.getByPath(filePath);
|
|
4121
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4085
4122
|
const classified = classify(filePath, existing);
|
|
4086
4123
|
const stat6 = classified.stat;
|
|
4087
4124
|
let change = classified.change;
|
|
@@ -4089,7 +4126,7 @@ var PersistentEngine = class {
|
|
|
4089
4126
|
this.markDeleted(filePath);
|
|
4090
4127
|
return { meta: null, change: "vanished" };
|
|
4091
4128
|
}
|
|
4092
|
-
if (change === "unchanged" && !
|
|
4129
|
+
if (change === "unchanged" && !refold) {
|
|
4093
4130
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4094
4131
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4095
4132
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4106,7 +4143,7 @@ var PersistentEngine = class {
|
|
|
4106
4143
|
);
|
|
4107
4144
|
return { meta: meta2, change };
|
|
4108
4145
|
}
|
|
4109
|
-
const resume = change === "appended" && !
|
|
4146
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4110
4147
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4111
4148
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4112
4149
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4267,6 +4304,10 @@ var PersistentEngine = class {
|
|
|
4267
4304
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4268
4305
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4269
4306
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4307
|
+
//
|
|
4308
|
+
// The window is over the conversation with its leading injected-context run
|
|
4309
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4310
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4270
4311
|
async getPage(id, options) {
|
|
4271
4312
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4272
4313
|
if (!meta) return null;
|
|
@@ -4274,24 +4315,28 @@ var PersistentEngine = class {
|
|
|
4274
4315
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4275
4316
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4276
4317
|
if (!conversation) return null;
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4285
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4286
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4287
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4318
|
+
return pageOfConversation(conversation, options);
|
|
4319
|
+
}
|
|
4320
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4321
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4322
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4323
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4324
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4288
4325
|
if (floor) {
|
|
4289
4326
|
try {
|
|
4290
4327
|
if (floor.byteOffset > statSync5(filePath).size) floor = null;
|
|
4291
4328
|
} catch {
|
|
4292
4329
|
}
|
|
4293
4330
|
}
|
|
4294
|
-
|
|
4331
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4332
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4333
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4334
|
+
return {
|
|
4335
|
+
messages: page.messages,
|
|
4336
|
+
total,
|
|
4337
|
+
fromIndex,
|
|
4338
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4339
|
+
};
|
|
4295
4340
|
}
|
|
4296
4341
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4297
4342
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4816,36 +4861,94 @@ var ConversationScanner = class {
|
|
|
4816
4861
|
}
|
|
4817
4862
|
}
|
|
4818
4863
|
// Return one bounded window of a conversation's messages plus the total
|
|
4819
|
-
// message count,
|
|
4820
|
-
// holding the whole
|
|
4864
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
4865
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
4866
|
+
// conversation itself.
|
|
4821
4867
|
//
|
|
4822
|
-
//
|
|
4868
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
4869
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
4870
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
4871
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4823
4872
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4824
4873
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4825
4874
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4826
4875
|
// same contract as getConversation.
|
|
4827
4876
|
//
|
|
4828
|
-
//
|
|
4829
|
-
//
|
|
4830
|
-
//
|
|
4831
|
-
//
|
|
4832
|
-
//
|
|
4833
|
-
//
|
|
4834
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4835
|
-
// which is identical by construction.
|
|
4877
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
4878
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
4879
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
4880
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
4881
|
+
// whole per call.
|
|
4836
4882
|
async getConversationPage(id, options) {
|
|
4883
|
+
const cid = canonicalPath(id);
|
|
4884
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
4885
|
+
if (!known) return null;
|
|
4886
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
4887
|
+
if (!this.persistent && isClaude) {
|
|
4888
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
4889
|
+
if (!index) return null;
|
|
4890
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
4891
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
4892
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
4893
|
+
let floor = null;
|
|
4894
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
4895
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
4896
|
+
return {
|
|
4897
|
+
messages: page.messages,
|
|
4898
|
+
total,
|
|
4899
|
+
fromIndex,
|
|
4900
|
+
meta: index.meta,
|
|
4901
|
+
turnDurations: index.turnDurations
|
|
4902
|
+
};
|
|
4903
|
+
}
|
|
4904
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
4905
|
+
if (!meta) return null;
|
|
4837
4906
|
if (this.persistent) {
|
|
4838
|
-
|
|
4907
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
4908
|
+
return page && { ...page, meta };
|
|
4909
|
+
}
|
|
4910
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
4911
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
4912
|
+
}
|
|
4913
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
4914
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
4915
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
4916
|
+
// file is never held, and the index LRU is capped.
|
|
4917
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
4918
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
4919
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
4920
|
+
pageIndexes = new LRUCache(32);
|
|
4921
|
+
async pageIndexFor(filePath, account) {
|
|
4922
|
+
let stat6;
|
|
4923
|
+
try {
|
|
4924
|
+
stat6 = statSync6(filePath);
|
|
4925
|
+
} catch {
|
|
4926
|
+
this.pageIndexes.delete(filePath);
|
|
4927
|
+
return null;
|
|
4839
4928
|
}
|
|
4840
|
-
const
|
|
4841
|
-
if (
|
|
4842
|
-
const
|
|
4843
|
-
const
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
const
|
|
4848
|
-
|
|
4929
|
+
const cached = this.pageIndexes.get(filePath);
|
|
4930
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
4931
|
+
const state = initialReducerState();
|
|
4932
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
4933
|
+
reduceLine(state, entry, this.lastTier);
|
|
4934
|
+
return false;
|
|
4935
|
+
});
|
|
4936
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
4937
|
+
if (!meta) {
|
|
4938
|
+
this.pageIndexes.delete(filePath);
|
|
4939
|
+
return null;
|
|
4940
|
+
}
|
|
4941
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
4942
|
+
const index = {
|
|
4943
|
+
size: stat6.size,
|
|
4944
|
+
mtimeMs: stat6.mtimeMs,
|
|
4945
|
+
checkpoints,
|
|
4946
|
+
meta,
|
|
4947
|
+
rawTotal: state.pageMessageCount,
|
|
4948
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
4949
|
+
};
|
|
4950
|
+
this.pageIndexes.set(filePath, index);
|
|
4951
|
+
return index;
|
|
4849
4952
|
}
|
|
4850
4953
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
4851
4954
|
// scan() or metadata index. This is the cold-start fast path: a single
|