@threadbase-sh/scanner 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +158 -45
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +157 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +157 -44
- package/dist/index.js.map +1 -1
- package/dist/parser.d.ts +2 -1
- package/dist/parser.d.ts.map +1 -1
- package/dist/persistent/index-engine.d.ts +4 -2
- package/dist/persistent/index-engine.d.ts.map +1 -1
- package/dist/persistent/metadata-reducer.d.ts +2 -1
- package/dist/persistent/metadata-reducer.d.ts.map +1 -1
- package/dist/persistent/paged-reader.d.ts +14 -2
- package/dist/persistent/paged-reader.d.ts.map +1 -1
- package/dist/scanner.d.ts +4 -2
- package/dist/scanner.d.ts.map +1 -1
- package/dist/types.d.ts +5 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -5,7 +5,7 @@ import { Command } from "commander";
|
|
|
5
5
|
import pino2 from "pino";
|
|
6
6
|
|
|
7
7
|
// package.json
|
|
8
|
-
var version = "0.
|
|
8
|
+
var version = "0.24.0";
|
|
9
9
|
|
|
10
10
|
// src/logger.ts
|
|
11
11
|
import pino from "pino";
|
|
@@ -977,7 +977,8 @@ function initialReducerState() {
|
|
|
977
977
|
snippetParts: [],
|
|
978
978
|
previewLength: 0,
|
|
979
979
|
snippetLength: 0,
|
|
980
|
-
badJsonLines: 0
|
|
980
|
+
badJsonLines: 0,
|
|
981
|
+
turnDurations: []
|
|
981
982
|
};
|
|
982
983
|
}
|
|
983
984
|
function reduceLine(state, entry, tier) {
|
|
@@ -991,6 +992,8 @@ function reduceLine(state, entry, tier) {
|
|
|
991
992
|
const ts = entry.timestamp;
|
|
992
993
|
if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
|
|
993
994
|
}
|
|
995
|
+
const turnDuration = turnDurationOf(entry);
|
|
996
|
+
if (turnDuration) state.turnDurations?.push(turnDuration);
|
|
994
997
|
const type = entry.type;
|
|
995
998
|
if (type === "last-prompt") {
|
|
996
999
|
if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
|
|
@@ -1089,6 +1092,16 @@ function getShortProjectName(fullPath) {
|
|
|
1089
1092
|
}
|
|
1090
1093
|
|
|
1091
1094
|
// src/parser.ts
|
|
1095
|
+
function turnDurationOf(entry) {
|
|
1096
|
+
if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
|
|
1097
|
+
return null;
|
|
1098
|
+
}
|
|
1099
|
+
return {
|
|
1100
|
+
durationMs: entry.durationMs,
|
|
1101
|
+
messageCount: entry.messageCount || 0,
|
|
1102
|
+
uuid: entry.uuid
|
|
1103
|
+
};
|
|
1104
|
+
}
|
|
1092
1105
|
async function parseMeta(filePath, account, tier, onEntry) {
|
|
1093
1106
|
const log = getLogger();
|
|
1094
1107
|
log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
|
|
@@ -1142,12 +1155,9 @@ async function parseConversation(filePath, account) {
|
|
|
1142
1155
|
badJsonLines++;
|
|
1143
1156
|
continue;
|
|
1144
1157
|
}
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
messageCount: entry.messageCount || 0,
|
|
1149
|
-
uuid: entry.uuid
|
|
1150
|
-
});
|
|
1158
|
+
const turnDuration = turnDurationOf(entry);
|
|
1159
|
+
if (turnDuration) {
|
|
1160
|
+
turnDurations.push(turnDuration);
|
|
1151
1161
|
continue;
|
|
1152
1162
|
}
|
|
1153
1163
|
const message = reduceConvLine(state, entry);
|
|
@@ -1364,7 +1374,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
|
|
|
1364
1374
|
}
|
|
1365
1375
|
return { offset, line };
|
|
1366
1376
|
}
|
|
1367
|
-
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
|
|
1377
|
+
async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
|
|
1368
1378
|
const checkpoints = [];
|
|
1369
1379
|
const state = from ? from.state : initialConvState();
|
|
1370
1380
|
let index = from ? from.messageIndex : 0;
|
|
@@ -1384,7 +1394,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
|
|
|
1384
1394
|
});
|
|
1385
1395
|
}
|
|
1386
1396
|
return false;
|
|
1387
|
-
}
|
|
1397
|
+
},
|
|
1398
|
+
onEntry
|
|
1388
1399
|
);
|
|
1389
1400
|
return checkpoints;
|
|
1390
1401
|
}
|
|
@@ -1405,6 +1416,41 @@ async function readPage(filePath, total, options, floor) {
|
|
|
1405
1416
|
applyTeamInfo(window, state);
|
|
1406
1417
|
return { messages: window, total, fromIndex };
|
|
1407
1418
|
}
|
|
1419
|
+
function isInjectedTurn(message, isInjected) {
|
|
1420
|
+
return message.role === "user" && isInjected(message.text ?? "");
|
|
1421
|
+
}
|
|
1422
|
+
async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
|
|
1423
|
+
let skip = 0;
|
|
1424
|
+
await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
|
|
1425
|
+
if (!isInjectedTurn(message, isInjected)) return true;
|
|
1426
|
+
skip += 1;
|
|
1427
|
+
return false;
|
|
1428
|
+
});
|
|
1429
|
+
return skip;
|
|
1430
|
+
}
|
|
1431
|
+
function filteredWindow(rawTotal, skip, options) {
|
|
1432
|
+
const total = Math.max(0, rawTotal - skip);
|
|
1433
|
+
const beforeIndex = options.beforeIndex ?? total;
|
|
1434
|
+
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
1435
|
+
return {
|
|
1436
|
+
total,
|
|
1437
|
+
fromIndex,
|
|
1438
|
+
raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
|
|
1439
|
+
};
|
|
1440
|
+
}
|
|
1441
|
+
function pageOfConversation(conversation, options) {
|
|
1442
|
+
const { messages } = conversation;
|
|
1443
|
+
const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
|
|
1444
|
+
const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
|
|
1445
|
+
const skip = firstKept === -1 ? messages.length : firstKept;
|
|
1446
|
+
const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
|
|
1447
|
+
return {
|
|
1448
|
+
messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
|
|
1449
|
+
total,
|
|
1450
|
+
fromIndex,
|
|
1451
|
+
turnDurations: conversation.turnDurations
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1408
1454
|
|
|
1409
1455
|
// src/persistent/conversation-stream.ts
|
|
1410
1456
|
async function foldTail(filePath, resume) {
|
|
@@ -4082,6 +4128,7 @@ var PersistentEngine = class {
|
|
|
4082
4128
|
const tier = resolveTier(tierName, customTiers);
|
|
4083
4129
|
const filePath = canonicalPath(rawFilePath);
|
|
4084
4130
|
const existing = this.files.getByPath(filePath);
|
|
4131
|
+
const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
|
|
4085
4132
|
const classified = classify(filePath, existing);
|
|
4086
4133
|
const stat6 = classified.stat;
|
|
4087
4134
|
let change = classified.change;
|
|
@@ -4089,7 +4136,7 @@ var PersistentEngine = class {
|
|
|
4089
4136
|
this.markDeleted(filePath);
|
|
4090
4137
|
return { meta: null, change: "vanished" };
|
|
4091
4138
|
}
|
|
4092
|
-
if (change === "unchanged" && !
|
|
4139
|
+
if (change === "unchanged" && !refold) {
|
|
4093
4140
|
const meta2 = this.conversations.getBySourcePath(filePath);
|
|
4094
4141
|
const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
|
|
4095
4142
|
if (!sideStoresMoved) return { meta: meta2, change };
|
|
@@ -4106,7 +4153,7 @@ var PersistentEngine = class {
|
|
|
4106
4153
|
);
|
|
4107
4154
|
return { meta: meta2, change };
|
|
4108
4155
|
}
|
|
4109
|
-
const resume = change === "appended" && !
|
|
4156
|
+
const resume = change === "appended" && !refold && existing?.reducer_state;
|
|
4110
4157
|
const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
|
|
4111
4158
|
const startOffset = resume ? existing.last_indexed_offset : 0;
|
|
4112
4159
|
const startLine = resume ? existing.last_indexed_line : 0;
|
|
@@ -4267,6 +4314,10 @@ var PersistentEngine = class {
|
|
|
4267
4314
|
// resolved to an indexed conversation. For conversations large enough to span
|
|
4268
4315
|
// a checkpoint interval, checkpoints are built lazily on first access and
|
|
4269
4316
|
// reused thereafter. Smaller conversations read from the start (cheap).
|
|
4317
|
+
//
|
|
4318
|
+
// The window is over the conversation with its leading injected-context run
|
|
4319
|
+
// dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
|
|
4320
|
+
// index current (refreshFile) first; this reads what the index describes.
|
|
4270
4321
|
async getPage(id, options) {
|
|
4271
4322
|
const meta = this.conversations.getByIdOrSession(id);
|
|
4272
4323
|
if (!meta) return null;
|
|
@@ -4274,24 +4325,28 @@ var PersistentEngine = class {
|
|
|
4274
4325
|
if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
|
|
4275
4326
|
const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
|
|
4276
4327
|
if (!conversation) return null;
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4281
|
-
|
|
4282
|
-
|
|
4283
|
-
|
|
4284
|
-
await this.ensureCheckpoints(filePath, total);
|
|
4285
|
-
const beforeIndex = options.beforeIndex ?? total;
|
|
4286
|
-
const fromIndex = Math.max(0, beforeIndex - options.limit);
|
|
4287
|
-
let floor = this.checkpoints.floor(filePath, fromIndex);
|
|
4328
|
+
return pageOfConversation(conversation, options);
|
|
4329
|
+
}
|
|
4330
|
+
const rawTotal = this.conversations.pageMessageCount(filePath);
|
|
4331
|
+
const skip = await countLeadingInjected(filePath, options.isInjectedContext);
|
|
4332
|
+
const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
|
|
4333
|
+
await this.ensureCheckpoints(filePath, rawTotal);
|
|
4334
|
+
let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
|
|
4288
4335
|
if (floor) {
|
|
4289
4336
|
try {
|
|
4290
4337
|
if (floor.byteOffset > statSync5(filePath).size) floor = null;
|
|
4291
4338
|
} catch {
|
|
4292
4339
|
}
|
|
4293
4340
|
}
|
|
4294
|
-
|
|
4341
|
+
const page = await readPage(filePath, rawTotal, raw, floor);
|
|
4342
|
+
const state = this.files.getByPath(filePath)?.reducer_state;
|
|
4343
|
+
const durations = state ? JSON.parse(state).turnDurations : void 0;
|
|
4344
|
+
return {
|
|
4345
|
+
messages: page.messages,
|
|
4346
|
+
total,
|
|
4347
|
+
fromIndex,
|
|
4348
|
+
turnDurations: durations?.length ? durations : void 0
|
|
4349
|
+
};
|
|
4295
4350
|
}
|
|
4296
4351
|
// Build or extend the checkpoint chain so it covers `total` messages. Cold
|
|
4297
4352
|
// file → full build; a file that grew → extend from the last persisted
|
|
@@ -4816,36 +4871,94 @@ var ConversationScanner = class {
|
|
|
4816
4871
|
}
|
|
4817
4872
|
}
|
|
4818
4873
|
// Return one bounded window of a conversation's messages plus the total
|
|
4819
|
-
// message count,
|
|
4820
|
-
// holding the whole
|
|
4874
|
+
// message count, the conversation's current meta and its turn durations, so a
|
|
4875
|
+
// caller can serve the last page and scroll back without holding the whole
|
|
4876
|
+
// conversation itself.
|
|
4821
4877
|
//
|
|
4822
|
-
//
|
|
4878
|
+
// Messages are those of the full read with the LEADING run of injected user
|
|
4879
|
+
// turns dropped (options.isInjectedContext, default isCodexInjectedContext);
|
|
4880
|
+
// `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
|
|
4881
|
+
// window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
|
|
4823
4882
|
// order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
|
|
4824
4883
|
// the window's start index, so the caller can derive `has_more_older`
|
|
4825
4884
|
// (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
|
|
4826
4885
|
// same contract as getConversation.
|
|
4827
4886
|
//
|
|
4828
|
-
//
|
|
4829
|
-
//
|
|
4830
|
-
//
|
|
4831
|
-
//
|
|
4832
|
-
//
|
|
4833
|
-
//
|
|
4834
|
-
// Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
|
|
4835
|
-
// which is identical by construction.
|
|
4887
|
+
// Every call reads the file as it is on disk now, never a cached parse:
|
|
4888
|
+
// persistent mode refreshes the index first (O(appended bytes)) and the engine
|
|
4889
|
+
// seeks from the nearest checkpoint. In-memory claude-code files keep a
|
|
4890
|
+
// stat-validated checkpoint index (pageIndexFor). Other providers are parsed
|
|
4891
|
+
// whole per call.
|
|
4836
4892
|
async getConversationPage(id, options) {
|
|
4893
|
+
const cid = canonicalPath(id);
|
|
4894
|
+
const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
|
|
4895
|
+
if (!known) return null;
|
|
4896
|
+
const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
|
|
4897
|
+
if (!this.persistent && isClaude) {
|
|
4898
|
+
const index = await this.pageIndexFor(known.filePath, known.account);
|
|
4899
|
+
if (!index) return null;
|
|
4900
|
+
const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
|
|
4901
|
+
const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
|
|
4902
|
+
const rawFrom = raw.beforeIndex - raw.limit;
|
|
4903
|
+
let floor = null;
|
|
4904
|
+
for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
|
|
4905
|
+
const page = await readPage(known.filePath, index.rawTotal, raw, floor);
|
|
4906
|
+
return {
|
|
4907
|
+
messages: page.messages,
|
|
4908
|
+
total,
|
|
4909
|
+
fromIndex,
|
|
4910
|
+
meta: index.meta,
|
|
4911
|
+
turnDurations: index.turnDurations
|
|
4912
|
+
};
|
|
4913
|
+
}
|
|
4914
|
+
const meta = await this.refreshFile(known.filePath, known.account);
|
|
4915
|
+
if (!meta) return null;
|
|
4837
4916
|
if (this.persistent) {
|
|
4838
|
-
|
|
4917
|
+
const page = await this.engine().getPage(meta.filePath, options);
|
|
4918
|
+
return page && { ...page, meta };
|
|
4919
|
+
}
|
|
4920
|
+
const conversation = await this.getConversation(meta.filePath);
|
|
4921
|
+
return conversation && { ...pageOfConversation(conversation, options), meta };
|
|
4922
|
+
}
|
|
4923
|
+
// In-memory claude-code paging index: one streamed pass builds the checkpoint
|
|
4924
|
+
// chain and folds the metadata (meta, page total, turn durations) alongside,
|
|
4925
|
+
// reused until the file's size or mtime changes. Memory stays bounded: the
|
|
4926
|
+
// file is never held, and the index LRU is capped.
|
|
4927
|
+
// ponytail: any change rebuilds from byte 0; extending from the last
|
|
4928
|
+
// checkpoint plus a persisted ReducerState (as the engine does) is the
|
|
4929
|
+
// upgrade path if live-appended transcripts make this hot.
|
|
4930
|
+
pageIndexes = new LRUCache(32);
|
|
4931
|
+
async pageIndexFor(filePath, account) {
|
|
4932
|
+
let stat6;
|
|
4933
|
+
try {
|
|
4934
|
+
stat6 = statSync6(filePath);
|
|
4935
|
+
} catch {
|
|
4936
|
+
this.pageIndexes.delete(filePath);
|
|
4937
|
+
return null;
|
|
4839
4938
|
}
|
|
4840
|
-
const
|
|
4841
|
-
if (
|
|
4842
|
-
const
|
|
4843
|
-
const
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
const
|
|
4848
|
-
|
|
4939
|
+
const cached = this.pageIndexes.get(filePath);
|
|
4940
|
+
if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
|
|
4941
|
+
const state = initialReducerState();
|
|
4942
|
+
const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
|
|
4943
|
+
reduceLine(state, entry, this.lastTier);
|
|
4944
|
+
return false;
|
|
4945
|
+
});
|
|
4946
|
+
const meta = finalizeMeta(state, filePath, account, this.lastTier);
|
|
4947
|
+
if (!meta) {
|
|
4948
|
+
this.pageIndexes.delete(filePath);
|
|
4949
|
+
return null;
|
|
4950
|
+
}
|
|
4951
|
+
meta.gitBranch = readGitBranch(meta.projectPath);
|
|
4952
|
+
const index = {
|
|
4953
|
+
size: stat6.size,
|
|
4954
|
+
mtimeMs: stat6.mtimeMs,
|
|
4955
|
+
checkpoints,
|
|
4956
|
+
meta,
|
|
4957
|
+
rawTotal: state.pageMessageCount,
|
|
4958
|
+
turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
|
|
4959
|
+
};
|
|
4960
|
+
this.pageIndexes.set(filePath, index);
|
|
4961
|
+
return index;
|
|
4849
4962
|
}
|
|
4850
4963
|
// Parse one JSONL file directly and slice a page window — without any prior
|
|
4851
4964
|
// scan() or metadata index. This is the cold-start fast path: a single
|