@threadbase-sh/scanner 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -5,7 +5,7 @@ import { Command } from "commander";
5
5
  import pino2 from "pino";
6
6
 
7
7
  // package.json
8
- var version = "0.23.0";
8
+ var version = "0.24.0";
9
9
 
10
10
  // src/logger.ts
11
11
  import pino from "pino";
@@ -977,7 +977,8 @@ function initialReducerState() {
977
977
  snippetParts: [],
978
978
  previewLength: 0,
979
979
  snippetLength: 0,
980
- badJsonLines: 0
980
+ badJsonLines: 0,
981
+ turnDurations: []
981
982
  };
982
983
  }
983
984
  function reduceLine(state, entry, tier) {
@@ -991,6 +992,8 @@ function reduceLine(state, entry, tier) {
991
992
  const ts = entry.timestamp;
992
993
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
993
994
  }
995
+ const turnDuration = turnDurationOf(entry);
996
+ if (turnDuration) state.turnDurations?.push(turnDuration);
994
997
  const type = entry.type;
995
998
  if (type === "last-prompt") {
996
999
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -1089,6 +1092,16 @@ function getShortProjectName(fullPath) {
1089
1092
  }
1090
1093
 
1091
1094
  // src/parser.ts
1095
+ function turnDurationOf(entry) {
1096
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
1097
+ return null;
1098
+ }
1099
+ return {
1100
+ durationMs: entry.durationMs,
1101
+ messageCount: entry.messageCount || 0,
1102
+ uuid: entry.uuid
1103
+ };
1104
+ }
1092
1105
  async function parseMeta(filePath, account, tier, onEntry) {
1093
1106
  const log = getLogger();
1094
1107
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -1142,12 +1155,9 @@ async function parseConversation(filePath, account) {
1142
1155
  badJsonLines++;
1143
1156
  continue;
1144
1157
  }
1145
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
1146
- turnDurations.push({
1147
- durationMs: entry.durationMs,
1148
- messageCount: entry.messageCount || 0,
1149
- uuid: entry.uuid
1150
- });
1158
+ const turnDuration = turnDurationOf(entry);
1159
+ if (turnDuration) {
1160
+ turnDurations.push(turnDuration);
1151
1161
  continue;
1152
1162
  }
1153
1163
  const message = reduceConvLine(state, entry);
@@ -1364,7 +1374,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
1364
1374
  }
1365
1375
  return { offset, line };
1366
1376
  }
1367
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
1377
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
1368
1378
  const checkpoints = [];
1369
1379
  const state = from ? from.state : initialConvState();
1370
1380
  let index = from ? from.messageIndex : 0;
@@ -1384,7 +1394,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
1384
1394
  });
1385
1395
  }
1386
1396
  return false;
1387
- }
1397
+ },
1398
+ onEntry
1388
1399
  );
1389
1400
  return checkpoints;
1390
1401
  }
@@ -1405,6 +1416,41 @@ async function readPage(filePath, total, options, floor) {
1405
1416
  applyTeamInfo(window, state);
1406
1417
  return { messages: window, total, fromIndex };
1407
1418
  }
1419
+ function isInjectedTurn(message, isInjected) {
1420
+ return message.role === "user" && isInjected(message.text ?? "");
1421
+ }
1422
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
1423
+ let skip = 0;
1424
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
1425
+ if (!isInjectedTurn(message, isInjected)) return true;
1426
+ skip += 1;
1427
+ return false;
1428
+ });
1429
+ return skip;
1430
+ }
1431
+ function filteredWindow(rawTotal, skip, options) {
1432
+ const total = Math.max(0, rawTotal - skip);
1433
+ const beforeIndex = options.beforeIndex ?? total;
1434
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
1435
+ return {
1436
+ total,
1437
+ fromIndex,
1438
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
1439
+ };
1440
+ }
1441
+ function pageOfConversation(conversation, options) {
1442
+ const { messages } = conversation;
1443
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
1444
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
1445
+ const skip = firstKept === -1 ? messages.length : firstKept;
1446
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
1447
+ return {
1448
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
1449
+ total,
1450
+ fromIndex,
1451
+ turnDurations: conversation.turnDurations
1452
+ };
1453
+ }
1408
1454
 
1409
1455
  // src/persistent/conversation-stream.ts
1410
1456
  async function foldTail(filePath, resume) {
@@ -4082,6 +4128,7 @@ var PersistentEngine = class {
4082
4128
  const tier = resolveTier(tierName, customTiers);
4083
4129
  const filePath = canonicalPath(rawFilePath);
4084
4130
  const existing = this.files.getByPath(filePath);
4131
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4085
4132
  const classified = classify(filePath, existing);
4086
4133
  const stat6 = classified.stat;
4087
4134
  let change = classified.change;
@@ -4089,7 +4136,7 @@ var PersistentEngine = class {
4089
4136
  this.markDeleted(filePath);
4090
4137
  return { meta: null, change: "vanished" };
4091
4138
  }
4092
- if (change === "unchanged" && !force) {
4139
+ if (change === "unchanged" && !refold) {
4093
4140
  const meta2 = this.conversations.getBySourcePath(filePath);
4094
4141
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4095
4142
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4106,7 +4153,7 @@ var PersistentEngine = class {
4106
4153
  );
4107
4154
  return { meta: meta2, change };
4108
4155
  }
4109
- const resume = change === "appended" && !force && existing?.reducer_state;
4156
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4110
4157
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4111
4158
  const startOffset = resume ? existing.last_indexed_offset : 0;
4112
4159
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4267,6 +4314,10 @@ var PersistentEngine = class {
4267
4314
  // resolved to an indexed conversation. For conversations large enough to span
4268
4315
  // a checkpoint interval, checkpoints are built lazily on first access and
4269
4316
  // reused thereafter. Smaller conversations read from the start (cheap).
4317
+ //
4318
+ // The window is over the conversation with its leading injected-context run
4319
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4320
+ // index current (refreshFile) first; this reads what the index describes.
4270
4321
  async getPage(id, options) {
4271
4322
  const meta = this.conversations.getByIdOrSession(id);
4272
4323
  if (!meta) return null;
@@ -4274,24 +4325,28 @@ var PersistentEngine = class {
4274
4325
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4275
4326
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4276
4327
  if (!conversation) return null;
4277
- const { messages } = conversation;
4278
- const total2 = messages.length;
4279
- const beforeIndex2 = options.beforeIndex ?? total2;
4280
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4281
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4282
- }
4283
- const total = this.conversations.pageMessageCount(filePath);
4284
- await this.ensureCheckpoints(filePath, total);
4285
- const beforeIndex = options.beforeIndex ?? total;
4286
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4287
- let floor = this.checkpoints.floor(filePath, fromIndex);
4328
+ return pageOfConversation(conversation, options);
4329
+ }
4330
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4331
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4332
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4333
+ await this.ensureCheckpoints(filePath, rawTotal);
4334
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4288
4335
  if (floor) {
4289
4336
  try {
4290
4337
  if (floor.byteOffset > statSync5(filePath).size) floor = null;
4291
4338
  } catch {
4292
4339
  }
4293
4340
  }
4294
- return readPage(filePath, total, options, floor);
4341
+ const page = await readPage(filePath, rawTotal, raw, floor);
4342
+ const state = this.files.getByPath(filePath)?.reducer_state;
4343
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4344
+ return {
4345
+ messages: page.messages,
4346
+ total,
4347
+ fromIndex,
4348
+ turnDurations: durations?.length ? durations : void 0
4349
+ };
4295
4350
  }
4296
4351
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4297
4352
  // file → full build; a file that grew → extend from the last persisted
@@ -4816,36 +4871,94 @@ var ConversationScanner = class {
4816
4871
  }
4817
4872
  }
4818
4873
  // Return one bounded window of a conversation's messages plus the total
4819
- // message count, so a caller can serve the last page and scroll back without
4820
- // holding the whole conversation itself.
4874
+ // message count, the conversation's current meta and its turn durations, so a
4875
+ // caller can serve the last page and scroll back without holding the whole
4876
+ // conversation itself.
4821
4877
  //
4822
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4878
+ // Messages are those of the full read with the LEADING run of injected user
4879
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
4880
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
4881
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4823
4882
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4824
4883
  // the window's start index, so the caller can derive `has_more_older`
4825
4884
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4826
4885
  // same contract as getConversation.
4827
4886
  //
4828
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4829
- // checkpoint and parses only the requested window (checkpoints are built
4830
- // lazily for large conversations). Windowed message indices are proven
4831
- // identical to parseConversation().messages.slice(...) by the paged-reader
4832
- // equivalence test.
4833
- //
4834
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4835
- // which is identical by construction.
4887
+ // Every call reads the file as it is on disk now, never a cached parse:
4888
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
4889
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
4890
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
4891
+ // whole per call.
4836
4892
  async getConversationPage(id, options) {
4893
+ const cid = canonicalPath(id);
4894
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
4895
+ if (!known) return null;
4896
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
4897
+ if (!this.persistent && isClaude) {
4898
+ const index = await this.pageIndexFor(known.filePath, known.account);
4899
+ if (!index) return null;
4900
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
4901
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
4902
+ const rawFrom = raw.beforeIndex - raw.limit;
4903
+ let floor = null;
4904
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
4905
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
4906
+ return {
4907
+ messages: page.messages,
4908
+ total,
4909
+ fromIndex,
4910
+ meta: index.meta,
4911
+ turnDurations: index.turnDurations
4912
+ };
4913
+ }
4914
+ const meta = await this.refreshFile(known.filePath, known.account);
4915
+ if (!meta) return null;
4837
4916
  if (this.persistent) {
4838
- return this.engine().getPage(canonicalPath(id), options);
4917
+ const page = await this.engine().getPage(meta.filePath, options);
4918
+ return page && { ...page, meta };
4919
+ }
4920
+ const conversation = await this.getConversation(meta.filePath);
4921
+ return conversation && { ...pageOfConversation(conversation, options), meta };
4922
+ }
4923
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
4924
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
4925
+ // reused until the file's size or mtime changes. Memory stays bounded: the
4926
+ // file is never held, and the index LRU is capped.
4927
+ // ponytail: any change rebuilds from byte 0; extending from the last
4928
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
4929
+ // upgrade path if live-appended transcripts make this hot.
4930
+ pageIndexes = new LRUCache(32);
4931
+ async pageIndexFor(filePath, account) {
4932
+ let stat6;
4933
+ try {
4934
+ stat6 = statSync6(filePath);
4935
+ } catch {
4936
+ this.pageIndexes.delete(filePath);
4937
+ return null;
4839
4938
  }
4840
- const conversation = await this.getConversation(id);
4841
- if (!conversation) return null;
4842
- const { messages } = conversation;
4843
- const total = messages.length;
4844
- const { limit } = options;
4845
- const beforeIndex = options.beforeIndex ?? total;
4846
- const fromIndex = Math.max(0, beforeIndex - limit);
4847
- const window = messages.slice(fromIndex, beforeIndex);
4848
- return { messages: window, total, fromIndex };
4939
+ const cached = this.pageIndexes.get(filePath);
4940
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
4941
+ const state = initialReducerState();
4942
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
4943
+ reduceLine(state, entry, this.lastTier);
4944
+ return false;
4945
+ });
4946
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
4947
+ if (!meta) {
4948
+ this.pageIndexes.delete(filePath);
4949
+ return null;
4950
+ }
4951
+ meta.gitBranch = readGitBranch(meta.projectPath);
4952
+ const index = {
4953
+ size: stat6.size,
4954
+ mtimeMs: stat6.mtimeMs,
4955
+ checkpoints,
4956
+ meta,
4957
+ rawTotal: state.pageMessageCount,
4958
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
4959
+ };
4960
+ this.pageIndexes.set(filePath, index);
4961
+ return index;
4849
4962
  }
4850
4963
  // Parse one JSONL file directly and slice a page window — without any prior
4851
4964
  // scan() or metadata index. This is the cold-start fast path: a single