@threadbase-sh/scanner 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -841,7 +841,8 @@ function initialReducerState() {
841
841
  snippetParts: [],
842
842
  previewLength: 0,
843
843
  snippetLength: 0,
844
- badJsonLines: 0
844
+ badJsonLines: 0,
845
+ turnDurations: []
845
846
  };
846
847
  }
847
848
  function reduceLine(state, entry, tier) {
@@ -855,6 +856,8 @@ function reduceLine(state, entry, tier) {
855
856
  const ts = entry.timestamp;
856
857
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
857
858
  }
859
+ const turnDuration = turnDurationOf(entry);
860
+ if (turnDuration) state.turnDurations?.push(turnDuration);
858
861
  const type = entry.type;
859
862
  if (type === "last-prompt") {
860
863
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -953,6 +956,16 @@ function getShortProjectName(fullPath) {
953
956
  }
954
957
 
955
958
  // src/parser.ts
959
+ function turnDurationOf(entry) {
960
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
961
+ return null;
962
+ }
963
+ return {
964
+ durationMs: entry.durationMs,
965
+ messageCount: entry.messageCount || 0,
966
+ uuid: entry.uuid
967
+ };
968
+ }
956
969
  async function parseMeta(filePath, account, tier, onEntry) {
957
970
  const log = getLogger();
958
971
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -1006,12 +1019,9 @@ async function parseConversation(filePath, account) {
1006
1019
  badJsonLines++;
1007
1020
  continue;
1008
1021
  }
1009
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
1010
- turnDurations.push({
1011
- durationMs: entry.durationMs,
1012
- messageCount: entry.messageCount || 0,
1013
- uuid: entry.uuid
1014
- });
1022
+ const turnDuration = turnDurationOf(entry);
1023
+ if (turnDuration) {
1024
+ turnDurations.push(turnDuration);
1015
1025
  continue;
1016
1026
  }
1017
1027
  const message = reduceConvLine(state, entry);
@@ -2698,7 +2708,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
2698
2708
  }
2699
2709
  return { offset, line };
2700
2710
  }
2701
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
2711
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
2702
2712
  const checkpoints = [];
2703
2713
  const state = from ? from.state : initialConvState();
2704
2714
  let index = from ? from.messageIndex : 0;
@@ -2718,7 +2728,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
2718
2728
  });
2719
2729
  }
2720
2730
  return false;
2721
- }
2731
+ },
2732
+ onEntry
2722
2733
  );
2723
2734
  return checkpoints;
2724
2735
  }
@@ -2739,6 +2750,41 @@ async function readPage(filePath, total, options, floor) {
2739
2750
  applyTeamInfo(window, state);
2740
2751
  return { messages: window, total, fromIndex };
2741
2752
  }
2753
+ function isInjectedTurn(message, isInjected) {
2754
+ return message.role === "user" && isInjected(message.text ?? "");
2755
+ }
2756
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
2757
+ let skip = 0;
2758
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
2759
+ if (!isInjectedTurn(message, isInjected)) return true;
2760
+ skip += 1;
2761
+ return false;
2762
+ });
2763
+ return skip;
2764
+ }
2765
+ function filteredWindow(rawTotal, skip, options) {
2766
+ const total = Math.max(0, rawTotal - skip);
2767
+ const beforeIndex = options.beforeIndex ?? total;
2768
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
2769
+ return {
2770
+ total,
2771
+ fromIndex,
2772
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
2773
+ };
2774
+ }
2775
+ function pageOfConversation(conversation, options) {
2776
+ const { messages } = conversation;
2777
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
2778
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
2779
+ const skip = firstKept === -1 ? messages.length : firstKept;
2780
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
2781
+ return {
2782
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
2783
+ total,
2784
+ fromIndex,
2785
+ turnDurations: conversation.turnDurations
2786
+ };
2787
+ }
2742
2788
 
2743
2789
  // src/persistent/conversation-stream.ts
2744
2790
  async function foldTail(filePath, resume) {
@@ -4268,6 +4314,7 @@ var PersistentEngine = class {
4268
4314
  const tier = resolveTier(tierName, customTiers);
4269
4315
  const filePath = canonicalPath(rawFilePath);
4270
4316
  const existing = this.files.getByPath(filePath);
4317
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4271
4318
  const classified = classify(filePath, existing);
4272
4319
  const stat6 = classified.stat;
4273
4320
  let change = classified.change;
@@ -4275,7 +4322,7 @@ var PersistentEngine = class {
4275
4322
  this.markDeleted(filePath);
4276
4323
  return { meta: null, change: "vanished" };
4277
4324
  }
4278
- if (change === "unchanged" && !force) {
4325
+ if (change === "unchanged" && !refold) {
4279
4326
  const meta2 = this.conversations.getBySourcePath(filePath);
4280
4327
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4281
4328
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4292,7 +4339,7 @@ var PersistentEngine = class {
4292
4339
  );
4293
4340
  return { meta: meta2, change };
4294
4341
  }
4295
- const resume = change === "appended" && !force && existing?.reducer_state;
4342
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4296
4343
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4297
4344
  const startOffset = resume ? existing.last_indexed_offset : 0;
4298
4345
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4453,6 +4500,10 @@ var PersistentEngine = class {
4453
4500
  // resolved to an indexed conversation. For conversations large enough to span
4454
4501
  // a checkpoint interval, checkpoints are built lazily on first access and
4455
4502
  // reused thereafter. Smaller conversations read from the start (cheap).
4503
+ //
4504
+ // The window is over the conversation with its leading injected-context run
4505
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4506
+ // index current (refreshFile) first; this reads what the index describes.
4456
4507
  async getPage(id, options) {
4457
4508
  const meta = this.conversations.getByIdOrSession(id);
4458
4509
  if (!meta) return null;
@@ -4460,24 +4511,28 @@ var PersistentEngine = class {
4460
4511
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4461
4512
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4462
4513
  if (!conversation) return null;
4463
- const { messages } = conversation;
4464
- const total2 = messages.length;
4465
- const beforeIndex2 = options.beforeIndex ?? total2;
4466
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4467
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4468
- }
4469
- const total = this.conversations.pageMessageCount(filePath);
4470
- await this.ensureCheckpoints(filePath, total);
4471
- const beforeIndex = options.beforeIndex ?? total;
4472
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4473
- let floor = this.checkpoints.floor(filePath, fromIndex);
4514
+ return pageOfConversation(conversation, options);
4515
+ }
4516
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4517
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4518
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4519
+ await this.ensureCheckpoints(filePath, rawTotal);
4520
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4474
4521
  if (floor) {
4475
4522
  try {
4476
4523
  if (floor.byteOffset > (0, import_fs13.statSync)(filePath).size) floor = null;
4477
4524
  } catch {
4478
4525
  }
4479
4526
  }
4480
- return readPage(filePath, total, options, floor);
4527
+ const page = await readPage(filePath, rawTotal, raw, floor);
4528
+ const state = this.files.getByPath(filePath)?.reducer_state;
4529
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4530
+ return {
4531
+ messages: page.messages,
4532
+ total,
4533
+ fromIndex,
4534
+ turnDurations: durations?.length ? durations : void 0
4535
+ };
4481
4536
  }
4482
4537
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4483
4538
  // file → full build; a file that grew → extend from the last persisted
@@ -4966,36 +5021,94 @@ var ConversationScanner = class {
4966
5021
  }
4967
5022
  }
4968
5023
  // Return one bounded window of a conversation's messages plus the total
4969
- // message count, so a caller can serve the last page and scroll back without
4970
- // holding the whole conversation itself.
5024
+ // message count, the conversation's current meta and its turn durations, so a
5025
+ // caller can serve the last page and scroll back without holding the whole
5026
+ // conversation itself.
4971
5027
  //
4972
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
5028
+ // Messages are those of the full read with the LEADING run of injected user
5029
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
5030
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
5031
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4973
5032
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4974
5033
  // the window's start index, so the caller can derive `has_more_older`
4975
5034
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4976
5035
  // same contract as getConversation.
4977
5036
  //
4978
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4979
- // checkpoint and parses only the requested window (checkpoints are built
4980
- // lazily for large conversations). Windowed message indices are proven
4981
- // identical to parseConversation().messages.slice(...) by the paged-reader
4982
- // equivalence test.
4983
- //
4984
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4985
- // which is identical by construction.
5037
+ // Every call reads the file as it is on disk now, never a cached parse:
5038
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
5039
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
5040
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
5041
+ // whole per call.
4986
5042
  async getConversationPage(id, options) {
5043
+ const cid = canonicalPath(id);
5044
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
5045
+ if (!known) return null;
5046
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
5047
+ if (!this.persistent && isClaude) {
5048
+ const index = await this.pageIndexFor(known.filePath, known.account);
5049
+ if (!index) return null;
5050
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
5051
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
5052
+ const rawFrom = raw.beforeIndex - raw.limit;
5053
+ let floor = null;
5054
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
5055
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
5056
+ return {
5057
+ messages: page.messages,
5058
+ total,
5059
+ fromIndex,
5060
+ meta: index.meta,
5061
+ turnDurations: index.turnDurations
5062
+ };
5063
+ }
5064
+ const meta = await this.refreshFile(known.filePath, known.account);
5065
+ if (!meta) return null;
4987
5066
  if (this.persistent) {
4988
- return this.engine().getPage(canonicalPath(id), options);
5067
+ const page = await this.engine().getPage(meta.filePath, options);
5068
+ return page && { ...page, meta };
5069
+ }
5070
+ const conversation = await this.getConversation(meta.filePath);
5071
+ return conversation && { ...pageOfConversation(conversation, options), meta };
5072
+ }
5073
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
5074
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
5075
+ // reused until the file's size or mtime changes. Memory stays bounded: the
5076
+ // file is never held, and the index LRU is capped.
5077
+ // ponytail: any change rebuilds from byte 0; extending from the last
5078
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
5079
+ // upgrade path if live-appended transcripts make this hot.
5080
+ pageIndexes = new LRUCache(32);
5081
+ async pageIndexFor(filePath, account) {
5082
+ let stat6;
5083
+ try {
5084
+ stat6 = (0, import_fs14.statSync)(filePath);
5085
+ } catch {
5086
+ this.pageIndexes.delete(filePath);
5087
+ return null;
4989
5088
  }
4990
- const conversation = await this.getConversation(id);
4991
- if (!conversation) return null;
4992
- const { messages } = conversation;
4993
- const total = messages.length;
4994
- const { limit } = options;
4995
- const beforeIndex = options.beforeIndex ?? total;
4996
- const fromIndex = Math.max(0, beforeIndex - limit);
4997
- const window = messages.slice(fromIndex, beforeIndex);
4998
- return { messages: window, total, fromIndex };
5089
+ const cached = this.pageIndexes.get(filePath);
5090
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
5091
+ const state = initialReducerState();
5092
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
5093
+ reduceLine(state, entry, this.lastTier);
5094
+ return false;
5095
+ });
5096
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
5097
+ if (!meta) {
5098
+ this.pageIndexes.delete(filePath);
5099
+ return null;
5100
+ }
5101
+ meta.gitBranch = readGitBranch(meta.projectPath);
5102
+ const index = {
5103
+ size: stat6.size,
5104
+ mtimeMs: stat6.mtimeMs,
5105
+ checkpoints,
5106
+ meta,
5107
+ rawTotal: state.pageMessageCount,
5108
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
5109
+ };
5110
+ this.pageIndexes.set(filePath, index);
5111
+ return index;
4999
5112
  }
5000
5113
  // Parse one JSONL file directly and slice a page window — without any prior
5001
5114
  // scan() or metadata index. This is the cold-start fast path: a single