@threadbase-sh/scanner 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -751,7 +751,8 @@ function initialReducerState() {
751
751
  snippetParts: [],
752
752
  previewLength: 0,
753
753
  snippetLength: 0,
754
- badJsonLines: 0
754
+ badJsonLines: 0,
755
+ turnDurations: []
755
756
  };
756
757
  }
757
758
  function reduceLine(state, entry, tier) {
@@ -765,6 +766,8 @@ function reduceLine(state, entry, tier) {
765
766
  const ts = entry.timestamp;
766
767
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
767
768
  }
769
+ const turnDuration = turnDurationOf(entry);
770
+ if (turnDuration) state.turnDurations?.push(turnDuration);
768
771
  const type = entry.type;
769
772
  if (type === "last-prompt") {
770
773
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -863,6 +866,16 @@ function getShortProjectName(fullPath) {
863
866
  }
864
867
 
865
868
  // src/parser.ts
869
+ function turnDurationOf(entry) {
870
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
871
+ return null;
872
+ }
873
+ return {
874
+ durationMs: entry.durationMs,
875
+ messageCount: entry.messageCount || 0,
876
+ uuid: entry.uuid
877
+ };
878
+ }
866
879
  async function parseMeta(filePath, account, tier, onEntry) {
867
880
  const log = getLogger();
868
881
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -916,12 +929,9 @@ async function parseConversation(filePath, account) {
916
929
  badJsonLines++;
917
930
  continue;
918
931
  }
919
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
920
- turnDurations.push({
921
- durationMs: entry.durationMs,
922
- messageCount: entry.messageCount || 0,
923
- uuid: entry.uuid
924
- });
932
+ const turnDuration = turnDurationOf(entry);
933
+ if (turnDuration) {
934
+ turnDurations.push(turnDuration);
925
935
  continue;
926
936
  }
927
937
  const message = reduceConvLine(state, entry);
@@ -2608,7 +2618,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
2608
2618
  }
2609
2619
  return { offset, line };
2610
2620
  }
2611
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
2621
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
2612
2622
  const checkpoints = [];
2613
2623
  const state = from ? from.state : initialConvState();
2614
2624
  let index = from ? from.messageIndex : 0;
@@ -2628,7 +2638,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
2628
2638
  });
2629
2639
  }
2630
2640
  return false;
2631
- }
2641
+ },
2642
+ onEntry
2632
2643
  );
2633
2644
  return checkpoints;
2634
2645
  }
@@ -2649,6 +2660,41 @@ async function readPage(filePath, total, options, floor) {
2649
2660
  applyTeamInfo(window, state);
2650
2661
  return { messages: window, total, fromIndex };
2651
2662
  }
2663
+ function isInjectedTurn(message, isInjected) {
2664
+ return message.role === "user" && isInjected(message.text ?? "");
2665
+ }
2666
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
2667
+ let skip = 0;
2668
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
2669
+ if (!isInjectedTurn(message, isInjected)) return true;
2670
+ skip += 1;
2671
+ return false;
2672
+ });
2673
+ return skip;
2674
+ }
2675
+ function filteredWindow(rawTotal, skip, options) {
2676
+ const total = Math.max(0, rawTotal - skip);
2677
+ const beforeIndex = options.beforeIndex ?? total;
2678
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
2679
+ return {
2680
+ total,
2681
+ fromIndex,
2682
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
2683
+ };
2684
+ }
2685
+ function pageOfConversation(conversation, options) {
2686
+ const { messages } = conversation;
2687
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
2688
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
2689
+ const skip = firstKept === -1 ? messages.length : firstKept;
2690
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
2691
+ return {
2692
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
2693
+ total,
2694
+ fromIndex,
2695
+ turnDurations: conversation.turnDurations
2696
+ };
2697
+ }
2652
2698
 
2653
2699
  // src/persistent/conversation-stream.ts
2654
2700
  async function foldTail(filePath, resume) {
@@ -4178,6 +4224,7 @@ var PersistentEngine = class {
4178
4224
  const tier = resolveTier(tierName, customTiers);
4179
4225
  const filePath = canonicalPath(rawFilePath);
4180
4226
  const existing = this.files.getByPath(filePath);
4227
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4181
4228
  const classified = classify(filePath, existing);
4182
4229
  const stat6 = classified.stat;
4183
4230
  let change = classified.change;
@@ -4185,7 +4232,7 @@ var PersistentEngine = class {
4185
4232
  this.markDeleted(filePath);
4186
4233
  return { meta: null, change: "vanished" };
4187
4234
  }
4188
- if (change === "unchanged" && !force) {
4235
+ if (change === "unchanged" && !refold) {
4189
4236
  const meta2 = this.conversations.getBySourcePath(filePath);
4190
4237
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4191
4238
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4202,7 +4249,7 @@ var PersistentEngine = class {
4202
4249
  );
4203
4250
  return { meta: meta2, change };
4204
4251
  }
4205
- const resume = change === "appended" && !force && existing?.reducer_state;
4252
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4206
4253
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4207
4254
  const startOffset = resume ? existing.last_indexed_offset : 0;
4208
4255
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4363,6 +4410,10 @@ var PersistentEngine = class {
4363
4410
  // resolved to an indexed conversation. For conversations large enough to span
4364
4411
  // a checkpoint interval, checkpoints are built lazily on first access and
4365
4412
  // reused thereafter. Smaller conversations read from the start (cheap).
4413
+ //
4414
+ // The window is over the conversation with its leading injected-context run
4415
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4416
+ // index current (refreshFile) first; this reads what the index describes.
4366
4417
  async getPage(id, options) {
4367
4418
  const meta = this.conversations.getByIdOrSession(id);
4368
4419
  if (!meta) return null;
@@ -4370,24 +4421,28 @@ var PersistentEngine = class {
4370
4421
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4371
4422
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4372
4423
  if (!conversation) return null;
4373
- const { messages } = conversation;
4374
- const total2 = messages.length;
4375
- const beforeIndex2 = options.beforeIndex ?? total2;
4376
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4377
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4378
- }
4379
- const total = this.conversations.pageMessageCount(filePath);
4380
- await this.ensureCheckpoints(filePath, total);
4381
- const beforeIndex = options.beforeIndex ?? total;
4382
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4383
- let floor = this.checkpoints.floor(filePath, fromIndex);
4424
+ return pageOfConversation(conversation, options);
4425
+ }
4426
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4427
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4428
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4429
+ await this.ensureCheckpoints(filePath, rawTotal);
4430
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4384
4431
  if (floor) {
4385
4432
  try {
4386
4433
  if (floor.byteOffset > statSync5(filePath).size) floor = null;
4387
4434
  } catch {
4388
4435
  }
4389
4436
  }
4390
- return readPage(filePath, total, options, floor);
4437
+ const page = await readPage(filePath, rawTotal, raw, floor);
4438
+ const state = this.files.getByPath(filePath)?.reducer_state;
4439
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4440
+ return {
4441
+ messages: page.messages,
4442
+ total,
4443
+ fromIndex,
4444
+ turnDurations: durations?.length ? durations : void 0
4445
+ };
4391
4446
  }
4392
4447
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4393
4448
  // file → full build; a file that grew → extend from the last persisted
@@ -4876,36 +4931,94 @@ var ConversationScanner = class {
4876
4931
  }
4877
4932
  }
4878
4933
  // Return one bounded window of a conversation's messages plus the total
4879
- // message count, so a caller can serve the last page and scroll back without
4880
- // holding the whole conversation itself.
4934
+ // message count, the conversation's current meta and its turn durations, so a
4935
+ // caller can serve the last page and scroll back without holding the whole
4936
+ // conversation itself.
4881
4937
  //
4882
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4938
+ // Messages are those of the full read with the LEADING run of injected user
4939
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
4940
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
4941
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4883
4942
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4884
4943
  // the window's start index, so the caller can derive `has_more_older`
4885
4944
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4886
4945
  // same contract as getConversation.
4887
4946
  //
4888
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4889
- // checkpoint and parses only the requested window (checkpoints are built
4890
- // lazily for large conversations). Windowed message indices are proven
4891
- // identical to parseConversation().messages.slice(...) by the paged-reader
4892
- // equivalence test.
4893
- //
4894
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4895
- // which is identical by construction.
4947
+ // Every call reads the file as it is on disk now, never a cached parse:
4948
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
4949
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
4950
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
4951
+ // whole per call.
4896
4952
  async getConversationPage(id, options) {
4953
+ const cid = canonicalPath(id);
4954
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
4955
+ if (!known) return null;
4956
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
4957
+ if (!this.persistent && isClaude) {
4958
+ const index = await this.pageIndexFor(known.filePath, known.account);
4959
+ if (!index) return null;
4960
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
4961
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
4962
+ const rawFrom = raw.beforeIndex - raw.limit;
4963
+ let floor = null;
4964
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
4965
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
4966
+ return {
4967
+ messages: page.messages,
4968
+ total,
4969
+ fromIndex,
4970
+ meta: index.meta,
4971
+ turnDurations: index.turnDurations
4972
+ };
4973
+ }
4974
+ const meta = await this.refreshFile(known.filePath, known.account);
4975
+ if (!meta) return null;
4897
4976
  if (this.persistent) {
4898
- return this.engine().getPage(canonicalPath(id), options);
4977
+ const page = await this.engine().getPage(meta.filePath, options);
4978
+ return page && { ...page, meta };
4979
+ }
4980
+ const conversation = await this.getConversation(meta.filePath);
4981
+ return conversation && { ...pageOfConversation(conversation, options), meta };
4982
+ }
4983
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
4984
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
4985
+ // reused until the file's size or mtime changes. Memory stays bounded: the
4986
+ // file is never held, and the index LRU is capped.
4987
+ // ponytail: any change rebuilds from byte 0; extending from the last
4988
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
4989
+ // upgrade path if live-appended transcripts make this hot.
4990
+ pageIndexes = new LRUCache(32);
4991
+ async pageIndexFor(filePath, account) {
4992
+ let stat6;
4993
+ try {
4994
+ stat6 = statSync6(filePath);
4995
+ } catch {
4996
+ this.pageIndexes.delete(filePath);
4997
+ return null;
4899
4998
  }
4900
- const conversation = await this.getConversation(id);
4901
- if (!conversation) return null;
4902
- const { messages } = conversation;
4903
- const total = messages.length;
4904
- const { limit } = options;
4905
- const beforeIndex = options.beforeIndex ?? total;
4906
- const fromIndex = Math.max(0, beforeIndex - limit);
4907
- const window = messages.slice(fromIndex, beforeIndex);
4908
- return { messages: window, total, fromIndex };
4999
+ const cached = this.pageIndexes.get(filePath);
5000
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
5001
+ const state = initialReducerState();
5002
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
5003
+ reduceLine(state, entry, this.lastTier);
5004
+ return false;
5005
+ });
5006
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
5007
+ if (!meta) {
5008
+ this.pageIndexes.delete(filePath);
5009
+ return null;
5010
+ }
5011
+ meta.gitBranch = readGitBranch(meta.projectPath);
5012
+ const index = {
5013
+ size: stat6.size,
5014
+ mtimeMs: stat6.mtimeMs,
5015
+ checkpoints,
5016
+ meta,
5017
+ rawTotal: state.pageMessageCount,
5018
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
5019
+ };
5020
+ this.pageIndexes.set(filePath, index);
5021
+ return index;
4909
5022
  }
4910
5023
  // Parse one JSONL file directly and slice a page window — without any prior
4911
5024
  // scan() or metadata index. This is the cold-start fast path: a single