@threadbase-sh/scanner 0.23.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -740,8 +740,6 @@ function initialReducerState() {
740
740
  lastMessageSender: "user",
741
741
  agentId: "",
742
742
  isSidechain: false,
743
- isTeammate: false,
744
- firstUserSeen: false,
745
743
  firstMessage: null,
746
744
  lastMessage: null,
747
745
  lastPrompt: "",
@@ -751,7 +749,8 @@ function initialReducerState() {
751
749
  snippetParts: [],
752
750
  previewLength: 0,
753
751
  snippetLength: 0,
754
- badJsonLines: 0
752
+ badJsonLines: 0,
753
+ turnDurations: []
755
754
  };
756
755
  }
757
756
  function reduceLine(state, entry, tier) {
@@ -765,6 +764,8 @@ function reduceLine(state, entry, tier) {
765
764
  const ts = entry.timestamp;
766
765
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
767
766
  }
767
+ const turnDuration = turnDurationOf(entry);
768
+ if (turnDuration) state.turnDurations?.push(turnDuration);
768
769
  const type = entry.type;
769
770
  if (type === "last-prompt") {
770
771
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -778,10 +779,6 @@ function reduceLine(state, entry, tier) {
778
779
  return;
779
780
  }
780
781
  if (state.model === null && msg?.model) state.model = msg.model;
781
- if (type === "user" && !state.firstUserSeen) {
782
- state.firstUserSeen = true;
783
- if (isTeammateContent(msg?.content)) state.isTeammate = true;
784
- }
785
782
  const content = extractTextContent(msg?.content);
786
783
  const hasToolUseResult = type === "user" && entry.toolUseResult != null;
787
784
  const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
@@ -848,7 +845,9 @@ function finalizeMeta(state, filePath, account, tier) {
848
845
  // A sidechain line's `sessionId` IS the parent's id — that is exactly the
849
846
  // conflation `subagentId` exists to undo, so it is the parent UUID here.
850
847
  parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
851
- isTeammate: state.isTeammate,
848
+ // A teammate's records carry teamName; the lead's never do, yet the lead
849
+ // receives <teammate-message> tags too, so the tag is not the discriminator.
850
+ isTeammate: Boolean(state.teamName),
852
851
  teamName: state.teamName || null,
853
852
  toolNames: state.toolNames,
854
853
  firstMessage: state.firstMessage,
@@ -863,6 +862,16 @@ function getShortProjectName(fullPath) {
863
862
  }
864
863
 
865
864
  // src/parser.ts
865
+ function turnDurationOf(entry) {
866
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
867
+ return null;
868
+ }
869
+ return {
870
+ durationMs: entry.durationMs,
871
+ messageCount: entry.messageCount || 0,
872
+ uuid: entry.uuid
873
+ };
874
+ }
866
875
  async function parseMeta(filePath, account, tier, onEntry) {
867
876
  const log = getLogger();
868
877
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -916,12 +925,9 @@ async function parseConversation(filePath, account) {
916
925
  badJsonLines++;
917
926
  continue;
918
927
  }
919
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
920
- turnDurations.push({
921
- durationMs: entry.durationMs,
922
- messageCount: entry.messageCount || 0,
923
- uuid: entry.uuid
924
- });
928
+ const turnDuration = turnDurationOf(entry);
929
+ if (turnDuration) {
930
+ turnDurations.push(turnDuration);
925
931
  continue;
926
932
  }
927
933
  const message = reduceConvLine(state, entry);
@@ -1019,12 +1025,6 @@ function isOnlyToolResultContent(content) {
1019
1025
  if (!Array.isArray(content)) return false;
1020
1026
  return content.length > 0 && content.every((item) => item?.type === "tool_result");
1021
1027
  }
1022
- function isTeammateContent(content) {
1023
- const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
1024
- (item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
1025
- ).join("") : "";
1026
- return raw.includes("<teammate-message");
1027
- }
1028
1028
  function extractThinking(content) {
1029
1029
  if (!Array.isArray(content)) return { content: "", signature: "" };
1030
1030
  const blocks = content.filter((item) => item?.type === "thinking");
@@ -2608,7 +2608,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
2608
2608
  }
2609
2609
  return { offset, line };
2610
2610
  }
2611
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
2611
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
2612
2612
  const checkpoints = [];
2613
2613
  const state = from ? from.state : initialConvState();
2614
2614
  let index = from ? from.messageIndex : 0;
@@ -2628,7 +2628,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
2628
2628
  });
2629
2629
  }
2630
2630
  return false;
2631
- }
2631
+ },
2632
+ onEntry
2632
2633
  );
2633
2634
  return checkpoints;
2634
2635
  }
@@ -2649,6 +2650,41 @@ async function readPage(filePath, total, options, floor) {
2649
2650
  applyTeamInfo(window, state);
2650
2651
  return { messages: window, total, fromIndex };
2651
2652
  }
2653
+ function isInjectedTurn(message, isInjected) {
2654
+ return message.role === "user" && isInjected(message.text ?? "");
2655
+ }
2656
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
2657
+ let skip = 0;
2658
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
2659
+ if (!isInjectedTurn(message, isInjected)) return true;
2660
+ skip += 1;
2661
+ return false;
2662
+ });
2663
+ return skip;
2664
+ }
2665
+ function filteredWindow(rawTotal, skip, options) {
2666
+ const total = Math.max(0, rawTotal - skip);
2667
+ const beforeIndex = options.beforeIndex ?? total;
2668
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
2669
+ return {
2670
+ total,
2671
+ fromIndex,
2672
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
2673
+ };
2674
+ }
2675
+ function pageOfConversation(conversation, options) {
2676
+ const { messages } = conversation;
2677
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
2678
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
2679
+ const skip = firstKept === -1 ? messages.length : firstKept;
2680
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
2681
+ return {
2682
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
2683
+ total,
2684
+ fromIndex,
2685
+ turnDurations: conversation.turnDurations
2686
+ };
2687
+ }
2652
2688
 
2653
2689
  // src/persistent/conversation-stream.ts
2654
2690
  async function foldTail(filePath, resume) {
@@ -4178,6 +4214,7 @@ var PersistentEngine = class {
4178
4214
  const tier = resolveTier(tierName, customTiers);
4179
4215
  const filePath = canonicalPath(rawFilePath);
4180
4216
  const existing = this.files.getByPath(filePath);
4217
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4181
4218
  const classified = classify(filePath, existing);
4182
4219
  const stat6 = classified.stat;
4183
4220
  let change = classified.change;
@@ -4185,7 +4222,7 @@ var PersistentEngine = class {
4185
4222
  this.markDeleted(filePath);
4186
4223
  return { meta: null, change: "vanished" };
4187
4224
  }
4188
- if (change === "unchanged" && !force) {
4225
+ if (change === "unchanged" && !refold) {
4189
4226
  const meta2 = this.conversations.getBySourcePath(filePath);
4190
4227
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4191
4228
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4202,7 +4239,7 @@ var PersistentEngine = class {
4202
4239
  );
4203
4240
  return { meta: meta2, change };
4204
4241
  }
4205
- const resume = change === "appended" && !force && existing?.reducer_state;
4242
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4206
4243
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4207
4244
  const startOffset = resume ? existing.last_indexed_offset : 0;
4208
4245
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4363,6 +4400,10 @@ var PersistentEngine = class {
4363
4400
  // resolved to an indexed conversation. For conversations large enough to span
4364
4401
  // a checkpoint interval, checkpoints are built lazily on first access and
4365
4402
  // reused thereafter. Smaller conversations read from the start (cheap).
4403
+ //
4404
+ // The window is over the conversation with its leading injected-context run
4405
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4406
+ // index current (refreshFile) first; this reads what the index describes.
4366
4407
  async getPage(id, options) {
4367
4408
  const meta = this.conversations.getByIdOrSession(id);
4368
4409
  if (!meta) return null;
@@ -4370,24 +4411,28 @@ var PersistentEngine = class {
4370
4411
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4371
4412
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4372
4413
  if (!conversation) return null;
4373
- const { messages } = conversation;
4374
- const total2 = messages.length;
4375
- const beforeIndex2 = options.beforeIndex ?? total2;
4376
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4377
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4378
- }
4379
- const total = this.conversations.pageMessageCount(filePath);
4380
- await this.ensureCheckpoints(filePath, total);
4381
- const beforeIndex = options.beforeIndex ?? total;
4382
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4383
- let floor = this.checkpoints.floor(filePath, fromIndex);
4414
+ return pageOfConversation(conversation, options);
4415
+ }
4416
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4417
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4418
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4419
+ await this.ensureCheckpoints(filePath, rawTotal);
4420
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4384
4421
  if (floor) {
4385
4422
  try {
4386
4423
  if (floor.byteOffset > statSync5(filePath).size) floor = null;
4387
4424
  } catch {
4388
4425
  }
4389
4426
  }
4390
- return readPage(filePath, total, options, floor);
4427
+ const page = await readPage(filePath, rawTotal, raw, floor);
4428
+ const state = this.files.getByPath(filePath)?.reducer_state;
4429
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4430
+ return {
4431
+ messages: page.messages,
4432
+ total,
4433
+ fromIndex,
4434
+ turnDurations: durations?.length ? durations : void 0
4435
+ };
4391
4436
  }
4392
4437
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4393
4438
  // file → full build; a file that grew → extend from the last persisted
@@ -4876,36 +4921,94 @@ var ConversationScanner = class {
4876
4921
  }
4877
4922
  }
4878
4923
  // Return one bounded window of a conversation's messages plus the total
4879
- // message count, so a caller can serve the last page and scroll back without
4880
- // holding the whole conversation itself.
4924
+ // message count, the conversation's current meta and its turn durations, so a
4925
+ // caller can serve the last page and scroll back without holding the whole
4926
+ // conversation itself.
4881
4927
  //
4882
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4928
+ // Messages are those of the full read with the LEADING run of injected user
4929
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
4930
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
4931
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4883
4932
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4884
4933
  // the window's start index, so the caller can derive `has_more_older`
4885
4934
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4886
4935
  // same contract as getConversation.
4887
4936
  //
4888
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4889
- // checkpoint and parses only the requested window (checkpoints are built
4890
- // lazily for large conversations). Windowed message indices are proven
4891
- // identical to parseConversation().messages.slice(...) by the paged-reader
4892
- // equivalence test.
4893
- //
4894
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4895
- // which is identical by construction.
4937
+ // Every call reads the file as it is on disk now, never a cached parse:
4938
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
4939
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
4940
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
4941
+ // whole per call.
4896
4942
  async getConversationPage(id, options) {
4943
+ const cid = canonicalPath(id);
4944
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
4945
+ if (!known) return null;
4946
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
4947
+ if (!this.persistent && isClaude) {
4948
+ const index = await this.pageIndexFor(known.filePath, known.account);
4949
+ if (!index) return null;
4950
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
4951
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
4952
+ const rawFrom = raw.beforeIndex - raw.limit;
4953
+ let floor = null;
4954
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
4955
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
4956
+ return {
4957
+ messages: page.messages,
4958
+ total,
4959
+ fromIndex,
4960
+ meta: index.meta,
4961
+ turnDurations: index.turnDurations
4962
+ };
4963
+ }
4964
+ const meta = await this.refreshFile(known.filePath, known.account);
4965
+ if (!meta) return null;
4897
4966
  if (this.persistent) {
4898
- return this.engine().getPage(canonicalPath(id), options);
4967
+ const page = await this.engine().getPage(meta.filePath, options);
4968
+ return page && { ...page, meta };
4969
+ }
4970
+ const conversation = await this.getConversation(meta.filePath);
4971
+ return conversation && { ...pageOfConversation(conversation, options), meta };
4972
+ }
4973
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
4974
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
4975
+ // reused until the file's size or mtime changes. Memory stays bounded: the
4976
+ // file is never held, and the index LRU is capped.
4977
+ // ponytail: any change rebuilds from byte 0; extending from the last
4978
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
4979
+ // upgrade path if live-appended transcripts make this hot.
4980
+ pageIndexes = new LRUCache(32);
4981
+ async pageIndexFor(filePath, account) {
4982
+ let stat6;
4983
+ try {
4984
+ stat6 = statSync6(filePath);
4985
+ } catch {
4986
+ this.pageIndexes.delete(filePath);
4987
+ return null;
4899
4988
  }
4900
- const conversation = await this.getConversation(id);
4901
- if (!conversation) return null;
4902
- const { messages } = conversation;
4903
- const total = messages.length;
4904
- const { limit } = options;
4905
- const beforeIndex = options.beforeIndex ?? total;
4906
- const fromIndex = Math.max(0, beforeIndex - limit);
4907
- const window = messages.slice(fromIndex, beforeIndex);
4908
- return { messages: window, total, fromIndex };
4989
+ const cached = this.pageIndexes.get(filePath);
4990
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
4991
+ const state = initialReducerState();
4992
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
4993
+ reduceLine(state, entry, this.lastTier);
4994
+ return false;
4995
+ });
4996
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
4997
+ if (!meta) {
4998
+ this.pageIndexes.delete(filePath);
4999
+ return null;
5000
+ }
5001
+ meta.gitBranch = readGitBranch(meta.projectPath);
5002
+ const index = {
5003
+ size: stat6.size,
5004
+ mtimeMs: stat6.mtimeMs,
5005
+ checkpoints,
5006
+ meta,
5007
+ rawTotal: state.pageMessageCount,
5008
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
5009
+ };
5010
+ this.pageIndexes.set(filePath, index);
5011
+ return index;
4909
5012
  }
4910
5013
  // Parse one JSONL file directly and slice a page window — without any prior
4911
5014
  // scan() or metadata index. This is the cold-start fast path: a single