@threadbase-sh/scanner 0.23.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -830,8 +830,6 @@ function initialReducerState() {
830
830
  lastMessageSender: "user",
831
831
  agentId: "",
832
832
  isSidechain: false,
833
- isTeammate: false,
834
- firstUserSeen: false,
835
833
  firstMessage: null,
836
834
  lastMessage: null,
837
835
  lastPrompt: "",
@@ -841,7 +839,8 @@ function initialReducerState() {
841
839
  snippetParts: [],
842
840
  previewLength: 0,
843
841
  snippetLength: 0,
844
- badJsonLines: 0
842
+ badJsonLines: 0,
843
+ turnDurations: []
845
844
  };
846
845
  }
847
846
  function reduceLine(state, entry, tier) {
@@ -855,6 +854,8 @@ function reduceLine(state, entry, tier) {
855
854
  const ts = entry.timestamp;
856
855
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
857
856
  }
857
+ const turnDuration = turnDurationOf(entry);
858
+ if (turnDuration) state.turnDurations?.push(turnDuration);
858
859
  const type = entry.type;
859
860
  if (type === "last-prompt") {
860
861
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -868,10 +869,6 @@ function reduceLine(state, entry, tier) {
868
869
  return;
869
870
  }
870
871
  if (state.model === null && msg?.model) state.model = msg.model;
871
- if (type === "user" && !state.firstUserSeen) {
872
- state.firstUserSeen = true;
873
- if (isTeammateContent(msg?.content)) state.isTeammate = true;
874
- }
875
872
  const content = extractTextContent(msg?.content);
876
873
  const hasToolUseResult = type === "user" && entry.toolUseResult != null;
877
874
  const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
@@ -938,7 +935,9 @@ function finalizeMeta(state, filePath, account, tier) {
938
935
  // A sidechain line's `sessionId` IS the parent's id — that is exactly the
939
936
  // conflation `subagentId` exists to undo, so it is the parent UUID here.
940
937
  parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
941
- isTeammate: state.isTeammate,
938
+ // A teammate's records carry teamName; the lead's never do, yet the lead
939
+ // receives <teammate-message> tags too, so the tag is not the discriminator.
940
+ isTeammate: Boolean(state.teamName),
942
941
  teamName: state.teamName || null,
943
942
  toolNames: state.toolNames,
944
943
  firstMessage: state.firstMessage,
@@ -953,6 +952,16 @@ function getShortProjectName(fullPath) {
953
952
  }
954
953
 
955
954
  // src/parser.ts
955
+ function turnDurationOf(entry) {
956
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
957
+ return null;
958
+ }
959
+ return {
960
+ durationMs: entry.durationMs,
961
+ messageCount: entry.messageCount || 0,
962
+ uuid: entry.uuid
963
+ };
964
+ }
956
965
  async function parseMeta(filePath, account, tier, onEntry) {
957
966
  const log = getLogger();
958
967
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -1006,12 +1015,9 @@ async function parseConversation(filePath, account) {
1006
1015
  badJsonLines++;
1007
1016
  continue;
1008
1017
  }
1009
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
1010
- turnDurations.push({
1011
- durationMs: entry.durationMs,
1012
- messageCount: entry.messageCount || 0,
1013
- uuid: entry.uuid
1014
- });
1018
+ const turnDuration = turnDurationOf(entry);
1019
+ if (turnDuration) {
1020
+ turnDurations.push(turnDuration);
1015
1021
  continue;
1016
1022
  }
1017
1023
  const message = reduceConvLine(state, entry);
@@ -1109,12 +1115,6 @@ function isOnlyToolResultContent(content) {
1109
1115
  if (!Array.isArray(content)) return false;
1110
1116
  return content.length > 0 && content.every((item) => item?.type === "tool_result");
1111
1117
  }
1112
- function isTeammateContent(content) {
1113
- const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
1114
- (item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
1115
- ).join("") : "";
1116
- return raw.includes("<teammate-message");
1117
- }
1118
1118
  function extractThinking(content) {
1119
1119
  if (!Array.isArray(content)) return { content: "", signature: "" };
1120
1120
  const blocks = content.filter((item) => item?.type === "thinking");
@@ -2698,7 +2698,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
2698
2698
  }
2699
2699
  return { offset, line };
2700
2700
  }
2701
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
2701
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
2702
2702
  const checkpoints = [];
2703
2703
  const state = from ? from.state : initialConvState();
2704
2704
  let index = from ? from.messageIndex : 0;
@@ -2718,7 +2718,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
2718
2718
  });
2719
2719
  }
2720
2720
  return false;
2721
- }
2721
+ },
2722
+ onEntry
2722
2723
  );
2723
2724
  return checkpoints;
2724
2725
  }
@@ -2739,6 +2740,41 @@ async function readPage(filePath, total, options, floor) {
2739
2740
  applyTeamInfo(window, state);
2740
2741
  return { messages: window, total, fromIndex };
2741
2742
  }
2743
+ function isInjectedTurn(message, isInjected) {
2744
+ return message.role === "user" && isInjected(message.text ?? "");
2745
+ }
2746
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
2747
+ let skip = 0;
2748
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
2749
+ if (!isInjectedTurn(message, isInjected)) return true;
2750
+ skip += 1;
2751
+ return false;
2752
+ });
2753
+ return skip;
2754
+ }
2755
+ function filteredWindow(rawTotal, skip, options) {
2756
+ const total = Math.max(0, rawTotal - skip);
2757
+ const beforeIndex = options.beforeIndex ?? total;
2758
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
2759
+ return {
2760
+ total,
2761
+ fromIndex,
2762
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
2763
+ };
2764
+ }
2765
+ function pageOfConversation(conversation, options) {
2766
+ const { messages } = conversation;
2767
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
2768
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
2769
+ const skip = firstKept === -1 ? messages.length : firstKept;
2770
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
2771
+ return {
2772
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
2773
+ total,
2774
+ fromIndex,
2775
+ turnDurations: conversation.turnDurations
2776
+ };
2777
+ }
2742
2778
 
2743
2779
  // src/persistent/conversation-stream.ts
2744
2780
  async function foldTail(filePath, resume) {
@@ -4268,6 +4304,7 @@ var PersistentEngine = class {
4268
4304
  const tier = resolveTier(tierName, customTiers);
4269
4305
  const filePath = canonicalPath(rawFilePath);
4270
4306
  const existing = this.files.getByPath(filePath);
4307
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4271
4308
  const classified = classify(filePath, existing);
4272
4309
  const stat6 = classified.stat;
4273
4310
  let change = classified.change;
@@ -4275,7 +4312,7 @@ var PersistentEngine = class {
4275
4312
  this.markDeleted(filePath);
4276
4313
  return { meta: null, change: "vanished" };
4277
4314
  }
4278
- if (change === "unchanged" && !force) {
4315
+ if (change === "unchanged" && !refold) {
4279
4316
  const meta2 = this.conversations.getBySourcePath(filePath);
4280
4317
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4281
4318
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4292,7 +4329,7 @@ var PersistentEngine = class {
4292
4329
  );
4293
4330
  return { meta: meta2, change };
4294
4331
  }
4295
- const resume = change === "appended" && !force && existing?.reducer_state;
4332
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4296
4333
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4297
4334
  const startOffset = resume ? existing.last_indexed_offset : 0;
4298
4335
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4453,6 +4490,10 @@ var PersistentEngine = class {
4453
4490
  // resolved to an indexed conversation. For conversations large enough to span
4454
4491
  // a checkpoint interval, checkpoints are built lazily on first access and
4455
4492
  // reused thereafter. Smaller conversations read from the start (cheap).
4493
+ //
4494
+ // The window is over the conversation with its leading injected-context run
4495
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4496
+ // index current (refreshFile) first; this reads what the index describes.
4456
4497
  async getPage(id, options) {
4457
4498
  const meta = this.conversations.getByIdOrSession(id);
4458
4499
  if (!meta) return null;
@@ -4460,24 +4501,28 @@ var PersistentEngine = class {
4460
4501
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4461
4502
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4462
4503
  if (!conversation) return null;
4463
- const { messages } = conversation;
4464
- const total2 = messages.length;
4465
- const beforeIndex2 = options.beforeIndex ?? total2;
4466
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4467
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4468
- }
4469
- const total = this.conversations.pageMessageCount(filePath);
4470
- await this.ensureCheckpoints(filePath, total);
4471
- const beforeIndex = options.beforeIndex ?? total;
4472
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4473
- let floor = this.checkpoints.floor(filePath, fromIndex);
4504
+ return pageOfConversation(conversation, options);
4505
+ }
4506
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4507
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4508
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4509
+ await this.ensureCheckpoints(filePath, rawTotal);
4510
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4474
4511
  if (floor) {
4475
4512
  try {
4476
4513
  if (floor.byteOffset > (0, import_fs13.statSync)(filePath).size) floor = null;
4477
4514
  } catch {
4478
4515
  }
4479
4516
  }
4480
- return readPage(filePath, total, options, floor);
4517
+ const page = await readPage(filePath, rawTotal, raw, floor);
4518
+ const state = this.files.getByPath(filePath)?.reducer_state;
4519
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4520
+ return {
4521
+ messages: page.messages,
4522
+ total,
4523
+ fromIndex,
4524
+ turnDurations: durations?.length ? durations : void 0
4525
+ };
4481
4526
  }
4482
4527
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4483
4528
  // file → full build; a file that grew → extend from the last persisted
@@ -4966,36 +5011,94 @@ var ConversationScanner = class {
4966
5011
  }
4967
5012
  }
4968
5013
  // Return one bounded window of a conversation's messages plus the total
4969
- // message count, so a caller can serve the last page and scroll back without
4970
- // holding the whole conversation itself.
5014
+ // message count, the conversation's current meta and its turn durations, so a
5015
+ // caller can serve the last page and scroll back without holding the whole
5016
+ // conversation itself.
4971
5017
  //
4972
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
5018
+ // Messages are those of the full read with the LEADING run of injected user
5019
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
5020
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
5021
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4973
5022
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4974
5023
  // the window's start index, so the caller can derive `has_more_older`
4975
5024
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4976
5025
  // same contract as getConversation.
4977
5026
  //
4978
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4979
- // checkpoint and parses only the requested window (checkpoints are built
4980
- // lazily for large conversations). Windowed message indices are proven
4981
- // identical to parseConversation().messages.slice(...) by the paged-reader
4982
- // equivalence test.
4983
- //
4984
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4985
- // which is identical by construction.
5027
+ // Every call reads the file as it is on disk now, never a cached parse:
5028
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
5029
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
5030
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
5031
+ // whole per call.
4986
5032
  async getConversationPage(id, options) {
5033
+ const cid = canonicalPath(id);
5034
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
5035
+ if (!known) return null;
5036
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
5037
+ if (!this.persistent && isClaude) {
5038
+ const index = await this.pageIndexFor(known.filePath, known.account);
5039
+ if (!index) return null;
5040
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
5041
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
5042
+ const rawFrom = raw.beforeIndex - raw.limit;
5043
+ let floor = null;
5044
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
5045
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
5046
+ return {
5047
+ messages: page.messages,
5048
+ total,
5049
+ fromIndex,
5050
+ meta: index.meta,
5051
+ turnDurations: index.turnDurations
5052
+ };
5053
+ }
5054
+ const meta = await this.refreshFile(known.filePath, known.account);
5055
+ if (!meta) return null;
4987
5056
  if (this.persistent) {
4988
- return this.engine().getPage(canonicalPath(id), options);
5057
+ const page = await this.engine().getPage(meta.filePath, options);
5058
+ return page && { ...page, meta };
5059
+ }
5060
+ const conversation = await this.getConversation(meta.filePath);
5061
+ return conversation && { ...pageOfConversation(conversation, options), meta };
5062
+ }
5063
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
5064
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
5065
+ // reused until the file's size or mtime changes. Memory stays bounded: the
5066
+ // file is never held, and the index LRU is capped.
5067
+ // ponytail: any change rebuilds from byte 0; extending from the last
5068
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
5069
+ // upgrade path if live-appended transcripts make this hot.
5070
+ pageIndexes = new LRUCache(32);
5071
+ async pageIndexFor(filePath, account) {
5072
+ let stat6;
5073
+ try {
5074
+ stat6 = (0, import_fs14.statSync)(filePath);
5075
+ } catch {
5076
+ this.pageIndexes.delete(filePath);
5077
+ return null;
4989
5078
  }
4990
- const conversation = await this.getConversation(id);
4991
- if (!conversation) return null;
4992
- const { messages } = conversation;
4993
- const total = messages.length;
4994
- const { limit } = options;
4995
- const beforeIndex = options.beforeIndex ?? total;
4996
- const fromIndex = Math.max(0, beforeIndex - limit);
4997
- const window = messages.slice(fromIndex, beforeIndex);
4998
- return { messages: window, total, fromIndex };
5079
+ const cached = this.pageIndexes.get(filePath);
5080
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
5081
+ const state = initialReducerState();
5082
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
5083
+ reduceLine(state, entry, this.lastTier);
5084
+ return false;
5085
+ });
5086
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
5087
+ if (!meta) {
5088
+ this.pageIndexes.delete(filePath);
5089
+ return null;
5090
+ }
5091
+ meta.gitBranch = readGitBranch(meta.projectPath);
5092
+ const index = {
5093
+ size: stat6.size,
5094
+ mtimeMs: stat6.mtimeMs,
5095
+ checkpoints,
5096
+ meta,
5097
+ rawTotal: state.pageMessageCount,
5098
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
5099
+ };
5100
+ this.pageIndexes.set(filePath, index);
5101
+ return index;
4999
5102
  }
5000
5103
  // Parse one JSONL file directly and slice a page window — without any prior
5001
5104
  // scan() or metadata index. This is the cold-start fast path: a single