@threadbase-sh/scanner 0.23.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -5,7 +5,7 @@ import { Command } from "commander";
5
5
  import pino2 from "pino";
6
6
 
7
7
  // package.json
8
- var version = "0.23.0";
8
+ var version = "0.24.1";
9
9
 
10
10
  // src/logger.ts
11
11
  import pino from "pino";
@@ -966,8 +966,6 @@ function initialReducerState() {
966
966
  lastMessageSender: "user",
967
967
  agentId: "",
968
968
  isSidechain: false,
969
- isTeammate: false,
970
- firstUserSeen: false,
971
969
  firstMessage: null,
972
970
  lastMessage: null,
973
971
  lastPrompt: "",
@@ -977,7 +975,8 @@ function initialReducerState() {
977
975
  snippetParts: [],
978
976
  previewLength: 0,
979
977
  snippetLength: 0,
980
- badJsonLines: 0
978
+ badJsonLines: 0,
979
+ turnDurations: []
981
980
  };
982
981
  }
983
982
  function reduceLine(state, entry, tier) {
@@ -991,6 +990,8 @@ function reduceLine(state, entry, tier) {
991
990
  const ts = entry.timestamp;
992
991
  if (!state.latestTimestamp || ts > state.latestTimestamp) state.latestTimestamp = ts;
993
992
  }
993
+ const turnDuration = turnDurationOf(entry);
994
+ if (turnDuration) state.turnDurations?.push(turnDuration);
994
995
  const type = entry.type;
995
996
  if (type === "last-prompt") {
996
997
  if (entry.lastPrompt && !state.lastPrompt) state.lastPrompt = entry.lastPrompt;
@@ -1004,10 +1005,6 @@ function reduceLine(state, entry, tier) {
1004
1005
  return;
1005
1006
  }
1006
1007
  if (state.model === null && msg?.model) state.model = msg.model;
1007
- if (type === "user" && !state.firstUserSeen) {
1008
- state.firstUserSeen = true;
1009
- if (isTeammateContent(msg?.content)) state.isTeammate = true;
1010
- }
1011
1008
  const content = extractTextContent(msg?.content);
1012
1009
  const hasToolUseResult = type === "user" && entry.toolUseResult != null;
1013
1010
  const isOnlyToolResult = hasToolUseResult && isOnlyToolResultContent(msg?.content);
@@ -1074,7 +1071,9 @@ function finalizeMeta(state, filePath, account, tier) {
1074
1071
  // A sidechain line's `sessionId` IS the parent's id — that is exactly the
1075
1072
  // conflation `subagentId` exists to undo, so it is the parent UUID here.
1076
1073
  parentSessionUuid: isSidechain && state.sessionId ? state.sessionId : void 0,
1077
- isTeammate: state.isTeammate,
1074
+ // A teammate's records carry teamName; the lead's never do, yet the lead
1075
+ // receives <teammate-message> tags too, so the tag is not the discriminator.
1076
+ isTeammate: Boolean(state.teamName),
1078
1077
  teamName: state.teamName || null,
1079
1078
  toolNames: state.toolNames,
1080
1079
  firstMessage: state.firstMessage,
@@ -1089,6 +1088,16 @@ function getShortProjectName(fullPath) {
1089
1088
  }
1090
1089
 
1091
1090
  // src/parser.ts
1091
+ function turnDurationOf(entry) {
1092
+ if (entry.type !== "system" || entry.subtype !== "turn_duration" || typeof entry.durationMs !== "number") {
1093
+ return null;
1094
+ }
1095
+ return {
1096
+ durationMs: entry.durationMs,
1097
+ messageCount: entry.messageCount || 0,
1098
+ uuid: entry.uuid
1099
+ };
1100
+ }
1092
1101
  async function parseMeta(filePath, account, tier, onEntry) {
1093
1102
  const log = getLogger();
1094
1103
  log.trace({ filePath, account, tier: tier.name }, "parseMeta: start");
@@ -1142,12 +1151,9 @@ async function parseConversation(filePath, account) {
1142
1151
  badJsonLines++;
1143
1152
  continue;
1144
1153
  }
1145
- if (entry.type === "system" && entry.subtype === "turn_duration" && typeof entry.durationMs === "number") {
1146
- turnDurations.push({
1147
- durationMs: entry.durationMs,
1148
- messageCount: entry.messageCount || 0,
1149
- uuid: entry.uuid
1150
- });
1154
+ const turnDuration = turnDurationOf(entry);
1155
+ if (turnDuration) {
1156
+ turnDurations.push(turnDuration);
1151
1157
  continue;
1152
1158
  }
1153
1159
  const message = reduceConvLine(state, entry);
@@ -1245,12 +1251,6 @@ function isOnlyToolResultContent(content) {
1245
1251
  if (!Array.isArray(content)) return false;
1246
1252
  return content.length > 0 && content.every((item) => item?.type === "tool_result");
1247
1253
  }
1248
- function isTeammateContent(content) {
1249
- const raw = typeof content === "string" ? content : Array.isArray(content) ? content.map(
1250
- (item) => typeof item === "string" ? item : item?.type === "text" ? item.text ?? "" : ""
1251
- ).join("") : "";
1252
- return raw.includes("<teammate-message");
1253
- }
1254
1254
  function extractThinking(content) {
1255
1255
  if (!Array.isArray(content)) return { content: "", signature: "" };
1256
1256
  const blocks = content.filter((item) => item?.type === "thinking");
@@ -1364,7 +1364,7 @@ async function streamMessages(filePath, startOffset, startLine, state, onMessage
1364
1364
  }
1365
1365
  return { offset, line };
1366
1366
  }
1367
- async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null) {
1367
+ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from = null, onEntry) {
1368
1368
  const checkpoints = [];
1369
1369
  const state = from ? from.state : initialConvState();
1370
1370
  let index = from ? from.messageIndex : 0;
@@ -1384,7 +1384,8 @@ async function buildCheckpoints(filePath, interval = CHECKPOINT_INTERVAL, from =
1384
1384
  });
1385
1385
  }
1386
1386
  return false;
1387
- }
1387
+ },
1388
+ onEntry
1388
1389
  );
1389
1390
  return checkpoints;
1390
1391
  }
@@ -1405,6 +1406,41 @@ async function readPage(filePath, total, options, floor) {
1405
1406
  applyTeamInfo(window, state);
1406
1407
  return { messages: window, total, fromIndex };
1407
1408
  }
1409
+ function isInjectedTurn(message, isInjected) {
1410
+ return message.role === "user" && isInjected(message.text ?? "");
1411
+ }
1412
+ async function countLeadingInjected(filePath, isInjected = isCodexInjectedContext) {
1413
+ let skip = 0;
1414
+ await streamMessages(filePath, 0, 0, initialConvState(), (message) => {
1415
+ if (!isInjectedTurn(message, isInjected)) return true;
1416
+ skip += 1;
1417
+ return false;
1418
+ });
1419
+ return skip;
1420
+ }
1421
+ function filteredWindow(rawTotal, skip, options) {
1422
+ const total = Math.max(0, rawTotal - skip);
1423
+ const beforeIndex = options.beforeIndex ?? total;
1424
+ const fromIndex = Math.max(0, beforeIndex - options.limit);
1425
+ return {
1426
+ total,
1427
+ fromIndex,
1428
+ raw: { beforeIndex: beforeIndex + skip, limit: beforeIndex - fromIndex }
1429
+ };
1430
+ }
1431
+ function pageOfConversation(conversation, options) {
1432
+ const { messages } = conversation;
1433
+ const isInjected = options.isInjectedContext ?? isCodexInjectedContext;
1434
+ const firstKept = messages.findIndex((m) => !isInjectedTurn(m, isInjected));
1435
+ const skip = firstKept === -1 ? messages.length : firstKept;
1436
+ const { total, fromIndex, raw } = filteredWindow(messages.length, skip, options);
1437
+ return {
1438
+ messages: messages.slice(raw.beforeIndex - raw.limit, raw.beforeIndex),
1439
+ total,
1440
+ fromIndex,
1441
+ turnDurations: conversation.turnDurations
1442
+ };
1443
+ }
1408
1444
 
1409
1445
  // src/persistent/conversation-stream.ts
1410
1446
  async function foldTail(filePath, resume) {
@@ -4082,6 +4118,7 @@ var PersistentEngine = class {
4082
4118
  const tier = resolveTier(tierName, customTiers);
4083
4119
  const filePath = canonicalPath(rawFilePath);
4084
4120
  const existing = this.files.getByPath(filePath);
4121
+ const refold = force || (!provider || provider.name === CLAUDE_CODE_PROVIDER) && !!existing?.reducer_state && !existing.reducer_state.includes('"turnDurations"');
4085
4122
  const classified = classify(filePath, existing);
4086
4123
  const stat6 = classified.stat;
4087
4124
  let change = classified.change;
@@ -4089,7 +4126,7 @@ var PersistentEngine = class {
4089
4126
  this.markDeleted(filePath);
4090
4127
  return { meta: null, change: "vanished" };
4091
4128
  }
4092
- if (change === "unchanged" && !force) {
4129
+ if (change === "unchanged" && !refold) {
4093
4130
  const meta2 = this.conversations.getBySourcePath(filePath);
4094
4131
  const sideStoresMoved = provider && provider.name !== CLAUDE_CODE_PROVIDER && meta2 && JSON.stringify(nameStoreSlotsFor(provider.name, filePath, meta2.sessionId)) !== existing?.reducer_state;
4095
4132
  if (!sideStoresMoved) return { meta: meta2, change };
@@ -4106,7 +4143,7 @@ var PersistentEngine = class {
4106
4143
  );
4107
4144
  return { meta: meta2, change };
4108
4145
  }
4109
- const resume = change === "appended" && !force && existing?.reducer_state;
4146
+ const resume = change === "appended" && !refold && existing?.reducer_state;
4110
4147
  const state = resume ? JSON.parse(existing.reducer_state) : initialReducerState();
4111
4148
  const startOffset = resume ? existing.last_indexed_offset : 0;
4112
4149
  const startLine = resume ? existing.last_indexed_line : 0;
@@ -4267,6 +4304,10 @@ var PersistentEngine = class {
4267
4304
  // resolved to an indexed conversation. For conversations large enough to span
4268
4305
  // a checkpoint interval, checkpoints are built lazily on first access and
4269
4306
  // reused thereafter. Smaller conversations read from the start (cheap).
4307
+ //
4308
+ // The window is over the conversation with its leading injected-context run
4309
+ // dropped (options.isInjectedContext, see filteredWindow). The caller keeps the
4310
+ // index current (refreshFile) first; this reads what the index describes.
4270
4311
  async getPage(id, options) {
4271
4312
  const meta = this.conversations.getByIdOrSession(id);
4272
4313
  if (!meta) return null;
@@ -4274,24 +4315,28 @@ var PersistentEngine = class {
4274
4315
  if (meta.provider === CODEX_CLI_PROVIDER || meta.provider === CURSOR_PROVIDER || meta.provider === COPILOT_PROVIDER) {
4275
4316
  const conversation = meta.provider === CODEX_CLI_PROVIDER ? await parseCodexConversation(filePath, meta.account) : meta.provider === CURSOR_PROVIDER ? await parseCursorConversation(filePath, meta.account) : await parseCopilotConversation(filePath, meta.account);
4276
4317
  if (!conversation) return null;
4277
- const { messages } = conversation;
4278
- const total2 = messages.length;
4279
- const beforeIndex2 = options.beforeIndex ?? total2;
4280
- const fromIndex2 = Math.max(0, beforeIndex2 - options.limit);
4281
- return { messages: messages.slice(fromIndex2, beforeIndex2), total: total2, fromIndex: fromIndex2 };
4282
- }
4283
- const total = this.conversations.pageMessageCount(filePath);
4284
- await this.ensureCheckpoints(filePath, total);
4285
- const beforeIndex = options.beforeIndex ?? total;
4286
- const fromIndex = Math.max(0, beforeIndex - options.limit);
4287
- let floor = this.checkpoints.floor(filePath, fromIndex);
4318
+ return pageOfConversation(conversation, options);
4319
+ }
4320
+ const rawTotal = this.conversations.pageMessageCount(filePath);
4321
+ const skip = await countLeadingInjected(filePath, options.isInjectedContext);
4322
+ const { total, fromIndex, raw } = filteredWindow(rawTotal, skip, options);
4323
+ await this.ensureCheckpoints(filePath, rawTotal);
4324
+ let floor = this.checkpoints.floor(filePath, raw.beforeIndex - raw.limit);
4288
4325
  if (floor) {
4289
4326
  try {
4290
4327
  if (floor.byteOffset > statSync5(filePath).size) floor = null;
4291
4328
  } catch {
4292
4329
  }
4293
4330
  }
4294
- return readPage(filePath, total, options, floor);
4331
+ const page = await readPage(filePath, rawTotal, raw, floor);
4332
+ const state = this.files.getByPath(filePath)?.reducer_state;
4333
+ const durations = state ? JSON.parse(state).turnDurations : void 0;
4334
+ return {
4335
+ messages: page.messages,
4336
+ total,
4337
+ fromIndex,
4338
+ turnDurations: durations?.length ? durations : void 0
4339
+ };
4295
4340
  }
4296
4341
  // Build or extend the checkpoint chain so it covers `total` messages. Cold
4297
4342
  // file → full build; a file that grew → extend from the last persisted
@@ -4816,36 +4861,94 @@ var ConversationScanner = class {
4816
4861
  }
4817
4862
  }
4818
4863
  // Return one bounded window of a conversation's messages plus the total
4819
- // message count, so a caller can serve the last page and scroll back without
4820
- // holding the whole conversation itself.
4864
+ // message count, the conversation's current meta and its turn durations, so a
4865
+ // caller can serve the last page and scroll back without holding the whole
4866
+ // conversation itself.
4821
4867
  //
4822
- // The window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4868
+ // Messages are those of the full read with the LEADING run of injected user
4869
+ // turns dropped (options.isInjectedContext, default isCodexInjectedContext);
4870
+ // `total`, `fromIndex` and `beforeIndex` all count in that filtered space. The
4871
+ // window is `[max(0, beforeIndex - limit), beforeIndex)` in chronological
4823
4872
  // order; `beforeIndex` defaults to `total` (the newest page). `fromIndex` is
4824
4873
  // the window's start index, so the caller can derive `has_more_older`
4825
4874
  // (fromIndex > 0). Returns null when the id can't be resolved/parsed — the
4826
4875
  // same contract as getConversation.
4827
4876
  //
4828
- // Persistent mode: a true bounded read — the engine seeks from the nearest
4829
- // checkpoint and parses only the requested window (checkpoints are built
4830
- // lazily for large conversations). Windowed message indices are proven
4831
- // identical to parseConversation().messages.slice(...) by the paged-reader
4832
- // equivalence test.
4833
- //
4834
- // Legacy mode: parse-once-then-slice via getConversation (cached in the LRU),
4835
- // which is identical by construction.
4877
+ // Every call reads the file as it is on disk now, never a cached parse:
4878
+ // persistent mode refreshes the index first (O(appended bytes)) and the engine
4879
+ // seeks from the nearest checkpoint. In-memory claude-code files keep a
4880
+ // stat-validated checkpoint index (pageIndexFor). Other providers are parsed
4881
+ // whole per call.
4836
4882
  async getConversationPage(id, options) {
4883
+ const cid = canonicalPath(id);
4884
+ const known = this.persistent ? this.engine().getByIdOrSession(cid) : this.metadataCache.get(cid) ?? this.resolveSessionId(cid);
4885
+ if (!known) return null;
4886
+ const isClaude = known.provider !== CODEX_CLI_PROVIDER && known.provider !== CURSOR_PROVIDER && known.provider !== COPILOT_PROVIDER;
4887
+ if (!this.persistent && isClaude) {
4888
+ const index = await this.pageIndexFor(known.filePath, known.account);
4889
+ if (!index) return null;
4890
+ const skip = await countLeadingInjected(known.filePath, options.isInjectedContext);
4891
+ const { total, fromIndex, raw } = filteredWindow(index.rawTotal, skip, options);
4892
+ const rawFrom = raw.beforeIndex - raw.limit;
4893
+ let floor = null;
4894
+ for (const c of index.checkpoints) if (c.messageIndex <= rawFrom) floor = c;
4895
+ const page = await readPage(known.filePath, index.rawTotal, raw, floor);
4896
+ return {
4897
+ messages: page.messages,
4898
+ total,
4899
+ fromIndex,
4900
+ meta: index.meta,
4901
+ turnDurations: index.turnDurations
4902
+ };
4903
+ }
4904
+ const meta = await this.refreshFile(known.filePath, known.account);
4905
+ if (!meta) return null;
4837
4906
  if (this.persistent) {
4838
- return this.engine().getPage(canonicalPath(id), options);
4907
+ const page = await this.engine().getPage(meta.filePath, options);
4908
+ return page && { ...page, meta };
4909
+ }
4910
+ const conversation = await this.getConversation(meta.filePath);
4911
+ return conversation && { ...pageOfConversation(conversation, options), meta };
4912
+ }
4913
+ // In-memory claude-code paging index: one streamed pass builds the checkpoint
4914
+ // chain and folds the metadata (meta, page total, turn durations) alongside,
4915
+ // reused until the file's size or mtime changes. Memory stays bounded: the
4916
+ // file is never held, and the index LRU is capped.
4917
+ // ponytail: any change rebuilds from byte 0; extending from the last
4918
+ // checkpoint plus a persisted ReducerState (as the engine does) is the
4919
+ // upgrade path if live-appended transcripts make this hot.
4920
+ pageIndexes = new LRUCache(32);
4921
+ async pageIndexFor(filePath, account) {
4922
+ let stat6;
4923
+ try {
4924
+ stat6 = statSync6(filePath);
4925
+ } catch {
4926
+ this.pageIndexes.delete(filePath);
4927
+ return null;
4839
4928
  }
4840
- const conversation = await this.getConversation(id);
4841
- if (!conversation) return null;
4842
- const { messages } = conversation;
4843
- const total = messages.length;
4844
- const { limit } = options;
4845
- const beforeIndex = options.beforeIndex ?? total;
4846
- const fromIndex = Math.max(0, beforeIndex - limit);
4847
- const window = messages.slice(fromIndex, beforeIndex);
4848
- return { messages: window, total, fromIndex };
4929
+ const cached = this.pageIndexes.get(filePath);
4930
+ if (cached && cached.size === stat6.size && cached.mtimeMs === stat6.mtimeMs) return cached;
4931
+ const state = initialReducerState();
4932
+ const checkpoints = await buildCheckpoints(filePath, CHECKPOINT_INTERVAL, null, (entry) => {
4933
+ reduceLine(state, entry, this.lastTier);
4934
+ return false;
4935
+ });
4936
+ const meta = finalizeMeta(state, filePath, account, this.lastTier);
4937
+ if (!meta) {
4938
+ this.pageIndexes.delete(filePath);
4939
+ return null;
4940
+ }
4941
+ meta.gitBranch = readGitBranch(meta.projectPath);
4942
+ const index = {
4943
+ size: stat6.size,
4944
+ mtimeMs: stat6.mtimeMs,
4945
+ checkpoints,
4946
+ meta,
4947
+ rawTotal: state.pageMessageCount,
4948
+ turnDurations: state.turnDurations?.length ? state.turnDurations : void 0
4949
+ };
4950
+ this.pageIndexes.set(filePath, index);
4951
+ return index;
4849
4952
  }
4850
4953
  // Parse one JSONL file directly and slice a page window — without any prior
4851
4954
  // scan() or metadata index. This is the cold-start fast path: a single