@cortexkit/opencode-magic-context 0.31.5 → 0.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +3 -3
  2. package/dist/agents/magic-context-prompt.d.ts.map +1 -1
  3. package/dist/config/project-security.d.ts +2 -0
  4. package/dist/config/project-security.d.ts.map +1 -1
  5. package/dist/config/schema/magic-context.d.ts +9 -0
  6. package/dist/config/schema/magic-context.d.ts.map +1 -1
  7. package/dist/features/magic-context/dreamer/cron.d.ts +2 -2
  8. package/dist/features/magic-context/dreamer/cron.d.ts.map +1 -1
  9. package/dist/features/magic-context/dreamer/evaluate-smart-notes.d.ts.map +1 -1
  10. package/dist/features/magic-context/message-index-async.d.ts +6 -2
  11. package/dist/features/magic-context/message-index-async.d.ts.map +1 -1
  12. package/dist/features/magic-context/message-index.d.ts +7 -4
  13. package/dist/features/magic-context/message-index.d.ts.map +1 -1
  14. package/dist/features/magic-context/migrations.d.ts +8 -16
  15. package/dist/features/magic-context/migrations.d.ts.map +1 -1
  16. package/dist/features/magic-context/plugin-messages.d.ts.map +1 -1
  17. package/dist/features/magic-context/project-embedding-registry.d.ts.map +1 -1
  18. package/dist/features/magic-context/session-project-backfill.d.ts +6 -3
  19. package/dist/features/magic-context/session-project-backfill.d.ts.map +1 -1
  20. package/dist/features/magic-context/session-project-storage.d.ts.map +1 -1
  21. package/dist/features/magic-context/smart-notes/capabilities.d.ts.map +1 -1
  22. package/dist/features/magic-context/smart-notes/compiler.d.ts +3 -1
  23. package/dist/features/magic-context/smart-notes/compiler.d.ts.map +1 -1
  24. package/dist/features/magic-context/smart-notes/runner.d.ts +1 -0
  25. package/dist/features/magic-context/smart-notes/runner.d.ts.map +1 -1
  26. package/dist/features/magic-context/smart-notes/sandbox-runner.d.ts +8 -1
  27. package/dist/features/magic-context/smart-notes/sandbox-runner.d.ts.map +1 -1
  28. package/dist/features/magic-context/smart-notes/ssrf-guard.d.ts +10 -0
  29. package/dist/features/magic-context/smart-notes/ssrf-guard.d.ts.map +1 -1
  30. package/dist/features/magic-context/smart-notes/storage.d.ts +16 -1
  31. package/dist/features/magic-context/smart-notes/storage.d.ts.map +1 -1
  32. package/dist/features/magic-context/storage-clone.d.ts +34 -0
  33. package/dist/features/magic-context/storage-clone.d.ts.map +1 -0
  34. package/dist/features/magic-context/storage-db.d.ts +1 -1
  35. package/dist/features/magic-context/storage-db.d.ts.map +1 -1
  36. package/dist/features/magic-context/storage-meta-persisted.d.ts +8 -5
  37. package/dist/features/magic-context/storage-meta-persisted.d.ts.map +1 -1
  38. package/dist/features/magic-context/storage-meta-session.d.ts.map +1 -1
  39. package/dist/features/magic-context/storage-schema-helpers.d.ts +3 -4
  40. package/dist/features/magic-context/storage-schema-helpers.d.ts.map +1 -1
  41. package/dist/features/magic-context/storage-tags.d.ts +11 -0
  42. package/dist/features/magic-context/storage-tags.d.ts.map +1 -1
  43. package/dist/features/magic-context/storage.d.ts +2 -1
  44. package/dist/features/magic-context/storage.d.ts.map +1 -1
  45. package/dist/features/magic-context/tagger.d.ts +10 -0
  46. package/dist/features/magic-context/tagger.d.ts.map +1 -1
  47. package/dist/features/magic-context/tool-owner-backfill.d.ts +2 -0
  48. package/dist/features/magic-context/tool-owner-backfill.d.ts.map +1 -1
  49. package/dist/features/magic-context/transform-decision-log.d.ts +1 -1
  50. package/dist/features/magic-context/transform-decision-log.d.ts.map +1 -1
  51. package/dist/features/magic-context/user-memory/review-user-memories.d.ts.map +1 -1
  52. package/dist/hooks/magic-context/apply-operations.d.ts +24 -0
  53. package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
  54. package/dist/hooks/magic-context/compartment-render-epoch.d.ts +12 -0
  55. package/dist/hooks/magic-context/compartment-render-epoch.d.ts.map +1 -0
  56. package/dist/hooks/magic-context/compartment-runner-validation.d.ts.map +1 -1
  57. package/dist/hooks/magic-context/compartment-trigger.d.ts +7 -1
  58. package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
  59. package/dist/hooks/magic-context/decay-render.d.ts +2 -0
  60. package/dist/hooks/magic-context/decay-render.d.ts.map +1 -1
  61. package/dist/hooks/magic-context/emergency-drop.d.ts +1 -0
  62. package/dist/hooks/magic-context/emergency-drop.d.ts.map +1 -1
  63. package/dist/hooks/magic-context/final-wire-token-estimate.d.ts +29 -0
  64. package/dist/hooks/magic-context/final-wire-token-estimate.d.ts.map +1 -0
  65. package/dist/hooks/magic-context/heuristic-cleanup.d.ts +1 -0
  66. package/dist/hooks/magic-context/heuristic-cleanup.d.ts.map +1 -1
  67. package/dist/hooks/magic-context/historian-prompt.generated.d.ts +1 -1
  68. package/dist/hooks/magic-context/historian-prompt.generated.d.ts.map +1 -1
  69. package/dist/hooks/magic-context/hook.d.ts +3 -0
  70. package/dist/hooks/magic-context/hook.d.ts.map +1 -1
  71. package/dist/hooks/magic-context/inject-compartments.d.ts +8 -9
  72. package/dist/hooks/magic-context/inject-compartments.d.ts.map +1 -1
  73. package/dist/hooks/magic-context/m0-token-breakdown.d.ts +5 -4
  74. package/dist/hooks/magic-context/m0-token-breakdown.d.ts.map +1 -1
  75. package/dist/hooks/magic-context/read-session-chunk.d.ts +17 -1
  76. package/dist/hooks/magic-context/read-session-chunk.d.ts.map +1 -1
  77. package/dist/hooks/magic-context/read-session-db.d.ts +2 -2
  78. package/dist/hooks/magic-context/read-session-raw.d.ts +32 -2
  79. package/dist/hooks/magic-context/read-session-raw.d.ts.map +1 -1
  80. package/dist/hooks/magic-context/shadow-sender.d.ts +300 -0
  81. package/dist/hooks/magic-context/shadow-sender.d.ts.map +1 -0
  82. package/dist/hooks/magic-context/tag-content-primitives.d.ts.map +1 -1
  83. package/dist/hooks/magic-context/tag-messages.d.ts +8 -0
  84. package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
  85. package/dist/hooks/magic-context/temporal-awareness.d.ts +3 -3
  86. package/dist/hooks/magic-context/transform-operations.d.ts +1 -1
  87. package/dist/hooks/magic-context/transform-operations.d.ts.map +1 -1
  88. package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +32 -0
  89. package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
  90. package/dist/hooks/magic-context/transform.d.ts +4 -1
  91. package/dist/hooks/magic-context/transform.d.ts.map +1 -1
  92. package/dist/index.d.ts.map +1 -1
  93. package/dist/index.js +5418 -972
  94. package/dist/plugin/hooks/create-session-hooks.d.ts +48 -175
  95. package/dist/plugin/hooks/create-session-hooks.d.ts.map +1 -1
  96. package/dist/plugin/rpc-handlers.d.ts +3 -0
  97. package/dist/plugin/rpc-handlers.d.ts.map +1 -1
  98. package/dist/shared/models-dev-cache.d.ts +5 -0
  99. package/dist/shared/models-dev-cache.d.ts.map +1 -1
  100. package/dist/shared/rpc-client.d.ts +3 -1
  101. package/dist/shared/rpc-client.d.ts.map +1 -1
  102. package/dist/shared/rpc-notifications.d.ts +11 -6
  103. package/dist/shared/rpc-notifications.d.ts.map +1 -1
  104. package/dist/shared/rpc-server.d.ts +2 -2
  105. package/dist/shared/rpc-server.d.ts.map +1 -1
  106. package/dist/shared/sqlite.d.ts.map +1 -1
  107. package/dist/shared/tag-transcript.d.ts +18 -0
  108. package/dist/shared/tag-transcript.d.ts.map +1 -1
  109. package/dist/shared/tui-config.d.ts +5 -3
  110. package/dist/shared/tui-config.d.ts.map +1 -1
  111. package/dist/tools/ctx-expand/constants.d.ts +1 -1
  112. package/dist/tools/ctx-expand/constants.d.ts.map +1 -1
  113. package/dist/tui/data/context-db.d.ts +9 -2
  114. package/dist/tui/data/context-db.d.ts.map +1 -1
  115. package/dist/tui/data/notification-socket.d.ts +4 -5
  116. package/dist/tui/data/notification-socket.d.ts.map +1 -1
  117. package/dist/tui-compiled/data/context-db.d.ts +9 -2
  118. package/dist/tui-compiled/data/context-db.d.ts.map +1 -1
  119. package/dist/tui-compiled/data/notification-socket.d.ts +4 -5
  120. package/dist/tui-compiled/data/notification-socket.d.ts.map +1 -1
  121. package/package.json +2 -1
  122. package/src/shared/models-dev-cache.test.ts +5 -0
  123. package/src/shared/models-dev-cache.ts +44 -6
  124. package/src/shared/opencode-compaction-detector.test.ts +11 -2
  125. package/src/shared/rpc-client.test.ts +213 -9
  126. package/src/shared/rpc-client.ts +48 -39
  127. package/src/shared/rpc-notifications.test.ts +26 -5
  128. package/src/shared/rpc-notifications.ts +44 -35
  129. package/src/shared/rpc-server.ts +77 -33
  130. package/src/shared/sqlite.ts +22 -3
  131. package/src/shared/tag-transcript.test.ts +253 -7
  132. package/src/shared/tag-transcript.ts +422 -114
  133. package/src/shared/tui-config.test.ts +4 -12
  134. package/src/shared/tui-config.ts +6 -6
  135. package/src/tui/data/context-db.test.ts +113 -0
  136. package/src/tui/data/context-db.ts +26 -19
  137. package/src/tui/data/notification-socket.test.ts +277 -11
  138. package/src/tui/data/notification-socket.ts +257 -86
  139. package/src/tui/index.tsx +33 -25
  140. package/src/tui/slots/sidebar-content.tsx +62 -24
  141. package/src/tui-compiled/data/context-db.ts +26 -19
  142. package/src/tui-compiled/data/notification-socket.ts +257 -86
  143. package/src/tui-compiled/index.tsx +41 -24
  144. package/src/tui-compiled/slots/sidebar-content.tsx +45 -23
@@ -83,6 +83,21 @@ export interface TagTranscriptOptions {
83
83
  * contentId) since all parts of a message share one fingerprint.
84
84
  */
85
85
  entryFingerprintByMessageId?: ReadonlyMap<string, string>;
86
+ /**
87
+ * Stable Pi message ids observed on a prior pass. Their immutable parts may
88
+ * reuse tag assignments while this pass still reapplies visible prefixes and
89
+ * rebuilds the complete set of messages affected by each tag.
90
+ */
91
+ reuseMessageIds?: ReadonlySet<string>;
92
+ /** Exact text/count pairs retained by Pi for safe lazy-token backfill reuse. */
93
+ textTokenCache?: Map<string, { text: string; tokenCount: number }>;
94
+ /** Exact tool-result text/count pairs retained under composite tag identity. */
95
+ toolTokenCache?: Map<string, { text: string; tokenCount: number }>;
96
+ /** Optional process-local benchmark callback; production callers omit it. */
97
+ onTiming?: (
98
+ phase: "identity" | "prefix" | "targets" | "tokenCounting",
99
+ elapsedMs: number,
100
+ ) => void;
86
101
  }
87
102
 
88
103
  export interface TagTranscriptResult {
@@ -134,8 +149,17 @@ interface ToolOccurrence {
134
149
  kind: "tool_use" | "tool_result";
135
150
  }
136
151
 
152
+ interface TagTranscriptTiming {
153
+ identity: number;
154
+ prefix: number;
155
+ targets: number;
156
+ tokenCounting: number;
157
+ }
158
+
137
159
  interface ToolAggregate {
138
160
  callId: string;
161
+ /** Every occurrence seen so far belongs to a previously tagged stable message. */
162
+ identityReusable: boolean;
139
163
  occurrences: ToolOccurrence[];
140
164
  /** Largest byteSize seen across occurrences — used as the tag size. */
141
165
  maxByteSize: number;
@@ -145,8 +169,8 @@ interface ToolAggregate {
145
169
  toolName: string | null;
146
170
  /** Input byte size from the invocation occurrence (for storage projection). */
147
171
  inputByteSize: number;
148
- /** Whether input_token_count has been persisted (from the tool_use occurrence). */
149
- inputTokenStored: boolean;
172
+ /** Persisted input-token count, or null for a legacy/unobserved invocation. */
173
+ inputTokenCount: number | null;
150
174
  }
151
175
 
152
176
  export function tagTranscript(
@@ -158,6 +182,9 @@ export function tagTranscript(
158
182
  ): TagTranscriptResult {
159
183
  const skipPrefixInjection = options.skipPrefixInjection === true;
160
184
  const targets = new Map<number, TagTarget>();
185
+ const timing: TagTranscriptTiming | undefined = options.onTiming
186
+ ? { identity: 0, prefix: 0, targets: 0, tokenCounting: 0 }
187
+ : undefined;
161
188
 
162
189
  // Tool aggregation is keyed by the same owner+callId identity used by
163
190
  // assignToolTag. OpenCode/Pi callId counters can repeat across turns, so
@@ -179,6 +206,8 @@ export function tagTranscript(
179
206
  if (message === undefined) continue;
180
207
  activeToolResultRun = undefined;
181
208
  const messageId = message.info.id;
209
+ const reuseIdentity =
210
+ messageId !== undefined && options.reuseMessageIds?.has(messageId) === true;
182
211
 
183
212
  let textOrdinal = 0;
184
213
  const parts = message.parts;
@@ -213,6 +242,9 @@ export function tagTranscript(
213
242
  targets,
214
243
  skipPrefixInjection,
215
244
  entryFingerprint: options.entryFingerprintByMessageId?.get(messageId) ?? null,
245
+ reuseIdentity,
246
+ timing,
247
+ textTokenCache: options.textTokenCache,
216
248
  });
217
249
  textOrdinal += 1;
218
250
  continue;
@@ -224,13 +256,8 @@ export function tagTranscript(
224
256
  continue;
225
257
  }
226
258
 
259
+ const identityStart = timing ? performance.now() : 0;
227
260
  const callId = part.id;
228
- const text = part.getText() ?? "";
229
- const toolByteSize = getToolPartByteSize(part, text);
230
- const toolTokenCount = getToolPartTokenCount(part, text);
231
- const meta = part.getToolMetadata();
232
- const inputTokenCount = meta.inputTokenCount;
233
-
234
261
  if (typeof callId !== "string" || callId.length === 0) {
235
262
  activeToolResultRun = undefined;
236
263
  // No stable callId to aggregate on. Tag independently.
@@ -245,6 +272,8 @@ export function tagTranscript(
245
272
  db,
246
273
  targets,
247
274
  skipPrefixInjection,
275
+ reuseIdentity,
276
+ timing,
248
277
  });
249
278
  continue;
250
279
  }
@@ -267,58 +296,92 @@ export function tagTranscript(
267
296
  const aggregateKey: string = existingKey ?? makeToolCompositeKey(messageId, callId);
268
297
  const existing = toolAggregates.get(aggregateKey);
269
298
  if (existing) {
270
- // Later occurrence for this owner+callId pair. Merge into the
271
- // aggregate, update byte accounting if larger, and rebuild the
272
- // TagTarget so drops mutate both invocation and result.
273
- existing.occurrences.push({
274
- message,
299
+ existing.occurrences.push({ message, part, kind: part.kind });
300
+ const canReuseIdentity = reuseIdentity && existing.identityReusable;
301
+ let text = "";
302
+ if (canReuseIdentity) {
303
+ // Prefix replay always needs result text. Byte length is also
304
+ // cheap enough to guard the durable growth invariant; BPE and
305
+ // DB writes remain staged behind an actual persisted-size bump.
306
+ if (part.kind === "tool_result") {
307
+ text = part.getText() ?? "";
308
+ applyGrownToolResultAccounting({
309
+ db,
310
+ sessionId,
311
+ tagger,
312
+ aggregate: existing,
313
+ byteSize: getToolPartByteSize(part, text),
314
+ part,
315
+ text,
316
+ timing,
317
+ tokenCache: options.toolTokenCache,
318
+ tokenCacheKey: aggregateKey,
319
+ });
320
+ }
321
+ if (timing) timing.identity += performance.now() - identityStart;
322
+ } else {
323
+ const accounting = readAggregateToolAccounting(
324
+ part,
325
+ timing,
326
+ options.toolTokenCache,
327
+ aggregateKey,
328
+ );
329
+ text = accounting.text;
330
+ if (part.kind === "tool_result") {
331
+ applyGrownToolResultAccounting({
332
+ db,
333
+ sessionId,
334
+ tagger,
335
+ aggregate: existing,
336
+ byteSize: accounting.byteSize,
337
+ part,
338
+ text,
339
+ timing,
340
+ knownTokenCount: accounting.tokenCount,
341
+ });
342
+ }
343
+ if (existing.toolName === null && accounting.toolName) {
344
+ existing.toolName = accounting.toolName;
345
+ }
346
+ if (
347
+ existing.inputByteSize === 0 &&
348
+ part.kind === "tool_use" &&
349
+ accounting.inputByteSize > 0
350
+ ) {
351
+ existing.inputByteSize = accounting.inputByteSize;
352
+ updateTagInputByteSize(
353
+ db,
354
+ sessionId,
355
+ existing.tagId,
356
+ accounting.inputByteSize,
357
+ );
358
+ }
359
+ if (
360
+ existing.inputTokenCount === null &&
361
+ part.kind === "tool_use" &&
362
+ accounting.inputTokenCount > 0
363
+ ) {
364
+ existing.inputTokenCount = accounting.inputTokenCount;
365
+ updateTagInputTokenCount(
366
+ db,
367
+ sessionId,
368
+ existing.tagId,
369
+ accounting.inputTokenCount,
370
+ );
371
+ }
372
+ syncToolAggregateAccounting(tagger, sessionId, existing);
373
+ if (timing) timing.identity += performance.now() - identityStart;
374
+ }
375
+ existing.identityReusable &&= reuseIdentity;
376
+ applyToolPrefixAndTarget({
377
+ skipPrefixInjection,
275
378
  part,
276
- kind: part.kind,
379
+ text,
380
+ tagId: existing.tagId,
381
+ aggregate: existing,
382
+ targets,
383
+ timing,
277
384
  });
278
- // byte_size tracks OUTPUT bytes only (the tool_result
279
- // occurrence). The invocation args are captured separately in
280
- // inputByteSize below; counting tool_use bytes here too would
281
- // double-count args in the emergency-drop reclaim formula
282
- // (byteSize + inputByteSize + reasoning). Mirrors OpenCode,
283
- // which assigns the tool tag on the result path.
284
- if (part.kind === "tool_result" && toolByteSize > existing.maxByteSize) {
285
- existing.maxByteSize = toolByteSize;
286
- existing.maxTokenCount = toolTokenCount;
287
- updateTagByteSize(db, sessionId, existing.tagId, toolByteSize);
288
- // Keep token_count in lockstep with the byte bump so the
289
- // grown output's tokens aren't undercounted by readers.
290
- updateTagTokenCount(db, sessionId, existing.tagId, toolTokenCount);
291
- }
292
- if (existing.toolName === null && meta.toolName) {
293
- existing.toolName = meta.toolName;
294
- }
295
- if (
296
- existing.inputByteSize === 0 &&
297
- part.kind === "tool_use" &&
298
- meta.inputByteSize > 0
299
- ) {
300
- existing.inputByteSize = meta.inputByteSize;
301
- updateTagInputByteSize(db, sessionId, existing.tagId, meta.inputByteSize);
302
- }
303
- if (
304
- !existing.inputTokenStored &&
305
- part.kind === "tool_use" &&
306
- inputTokenCount > 0
307
- ) {
308
- existing.inputTokenStored = true;
309
- updateTagInputTokenCount(db, sessionId, existing.tagId, inputTokenCount);
310
- }
311
- // Inject §N§ prefix into this tool_result occurrence
312
- // (matches OpenCode behavior — only result gets the prefix).
313
- if (!skipPrefixInjection && part.kind === "tool_result") {
314
- part.setText(prependTag(existing.tagId, text));
315
- }
316
- // Rebuild the aggregate target so it walks the now-
317
- // longer occurrences list.
318
- targets.set(
319
- existing.tagId,
320
- buildAggregateTarget(existing.tagId, existing.occurrences),
321
- );
322
385
  if (part.kind === "tool_result") {
323
386
  markToolAggregateResolved(
324
387
  callId,
@@ -327,18 +390,61 @@ export function tagTranscript(
327
390
  );
328
391
  activeToolResultRun = { callId, aggregateKey };
329
392
  }
393
+ continue;
394
+ }
395
+
396
+ const reusableTagId = reuseIdentity
397
+ ? tagger.getToolTag(sessionId, callId, messageId)
398
+ : undefined;
399
+ const reusableAccounting = reuseIdentity
400
+ ? tagger.getToolTagAccounting(sessionId, callId, messageId)
401
+ : undefined;
402
+ let aggregate: ToolAggregate & { tagId: number };
403
+ let text = "";
404
+ if (reusableTagId !== undefined && reusableAccounting !== undefined) {
405
+ aggregate = {
406
+ callId,
407
+ tagId: reusableTagId,
408
+ identityReusable: true,
409
+ occurrences: [{ message, part, kind: part.kind }],
410
+ // Stable identity does not imply stable payload size. Seed from
411
+ // the persisted row so unchanged results avoid BPE while a
412
+ // genuinely larger folded result can still bump accounting.
413
+ maxByteSize: reusableAccounting.byteSize,
414
+ maxTokenCount: reusableAccounting.tokenCount ?? 0,
415
+ toolName: null,
416
+ inputByteSize: reusableAccounting.inputByteSize,
417
+ inputTokenCount: reusableAccounting.inputTokenCount,
418
+ };
419
+ if (part.kind === "tool_result") {
420
+ text = part.getText() ?? "";
421
+ applyGrownToolResultAccounting({
422
+ db,
423
+ sessionId,
424
+ tagger,
425
+ aggregate,
426
+ byteSize: getToolPartByteSize(part, text),
427
+ part,
428
+ text,
429
+ timing,
430
+ tokenCache: options.toolTokenCache,
431
+ tokenCacheKey: aggregateKey,
432
+ });
433
+ }
434
+ if (timing) timing.identity += performance.now() - identityStart;
330
435
  } else {
331
- // First occurrence for this owner+callId identity — reserve
332
- // the tag number. Owner stays stable across passes because
333
- // transcript message ids are durable.
334
- // byte_size is OUTPUT-only (0 until the tool_result occurrence
335
- // is seen); the invocation args live in inputByteSize. This
336
- // keeps the emergency-drop reclaim formula (byteSize +
337
- // inputByteSize + reasoning) from double-counting args when the
338
- // first occurrence is a large tool_use. Mirrors OpenCode.
339
- const outputByteSize = part.kind === "tool_result" ? toolByteSize : 0;
340
- const outputTokenCount = part.kind === "tool_result" ? toolTokenCount : 0;
341
- const firstInputTokenCount = part.kind === "tool_use" ? inputTokenCount : 0;
436
+ const accounting = readAggregateToolAccounting(
437
+ part,
438
+ timing,
439
+ options.toolTokenCache,
440
+ aggregateKey,
441
+ );
442
+ text = accounting.text;
443
+ const outputByteSize = part.kind === "tool_result" ? accounting.byteSize : 0;
444
+ const outputTokenCount =
445
+ part.kind === "tool_result" ? accounting.tokenCount : 0;
446
+ const firstInputTokenCount =
447
+ part.kind === "tool_use" ? accounting.inputTokenCount : 0;
342
448
  const tagId = tagger.assignToolTag(
343
449
  sessionId,
344
450
  callId,
@@ -346,58 +452,188 @@ export function tagTranscript(
346
452
  outputByteSize,
347
453
  db,
348
454
  0,
349
- meta.toolName ?? null,
350
- meta.inputByteSize,
455
+ accounting.toolName,
456
+ accounting.inputByteSize,
351
457
  () => ({
352
458
  tokenCount: outputTokenCount,
353
459
  inputTokenCount: firstInputTokenCount,
354
460
  reasoningTokenCount: null,
355
461
  }),
356
462
  );
357
- const aggregate = {
463
+ const persistedAccounting = tagger.getToolTagAccounting(
464
+ sessionId,
465
+ callId,
466
+ messageId,
467
+ );
468
+ aggregate = {
358
469
  callId,
359
470
  tagId,
360
- occurrences: [
361
- {
362
- message,
363
- part,
364
- kind: part.kind,
365
- },
366
- ],
367
- maxByteSize: outputByteSize,
368
- maxTokenCount: outputTokenCount,
369
- toolName: meta.toolName ?? null,
370
- inputByteSize: part.kind === "tool_use" ? meta.inputByteSize : 0,
371
- inputTokenStored: part.kind === "tool_use" && firstInputTokenCount > 0,
471
+ identityReusable: false,
472
+ occurrences: [{ message, part, kind: part.kind }],
473
+ maxByteSize: persistedAccounting?.byteSize ?? outputByteSize,
474
+ maxTokenCount: persistedAccounting?.tokenCount ?? outputTokenCount,
475
+ toolName: accounting.toolName,
476
+ inputByteSize:
477
+ persistedAccounting?.inputByteSize ??
478
+ (part.kind === "tool_use" ? accounting.inputByteSize : 0),
479
+ inputTokenCount:
480
+ persistedAccounting?.inputTokenCount ??
481
+ (part.kind === "tool_use" ? firstInputTokenCount : null),
372
482
  };
373
- toolAggregates.set(aggregateKey, aggregate);
374
- if (part.kind === "tool_use") {
375
- openToolAggregateKeysByCallId.set(callId, [...pendingKeys, aggregateKey]);
376
- }
377
- // Inject §N§ prefix into this occurrence's visible text
378
- // when it's a tool_result. (OpenCode parity: prefix
379
- // only goes on the result, not the invocation.)
380
- if (!skipPrefixInjection && part.kind === "tool_result") {
381
- part.setText(prependTag(tagId, text));
382
- }
383
- targets.set(tagId, buildAggregateTarget(tagId, aggregate.occurrences));
384
483
  if (part.kind === "tool_result") {
385
- markToolAggregateResolved(
386
- callId,
387
- aggregateKey,
388
- openToolAggregateKeysByCallId,
389
- );
390
- activeToolResultRun = { callId, aggregateKey };
484
+ applyGrownToolResultAccounting({
485
+ db,
486
+ sessionId,
487
+ tagger,
488
+ aggregate,
489
+ byteSize: accounting.byteSize,
490
+ part,
491
+ text,
492
+ timing,
493
+ knownTokenCount: accounting.tokenCount,
494
+ });
391
495
  }
496
+ syncToolAggregateAccounting(tagger, sessionId, aggregate);
497
+ if (timing) timing.identity += performance.now() - identityStart;
498
+ }
499
+
500
+ toolAggregates.set(aggregateKey, aggregate);
501
+ if (part.kind === "tool_use") {
502
+ openToolAggregateKeysByCallId.set(callId, [...pendingKeys, aggregateKey]);
503
+ }
504
+ applyToolPrefixAndTarget({
505
+ skipPrefixInjection,
506
+ part,
507
+ text,
508
+ tagId: aggregate.tagId,
509
+ aggregate,
510
+ targets,
511
+ timing,
512
+ });
513
+ if (part.kind === "tool_result") {
514
+ markToolAggregateResolved(callId, aggregateKey, openToolAggregateKeysByCallId);
515
+ activeToolResultRun = { callId, aggregateKey };
392
516
  }
393
517
  }
394
518
  // thinking, image, file, structural, unknown → skip.
395
519
  }
396
520
  }
397
521
 
522
+ if (timing && options.onTiming) {
523
+ options.onTiming("identity", timing.identity);
524
+ options.onTiming("prefix", timing.prefix);
525
+ options.onTiming("targets", timing.targets);
526
+ options.onTiming("tokenCounting", timing.tokenCounting);
527
+ }
398
528
  return { targets };
399
529
  }
400
530
 
531
+ interface AggregateToolAccounting {
532
+ text: string;
533
+ byteSize: number;
534
+ tokenCount: number;
535
+ toolName: string | null;
536
+ inputByteSize: number;
537
+ inputTokenCount: number;
538
+ }
539
+
540
+ interface GrownToolResultAccountingArgs {
541
+ db: ContextDatabase;
542
+ sessionId: string;
543
+ tagger: Tagger;
544
+ aggregate: ToolAggregate & { tagId: number };
545
+ byteSize: number;
546
+ part: TranscriptPart;
547
+ text: string;
548
+ timing: TagTranscriptTiming | undefined;
549
+ tokenCache?: Map<string, { text: string; tokenCount: number }>;
550
+ tokenCacheKey?: string;
551
+ knownTokenCount?: number;
552
+ }
553
+
554
+ function syncToolAggregateAccounting(
555
+ tagger: Tagger,
556
+ sessionId: string,
557
+ aggregate: ToolAggregate & { tagId: number },
558
+ ): void {
559
+ tagger.setToolTagAccounting(sessionId, aggregate.tagId, {
560
+ byteSize: aggregate.maxByteSize,
561
+ tokenCount: aggregate.maxTokenCount,
562
+ inputByteSize: aggregate.inputByteSize,
563
+ inputTokenCount: aggregate.inputTokenCount,
564
+ });
565
+ }
566
+
567
+ function applyGrownToolResultAccounting(args: GrownToolResultAccountingArgs): void {
568
+ if (args.byteSize <= args.aggregate.maxByteSize) return;
569
+
570
+ let tokenCount = args.knownTokenCount;
571
+ if (tokenCount !== undefined && args.tokenCacheKey) {
572
+ args.tokenCache?.set(args.tokenCacheKey, { text: args.text, tokenCount });
573
+ }
574
+ if (tokenCount === undefined) {
575
+ const tokenStart = args.timing ? performance.now() : 0;
576
+ tokenCount = getCachedToolPartTokenCount(
577
+ args.part,
578
+ args.text,
579
+ args.tokenCache,
580
+ args.tokenCacheKey,
581
+ );
582
+ if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
583
+ }
584
+ args.aggregate.maxByteSize = args.byteSize;
585
+ args.aggregate.maxTokenCount = tokenCount;
586
+ updateTagByteSize(args.db, args.sessionId, args.aggregate.tagId, args.byteSize);
587
+ updateTagTokenCount(args.db, args.sessionId, args.aggregate.tagId, tokenCount);
588
+ syncToolAggregateAccounting(args.tagger, args.sessionId, args.aggregate);
589
+ }
590
+
591
+ function readAggregateToolAccounting(
592
+ part: TranscriptPart,
593
+ timing: TagTranscriptTiming | undefined,
594
+ tokenCache?: Map<string, { text: string; tokenCount: number }>,
595
+ tokenCacheKey?: string,
596
+ ): AggregateToolAccounting {
597
+ const text = part.getText() ?? "";
598
+ const byteSize = getToolPartByteSize(part, text);
599
+ const metadata = part.getToolMetadata();
600
+ let tokenCount = 0;
601
+ if (part.kind === "tool_result") {
602
+ const tokenStart = timing ? performance.now() : 0;
603
+ tokenCount = getCachedToolPartTokenCount(part, text, tokenCache, tokenCacheKey);
604
+ if (timing) timing.tokenCounting += performance.now() - tokenStart;
605
+ }
606
+ return {
607
+ text,
608
+ byteSize,
609
+ tokenCount,
610
+ toolName: metadata.toolName ?? null,
611
+ inputByteSize: metadata.inputByteSize,
612
+ inputTokenCount: metadata.inputTokenCount,
613
+ };
614
+ }
615
+
616
+ interface ApplyToolPrefixAndTargetArgs {
617
+ skipPrefixInjection: boolean;
618
+ part: TranscriptPart;
619
+ text: string;
620
+ tagId: number;
621
+ aggregate: ToolAggregate;
622
+ targets: Map<number, TagTarget>;
623
+ timing: TagTranscriptTiming | undefined;
624
+ }
625
+
626
+ function applyToolPrefixAndTarget(args: ApplyToolPrefixAndTargetArgs): void {
627
+ if (!args.skipPrefixInjection && args.part.kind === "tool_result") {
628
+ const prefixStart = args.timing ? performance.now() : 0;
629
+ args.part.setText(prependTag(args.tagId, args.text));
630
+ if (args.timing) args.timing.prefix += performance.now() - prefixStart;
631
+ }
632
+ const targetStart = args.timing ? performance.now() : 0;
633
+ args.targets.set(args.tagId, buildAggregateTarget(args.tagId, args.aggregate.occurrences));
634
+ if (args.timing) args.timing.targets += performance.now() - targetStart;
635
+ }
636
+
401
637
  function findLastUnresolvedToolAggregateKey(
402
638
  pendingKeys: string[],
403
639
  toolAggregates: Map<string, ToolAggregate & { tagId: number }>,
@@ -466,6 +702,19 @@ function getToolPartTokenCount(part: TranscriptPart, text: string): number {
466
702
  return 0;
467
703
  }
468
704
 
705
+ function getCachedToolPartTokenCount(
706
+ part: TranscriptPart,
707
+ text: string,
708
+ cache?: Map<string, { text: string; tokenCount: number }>,
709
+ cacheKey?: string,
710
+ ): number {
711
+ const cached = cacheKey ? cache?.get(cacheKey) : undefined;
712
+ if (cached?.text === text) return cached.tokenCount;
713
+ const tokenCount = getToolPartTokenCount(part, text);
714
+ if (cacheKey) cache?.set(cacheKey, { text, tokenCount });
715
+ return tokenCount;
716
+ }
717
+
469
718
  function getNonTextToolResultByteSize(part: TranscriptPart): number {
470
719
  // Prefer the adapter's exact raw-payload size when available (Pi's
471
720
  // tool_result proxy can serialize the real content array, incl. images).
@@ -508,11 +757,23 @@ interface TagTextPartArgs {
508
757
  targets: Map<number, TagTarget>;
509
758
  skipPrefixInjection: boolean;
510
759
  entryFingerprint: string | null;
760
+ reuseIdentity: boolean;
761
+ timing?: TagTranscriptTiming;
762
+ textTokenCache?: Map<string, { text: string; tokenCount: number }>;
511
763
  }
512
764
 
513
765
  function tagTextPart(args: TagTextPartArgs): void {
766
+ const identityStart = args.timing ? performance.now() : 0;
514
767
  const text = args.part.getText() ?? "";
515
768
  const contentId = `${args.messageId}:p${args.textOrdinal}`;
769
+ const reusableTagId = args.reuseIdentity
770
+ ? args.tagger.getTag(args.sessionId, contentId, "message")
771
+ : undefined;
772
+ if (reusableTagId !== undefined) {
773
+ if (args.timing) args.timing.identity += performance.now() - identityStart;
774
+ applyTextPrefixAndTarget(args, reusableTagId, text);
775
+ return;
776
+ }
516
777
  const tagId = args.tagger.assignTag(
517
778
  args.sessionId,
518
779
  contentId,
@@ -525,11 +786,24 @@ function tagTextPart(args: TagTextPartArgs): void {
525
786
  args.entryFingerprint,
526
787
  // Lazy: fires only on fresh insert. Strip any §N§ prefix so a re-tag
527
788
  // from already-prefixed text still tokenizes the pristine content.
528
- () => ({
529
- tokenCount: estimateTagTextTokens(stripTagPrefix(text)),
530
- inputTokenCount: null,
531
- reasoningTokenCount: null,
532
- }),
789
+ () => {
790
+ const tokenStart = args.timing ? performance.now() : 0;
791
+ const cached = args.textTokenCache?.get(contentId);
792
+ const tokenCount =
793
+ cached?.text === text
794
+ ? cached.tokenCount
795
+ : estimateTagTextTokens(stripTagPrefix(text));
796
+ if (cached?.text !== text) {
797
+ args.textTokenCache?.set(contentId, { text, tokenCount });
798
+ }
799
+ const counts = {
800
+ tokenCount,
801
+ inputTokenCount: null,
802
+ reasoningTokenCount: null,
803
+ };
804
+ if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
805
+ return counts;
806
+ },
533
807
  );
534
808
 
535
809
  // Persist the original (pre-tagged) source content so caveman
@@ -547,12 +821,20 @@ function tagTextPart(args: TagTextPartArgs): void {
547
821
  if (sourceContent.trim().length > 0) {
548
822
  saveSourceContent(args.db, args.sessionId, tagId, sourceContent);
549
823
  }
824
+ if (args.timing) args.timing.identity += performance.now() - identityStart;
825
+ applyTextPrefixAndTarget(args, tagId, text);
826
+ }
550
827
 
828
+ function applyTextPrefixAndTarget(args: TagTextPartArgs, tagId: number, text: string): void {
551
829
  if (!args.skipPrefixInjection) {
830
+ const prefixStart = args.timing ? performance.now() : 0;
552
831
  args.part.setText(prependTag(tagId, text));
832
+ if (args.timing) args.timing.prefix += performance.now() - prefixStart;
553
833
  }
554
834
 
835
+ const targetStart = args.timing ? performance.now() : 0;
555
836
  args.targets.set(tagId, buildTextTarget(args.part, args.message));
837
+ if (args.timing) args.timing.targets += performance.now() - targetStart;
556
838
  }
557
839
 
558
840
  interface TagToolPartArgs {
@@ -566,9 +848,12 @@ interface TagToolPartArgs {
566
848
  db: ContextDatabase;
567
849
  targets: Map<number, TagTarget>;
568
850
  skipPrefixInjection: boolean;
851
+ reuseIdentity: boolean;
852
+ timing?: TagTranscriptTiming;
569
853
  }
570
854
 
571
855
  function tagToolPart(args: TagToolPartArgs): void {
856
+ const identityStart = args.timing ? performance.now() : 0;
572
857
  // Prefer the part's stable id (tool call id from Pi/OpenCode); fall
573
858
  // back to a synthetic locator. Tool calls and their results MAY
574
859
  // share an id (Pi sets toolCallId on ToolResultMessage to match the
@@ -577,10 +862,21 @@ function tagToolPart(args: TagToolPartArgs): void {
577
862
  // target the call-id pair as a unit.
578
863
  const stableId = args.part.id;
579
864
  const contentId = stableId ?? `${args.messageId}:t${args.partIndex}`;
865
+ const reusableTagId = args.reuseIdentity
866
+ ? args.tagger.getToolTag(args.sessionId, contentId, contentId)
867
+ : undefined;
868
+ if (reusableTagId !== undefined) {
869
+ const text = args.part.kind === "tool_result" ? (args.part.getText() ?? "") : "";
870
+ if (args.timing) args.timing.identity += performance.now() - identityStart;
871
+ applySingleToolPrefixAndTarget(args, reusableTagId, text);
872
+ return;
873
+ }
580
874
  const text = args.part.getText() ?? "";
581
875
  const toolByteSize = getToolPartByteSize(args.part, text);
582
- const toolTokenCount = getToolPartTokenCount(args.part, text);
583
876
  const meta = args.part.getToolMetadata();
877
+ const tokenStart = args.timing ? performance.now() : 0;
878
+ const toolTokenCount = getToolPartTokenCount(args.part, text);
879
+ if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
584
880
  // v3.3.1 Layer C: synthetic ownership for the no-callId Pi
585
881
  // fallback. Owner == callId == contentId. The composite key
586
882
  // collapses to a unique synthetic identifier per part, preserving
@@ -596,22 +892,34 @@ function tagToolPart(args: TagToolPartArgs): void {
596
892
  0,
597
893
  meta.toolName ?? null,
598
894
  meta.inputByteSize,
599
- () => ({
600
- tokenCount: toolTokenCount,
601
- inputTokenCount: meta.inputTokenCount,
602
- reasoningTokenCount: null,
603
- }),
895
+ () => {
896
+ const tokenStart = args.timing ? performance.now() : 0;
897
+ const counts = {
898
+ tokenCount: toolTokenCount,
899
+ inputTokenCount: meta.inputTokenCount,
900
+ reasoningTokenCount: null,
901
+ };
902
+ if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
903
+ return counts;
904
+ },
604
905
  );
906
+ if (args.timing) args.timing.identity += performance.now() - identityStart;
907
+ applySingleToolPrefixAndTarget(args, tagId, text);
908
+ }
605
909
 
910
+ function applySingleToolPrefixAndTarget(args: TagToolPartArgs, tagId: number, text: string): void {
606
911
  // For tool parts, the visible payload is the tool result text. We
607
912
  // can inject the tag prefix into it for in-text references; this
608
913
  // matches the OpenCode behavior of tagging tool outputs.
609
914
  if (!args.skipPrefixInjection && args.part.kind === "tool_result") {
610
- const tagged = prependTag(tagId, text);
611
- args.part.setText(tagged);
915
+ const prefixStart = args.timing ? performance.now() : 0;
916
+ args.part.setText(prependTag(tagId, text));
917
+ if (args.timing) args.timing.prefix += performance.now() - prefixStart;
612
918
  }
613
919
 
920
+ const targetStart = args.timing ? performance.now() : 0;
614
921
  args.targets.set(tagId, buildToolTarget(args.part, args.message, tagId));
922
+ if (args.timing) args.timing.targets += performance.now() - targetStart;
615
923
  }
616
924
 
617
925
  function setToolContentOrText(part: TranscriptPart, content: string): boolean {