@cortexkit/opencode-magic-context 0.31.4 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/magic-context-prompt.d.ts.map +1 -1
- package/dist/config/project-security.d.ts +2 -0
- package/dist/config/project-security.d.ts.map +1 -1
- package/dist/config/schema/magic-context.d.ts +9 -0
- package/dist/config/schema/magic-context.d.ts.map +1 -1
- package/dist/features/magic-context/message-index-async.d.ts +2 -1
- package/dist/features/magic-context/message-index-async.d.ts.map +1 -1
- package/dist/features/magic-context/storage-clone.d.ts +34 -0
- package/dist/features/magic-context/storage-clone.d.ts.map +1 -0
- package/dist/features/magic-context/storage-tags.d.ts +11 -0
- package/dist/features/magic-context/storage-tags.d.ts.map +1 -1
- package/dist/features/magic-context/storage.d.ts +2 -1
- package/dist/features/magic-context/storage.d.ts.map +1 -1
- package/dist/features/magic-context/tagger.d.ts +10 -0
- package/dist/features/magic-context/tagger.d.ts.map +1 -1
- package/dist/hooks/magic-context/apply-operations.d.ts +24 -0
- package/dist/hooks/magic-context/apply-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts +7 -1
- package/dist/hooks/magic-context/compartment-trigger.d.ts.map +1 -1
- package/dist/hooks/magic-context/decay-render.d.ts +2 -0
- package/dist/hooks/magic-context/decay-render.d.ts.map +1 -1
- package/dist/hooks/magic-context/hook.d.ts +3 -0
- package/dist/hooks/magic-context/hook.d.ts.map +1 -1
- package/dist/hooks/magic-context/inject-compartments.d.ts +7 -9
- package/dist/hooks/magic-context/inject-compartments.d.ts.map +1 -1
- package/dist/hooks/magic-context/m0-token-breakdown.d.ts +5 -4
- package/dist/hooks/magic-context/m0-token-breakdown.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-chunk.d.ts +2 -0
- package/dist/hooks/magic-context/read-session-chunk.d.ts.map +1 -1
- package/dist/hooks/magic-context/read-session-db.d.ts +2 -2
- package/dist/hooks/magic-context/read-session-raw.d.ts +8 -0
- package/dist/hooks/magic-context/read-session-raw.d.ts.map +1 -1
- package/dist/hooks/magic-context/shadow-sender.d.ts +262 -0
- package/dist/hooks/magic-context/shadow-sender.d.ts.map +1 -0
- package/dist/hooks/magic-context/tag-content-primitives.d.ts.map +1 -1
- package/dist/hooks/magic-context/tag-messages.d.ts +8 -0
- package/dist/hooks/magic-context/tag-messages.d.ts.map +1 -1
- package/dist/hooks/magic-context/temporal-awareness.d.ts +3 -3
- package/dist/hooks/magic-context/transform-operations.d.ts +1 -1
- package/dist/hooks/magic-context/transform-operations.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts +5 -0
- package/dist/hooks/magic-context/transform-postprocess-phase.d.ts.map +1 -1
- package/dist/hooks/magic-context/transform.d.ts +4 -1
- package/dist/hooks/magic-context/transform.d.ts.map +1 -1
- package/dist/index.js +1665 -140
- package/dist/plugin/hooks/create-session-hooks.d.ts +48 -175
- package/dist/plugin/hooks/create-session-hooks.d.ts.map +1 -1
- package/dist/shared/rpc-server.d.ts.map +1 -1
- package/dist/shared/tag-transcript.d.ts +18 -0
- package/dist/shared/tag-transcript.d.ts.map +1 -1
- package/dist/tools/ctx-expand/constants.d.ts +1 -1
- package/dist/tools/ctx-expand/constants.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/shared/rpc-server.ts +8 -0
- package/src/shared/tag-transcript.test.ts +253 -7
- package/src/shared/tag-transcript.ts +422 -114
|
@@ -83,6 +83,21 @@ export interface TagTranscriptOptions {
|
|
|
83
83
|
* contentId) since all parts of a message share one fingerprint.
|
|
84
84
|
*/
|
|
85
85
|
entryFingerprintByMessageId?: ReadonlyMap<string, string>;
|
|
86
|
+
/**
|
|
87
|
+
* Stable Pi message ids observed on a prior pass. Their immutable parts may
|
|
88
|
+
* reuse tag assignments while this pass still reapplies visible prefixes and
|
|
89
|
+
* rebuilds the complete set of messages affected by each tag.
|
|
90
|
+
*/
|
|
91
|
+
reuseMessageIds?: ReadonlySet<string>;
|
|
92
|
+
/** Exact text/count pairs retained by Pi for safe lazy-token backfill reuse. */
|
|
93
|
+
textTokenCache?: Map<string, { text: string; tokenCount: number }>;
|
|
94
|
+
/** Exact tool-result text/count pairs retained under composite tag identity. */
|
|
95
|
+
toolTokenCache?: Map<string, { text: string; tokenCount: number }>;
|
|
96
|
+
/** Optional process-local benchmark callback; production callers omit it. */
|
|
97
|
+
onTiming?: (
|
|
98
|
+
phase: "identity" | "prefix" | "targets" | "tokenCounting",
|
|
99
|
+
elapsedMs: number,
|
|
100
|
+
) => void;
|
|
86
101
|
}
|
|
87
102
|
|
|
88
103
|
export interface TagTranscriptResult {
|
|
@@ -134,8 +149,17 @@ interface ToolOccurrence {
|
|
|
134
149
|
kind: "tool_use" | "tool_result";
|
|
135
150
|
}
|
|
136
151
|
|
|
152
|
+
interface TagTranscriptTiming {
|
|
153
|
+
identity: number;
|
|
154
|
+
prefix: number;
|
|
155
|
+
targets: number;
|
|
156
|
+
tokenCounting: number;
|
|
157
|
+
}
|
|
158
|
+
|
|
137
159
|
interface ToolAggregate {
|
|
138
160
|
callId: string;
|
|
161
|
+
/** Every occurrence seen so far belongs to a previously tagged stable message. */
|
|
162
|
+
identityReusable: boolean;
|
|
139
163
|
occurrences: ToolOccurrence[];
|
|
140
164
|
/** Largest byteSize seen across occurrences — used as the tag size. */
|
|
141
165
|
maxByteSize: number;
|
|
@@ -145,8 +169,8 @@ interface ToolAggregate {
|
|
|
145
169
|
toolName: string | null;
|
|
146
170
|
/** Input byte size from the invocation occurrence (for storage projection). */
|
|
147
171
|
inputByteSize: number;
|
|
148
|
-
/**
|
|
149
|
-
|
|
172
|
+
/** Persisted input-token count, or null for a legacy/unobserved invocation. */
|
|
173
|
+
inputTokenCount: number | null;
|
|
150
174
|
}
|
|
151
175
|
|
|
152
176
|
export function tagTranscript(
|
|
@@ -158,6 +182,9 @@ export function tagTranscript(
|
|
|
158
182
|
): TagTranscriptResult {
|
|
159
183
|
const skipPrefixInjection = options.skipPrefixInjection === true;
|
|
160
184
|
const targets = new Map<number, TagTarget>();
|
|
185
|
+
const timing: TagTranscriptTiming | undefined = options.onTiming
|
|
186
|
+
? { identity: 0, prefix: 0, targets: 0, tokenCounting: 0 }
|
|
187
|
+
: undefined;
|
|
161
188
|
|
|
162
189
|
// Tool aggregation is keyed by the same owner+callId identity used by
|
|
163
190
|
// assignToolTag. OpenCode/Pi callId counters can repeat across turns, so
|
|
@@ -179,6 +206,8 @@ export function tagTranscript(
|
|
|
179
206
|
if (message === undefined) continue;
|
|
180
207
|
activeToolResultRun = undefined;
|
|
181
208
|
const messageId = message.info.id;
|
|
209
|
+
const reuseIdentity =
|
|
210
|
+
messageId !== undefined && options.reuseMessageIds?.has(messageId) === true;
|
|
182
211
|
|
|
183
212
|
let textOrdinal = 0;
|
|
184
213
|
const parts = message.parts;
|
|
@@ -213,6 +242,9 @@ export function tagTranscript(
|
|
|
213
242
|
targets,
|
|
214
243
|
skipPrefixInjection,
|
|
215
244
|
entryFingerprint: options.entryFingerprintByMessageId?.get(messageId) ?? null,
|
|
245
|
+
reuseIdentity,
|
|
246
|
+
timing,
|
|
247
|
+
textTokenCache: options.textTokenCache,
|
|
216
248
|
});
|
|
217
249
|
textOrdinal += 1;
|
|
218
250
|
continue;
|
|
@@ -224,13 +256,8 @@ export function tagTranscript(
|
|
|
224
256
|
continue;
|
|
225
257
|
}
|
|
226
258
|
|
|
259
|
+
const identityStart = timing ? performance.now() : 0;
|
|
227
260
|
const callId = part.id;
|
|
228
|
-
const text = part.getText() ?? "";
|
|
229
|
-
const toolByteSize = getToolPartByteSize(part, text);
|
|
230
|
-
const toolTokenCount = getToolPartTokenCount(part, text);
|
|
231
|
-
const meta = part.getToolMetadata();
|
|
232
|
-
const inputTokenCount = meta.inputTokenCount;
|
|
233
|
-
|
|
234
261
|
if (typeof callId !== "string" || callId.length === 0) {
|
|
235
262
|
activeToolResultRun = undefined;
|
|
236
263
|
// No stable callId to aggregate on. Tag independently.
|
|
@@ -245,6 +272,8 @@ export function tagTranscript(
|
|
|
245
272
|
db,
|
|
246
273
|
targets,
|
|
247
274
|
skipPrefixInjection,
|
|
275
|
+
reuseIdentity,
|
|
276
|
+
timing,
|
|
248
277
|
});
|
|
249
278
|
continue;
|
|
250
279
|
}
|
|
@@ -267,58 +296,92 @@ export function tagTranscript(
|
|
|
267
296
|
const aggregateKey: string = existingKey ?? makeToolCompositeKey(messageId, callId);
|
|
268
297
|
const existing = toolAggregates.get(aggregateKey);
|
|
269
298
|
if (existing) {
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
299
|
+
existing.occurrences.push({ message, part, kind: part.kind });
|
|
300
|
+
const canReuseIdentity = reuseIdentity && existing.identityReusable;
|
|
301
|
+
let text = "";
|
|
302
|
+
if (canReuseIdentity) {
|
|
303
|
+
// Prefix replay always needs result text. Byte length is also
|
|
304
|
+
// cheap enough to guard the durable growth invariant; BPE and
|
|
305
|
+
// DB writes remain staged behind an actual persisted-size bump.
|
|
306
|
+
if (part.kind === "tool_result") {
|
|
307
|
+
text = part.getText() ?? "";
|
|
308
|
+
applyGrownToolResultAccounting({
|
|
309
|
+
db,
|
|
310
|
+
sessionId,
|
|
311
|
+
tagger,
|
|
312
|
+
aggregate: existing,
|
|
313
|
+
byteSize: getToolPartByteSize(part, text),
|
|
314
|
+
part,
|
|
315
|
+
text,
|
|
316
|
+
timing,
|
|
317
|
+
tokenCache: options.toolTokenCache,
|
|
318
|
+
tokenCacheKey: aggregateKey,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
if (timing) timing.identity += performance.now() - identityStart;
|
|
322
|
+
} else {
|
|
323
|
+
const accounting = readAggregateToolAccounting(
|
|
324
|
+
part,
|
|
325
|
+
timing,
|
|
326
|
+
options.toolTokenCache,
|
|
327
|
+
aggregateKey,
|
|
328
|
+
);
|
|
329
|
+
text = accounting.text;
|
|
330
|
+
if (part.kind === "tool_result") {
|
|
331
|
+
applyGrownToolResultAccounting({
|
|
332
|
+
db,
|
|
333
|
+
sessionId,
|
|
334
|
+
tagger,
|
|
335
|
+
aggregate: existing,
|
|
336
|
+
byteSize: accounting.byteSize,
|
|
337
|
+
part,
|
|
338
|
+
text,
|
|
339
|
+
timing,
|
|
340
|
+
knownTokenCount: accounting.tokenCount,
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
if (existing.toolName === null && accounting.toolName) {
|
|
344
|
+
existing.toolName = accounting.toolName;
|
|
345
|
+
}
|
|
346
|
+
if (
|
|
347
|
+
existing.inputByteSize === 0 &&
|
|
348
|
+
part.kind === "tool_use" &&
|
|
349
|
+
accounting.inputByteSize > 0
|
|
350
|
+
) {
|
|
351
|
+
existing.inputByteSize = accounting.inputByteSize;
|
|
352
|
+
updateTagInputByteSize(
|
|
353
|
+
db,
|
|
354
|
+
sessionId,
|
|
355
|
+
existing.tagId,
|
|
356
|
+
accounting.inputByteSize,
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
if (
|
|
360
|
+
existing.inputTokenCount === null &&
|
|
361
|
+
part.kind === "tool_use" &&
|
|
362
|
+
accounting.inputTokenCount > 0
|
|
363
|
+
) {
|
|
364
|
+
existing.inputTokenCount = accounting.inputTokenCount;
|
|
365
|
+
updateTagInputTokenCount(
|
|
366
|
+
db,
|
|
367
|
+
sessionId,
|
|
368
|
+
existing.tagId,
|
|
369
|
+
accounting.inputTokenCount,
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
syncToolAggregateAccounting(tagger, sessionId, existing);
|
|
373
|
+
if (timing) timing.identity += performance.now() - identityStart;
|
|
374
|
+
}
|
|
375
|
+
existing.identityReusable &&= reuseIdentity;
|
|
376
|
+
applyToolPrefixAndTarget({
|
|
377
|
+
skipPrefixInjection,
|
|
275
378
|
part,
|
|
276
|
-
|
|
379
|
+
text,
|
|
380
|
+
tagId: existing.tagId,
|
|
381
|
+
aggregate: existing,
|
|
382
|
+
targets,
|
|
383
|
+
timing,
|
|
277
384
|
});
|
|
278
|
-
// byte_size tracks OUTPUT bytes only (the tool_result
|
|
279
|
-
// occurrence). The invocation args are captured separately in
|
|
280
|
-
// inputByteSize below; counting tool_use bytes here too would
|
|
281
|
-
// double-count args in the emergency-drop reclaim formula
|
|
282
|
-
// (byteSize + inputByteSize + reasoning). Mirrors OpenCode,
|
|
283
|
-
// which assigns the tool tag on the result path.
|
|
284
|
-
if (part.kind === "tool_result" && toolByteSize > existing.maxByteSize) {
|
|
285
|
-
existing.maxByteSize = toolByteSize;
|
|
286
|
-
existing.maxTokenCount = toolTokenCount;
|
|
287
|
-
updateTagByteSize(db, sessionId, existing.tagId, toolByteSize);
|
|
288
|
-
// Keep token_count in lockstep with the byte bump so the
|
|
289
|
-
// grown output's tokens aren't undercounted by readers.
|
|
290
|
-
updateTagTokenCount(db, sessionId, existing.tagId, toolTokenCount);
|
|
291
|
-
}
|
|
292
|
-
if (existing.toolName === null && meta.toolName) {
|
|
293
|
-
existing.toolName = meta.toolName;
|
|
294
|
-
}
|
|
295
|
-
if (
|
|
296
|
-
existing.inputByteSize === 0 &&
|
|
297
|
-
part.kind === "tool_use" &&
|
|
298
|
-
meta.inputByteSize > 0
|
|
299
|
-
) {
|
|
300
|
-
existing.inputByteSize = meta.inputByteSize;
|
|
301
|
-
updateTagInputByteSize(db, sessionId, existing.tagId, meta.inputByteSize);
|
|
302
|
-
}
|
|
303
|
-
if (
|
|
304
|
-
!existing.inputTokenStored &&
|
|
305
|
-
part.kind === "tool_use" &&
|
|
306
|
-
inputTokenCount > 0
|
|
307
|
-
) {
|
|
308
|
-
existing.inputTokenStored = true;
|
|
309
|
-
updateTagInputTokenCount(db, sessionId, existing.tagId, inputTokenCount);
|
|
310
|
-
}
|
|
311
|
-
// Inject §N§ prefix into this tool_result occurrence
|
|
312
|
-
// (matches OpenCode behavior — only result gets the prefix).
|
|
313
|
-
if (!skipPrefixInjection && part.kind === "tool_result") {
|
|
314
|
-
part.setText(prependTag(existing.tagId, text));
|
|
315
|
-
}
|
|
316
|
-
// Rebuild the aggregate target so it walks the now-
|
|
317
|
-
// longer occurrences list.
|
|
318
|
-
targets.set(
|
|
319
|
-
existing.tagId,
|
|
320
|
-
buildAggregateTarget(existing.tagId, existing.occurrences),
|
|
321
|
-
);
|
|
322
385
|
if (part.kind === "tool_result") {
|
|
323
386
|
markToolAggregateResolved(
|
|
324
387
|
callId,
|
|
@@ -327,18 +390,61 @@ export function tagTranscript(
|
|
|
327
390
|
);
|
|
328
391
|
activeToolResultRun = { callId, aggregateKey };
|
|
329
392
|
}
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const reusableTagId = reuseIdentity
|
|
397
|
+
? tagger.getToolTag(sessionId, callId, messageId)
|
|
398
|
+
: undefined;
|
|
399
|
+
const reusableAccounting = reuseIdentity
|
|
400
|
+
? tagger.getToolTagAccounting(sessionId, callId, messageId)
|
|
401
|
+
: undefined;
|
|
402
|
+
let aggregate: ToolAggregate & { tagId: number };
|
|
403
|
+
let text = "";
|
|
404
|
+
if (reusableTagId !== undefined && reusableAccounting !== undefined) {
|
|
405
|
+
aggregate = {
|
|
406
|
+
callId,
|
|
407
|
+
tagId: reusableTagId,
|
|
408
|
+
identityReusable: true,
|
|
409
|
+
occurrences: [{ message, part, kind: part.kind }],
|
|
410
|
+
// Stable identity does not imply stable payload size. Seed from
|
|
411
|
+
// the persisted row so unchanged results avoid BPE while a
|
|
412
|
+
// genuinely larger folded result can still bump accounting.
|
|
413
|
+
maxByteSize: reusableAccounting.byteSize,
|
|
414
|
+
maxTokenCount: reusableAccounting.tokenCount ?? 0,
|
|
415
|
+
toolName: null,
|
|
416
|
+
inputByteSize: reusableAccounting.inputByteSize,
|
|
417
|
+
inputTokenCount: reusableAccounting.inputTokenCount,
|
|
418
|
+
};
|
|
419
|
+
if (part.kind === "tool_result") {
|
|
420
|
+
text = part.getText() ?? "";
|
|
421
|
+
applyGrownToolResultAccounting({
|
|
422
|
+
db,
|
|
423
|
+
sessionId,
|
|
424
|
+
tagger,
|
|
425
|
+
aggregate,
|
|
426
|
+
byteSize: getToolPartByteSize(part, text),
|
|
427
|
+
part,
|
|
428
|
+
text,
|
|
429
|
+
timing,
|
|
430
|
+
tokenCache: options.toolTokenCache,
|
|
431
|
+
tokenCacheKey: aggregateKey,
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
if (timing) timing.identity += performance.now() - identityStart;
|
|
330
435
|
} else {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
const
|
|
340
|
-
|
|
341
|
-
const firstInputTokenCount =
|
|
436
|
+
const accounting = readAggregateToolAccounting(
|
|
437
|
+
part,
|
|
438
|
+
timing,
|
|
439
|
+
options.toolTokenCache,
|
|
440
|
+
aggregateKey,
|
|
441
|
+
);
|
|
442
|
+
text = accounting.text;
|
|
443
|
+
const outputByteSize = part.kind === "tool_result" ? accounting.byteSize : 0;
|
|
444
|
+
const outputTokenCount =
|
|
445
|
+
part.kind === "tool_result" ? accounting.tokenCount : 0;
|
|
446
|
+
const firstInputTokenCount =
|
|
447
|
+
part.kind === "tool_use" ? accounting.inputTokenCount : 0;
|
|
342
448
|
const tagId = tagger.assignToolTag(
|
|
343
449
|
sessionId,
|
|
344
450
|
callId,
|
|
@@ -346,58 +452,188 @@ export function tagTranscript(
|
|
|
346
452
|
outputByteSize,
|
|
347
453
|
db,
|
|
348
454
|
0,
|
|
349
|
-
|
|
350
|
-
|
|
455
|
+
accounting.toolName,
|
|
456
|
+
accounting.inputByteSize,
|
|
351
457
|
() => ({
|
|
352
458
|
tokenCount: outputTokenCount,
|
|
353
459
|
inputTokenCount: firstInputTokenCount,
|
|
354
460
|
reasoningTokenCount: null,
|
|
355
461
|
}),
|
|
356
462
|
);
|
|
357
|
-
const
|
|
463
|
+
const persistedAccounting = tagger.getToolTagAccounting(
|
|
464
|
+
sessionId,
|
|
465
|
+
callId,
|
|
466
|
+
messageId,
|
|
467
|
+
);
|
|
468
|
+
aggregate = {
|
|
358
469
|
callId,
|
|
359
470
|
tagId,
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
inputTokenStored: part.kind === "tool_use" && firstInputTokenCount > 0,
|
|
471
|
+
identityReusable: false,
|
|
472
|
+
occurrences: [{ message, part, kind: part.kind }],
|
|
473
|
+
maxByteSize: persistedAccounting?.byteSize ?? outputByteSize,
|
|
474
|
+
maxTokenCount: persistedAccounting?.tokenCount ?? outputTokenCount,
|
|
475
|
+
toolName: accounting.toolName,
|
|
476
|
+
inputByteSize:
|
|
477
|
+
persistedAccounting?.inputByteSize ??
|
|
478
|
+
(part.kind === "tool_use" ? accounting.inputByteSize : 0),
|
|
479
|
+
inputTokenCount:
|
|
480
|
+
persistedAccounting?.inputTokenCount ??
|
|
481
|
+
(part.kind === "tool_use" ? firstInputTokenCount : null),
|
|
372
482
|
};
|
|
373
|
-
toolAggregates.set(aggregateKey, aggregate);
|
|
374
|
-
if (part.kind === "tool_use") {
|
|
375
|
-
openToolAggregateKeysByCallId.set(callId, [...pendingKeys, aggregateKey]);
|
|
376
|
-
}
|
|
377
|
-
// Inject §N§ prefix into this occurrence's visible text
|
|
378
|
-
// when it's a tool_result. (OpenCode parity: prefix
|
|
379
|
-
// only goes on the result, not the invocation.)
|
|
380
|
-
if (!skipPrefixInjection && part.kind === "tool_result") {
|
|
381
|
-
part.setText(prependTag(tagId, text));
|
|
382
|
-
}
|
|
383
|
-
targets.set(tagId, buildAggregateTarget(tagId, aggregate.occurrences));
|
|
384
483
|
if (part.kind === "tool_result") {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
484
|
+
applyGrownToolResultAccounting({
|
|
485
|
+
db,
|
|
486
|
+
sessionId,
|
|
487
|
+
tagger,
|
|
488
|
+
aggregate,
|
|
489
|
+
byteSize: accounting.byteSize,
|
|
490
|
+
part,
|
|
491
|
+
text,
|
|
492
|
+
timing,
|
|
493
|
+
knownTokenCount: accounting.tokenCount,
|
|
494
|
+
});
|
|
391
495
|
}
|
|
496
|
+
syncToolAggregateAccounting(tagger, sessionId, aggregate);
|
|
497
|
+
if (timing) timing.identity += performance.now() - identityStart;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
toolAggregates.set(aggregateKey, aggregate);
|
|
501
|
+
if (part.kind === "tool_use") {
|
|
502
|
+
openToolAggregateKeysByCallId.set(callId, [...pendingKeys, aggregateKey]);
|
|
503
|
+
}
|
|
504
|
+
applyToolPrefixAndTarget({
|
|
505
|
+
skipPrefixInjection,
|
|
506
|
+
part,
|
|
507
|
+
text,
|
|
508
|
+
tagId: aggregate.tagId,
|
|
509
|
+
aggregate,
|
|
510
|
+
targets,
|
|
511
|
+
timing,
|
|
512
|
+
});
|
|
513
|
+
if (part.kind === "tool_result") {
|
|
514
|
+
markToolAggregateResolved(callId, aggregateKey, openToolAggregateKeysByCallId);
|
|
515
|
+
activeToolResultRun = { callId, aggregateKey };
|
|
392
516
|
}
|
|
393
517
|
}
|
|
394
518
|
// thinking, image, file, structural, unknown → skip.
|
|
395
519
|
}
|
|
396
520
|
}
|
|
397
521
|
|
|
522
|
+
if (timing && options.onTiming) {
|
|
523
|
+
options.onTiming("identity", timing.identity);
|
|
524
|
+
options.onTiming("prefix", timing.prefix);
|
|
525
|
+
options.onTiming("targets", timing.targets);
|
|
526
|
+
options.onTiming("tokenCounting", timing.tokenCounting);
|
|
527
|
+
}
|
|
398
528
|
return { targets };
|
|
399
529
|
}
|
|
400
530
|
|
|
531
|
+
interface AggregateToolAccounting {
|
|
532
|
+
text: string;
|
|
533
|
+
byteSize: number;
|
|
534
|
+
tokenCount: number;
|
|
535
|
+
toolName: string | null;
|
|
536
|
+
inputByteSize: number;
|
|
537
|
+
inputTokenCount: number;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
interface GrownToolResultAccountingArgs {
|
|
541
|
+
db: ContextDatabase;
|
|
542
|
+
sessionId: string;
|
|
543
|
+
tagger: Tagger;
|
|
544
|
+
aggregate: ToolAggregate & { tagId: number };
|
|
545
|
+
byteSize: number;
|
|
546
|
+
part: TranscriptPart;
|
|
547
|
+
text: string;
|
|
548
|
+
timing: TagTranscriptTiming | undefined;
|
|
549
|
+
tokenCache?: Map<string, { text: string; tokenCount: number }>;
|
|
550
|
+
tokenCacheKey?: string;
|
|
551
|
+
knownTokenCount?: number;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function syncToolAggregateAccounting(
|
|
555
|
+
tagger: Tagger,
|
|
556
|
+
sessionId: string,
|
|
557
|
+
aggregate: ToolAggregate & { tagId: number },
|
|
558
|
+
): void {
|
|
559
|
+
tagger.setToolTagAccounting(sessionId, aggregate.tagId, {
|
|
560
|
+
byteSize: aggregate.maxByteSize,
|
|
561
|
+
tokenCount: aggregate.maxTokenCount,
|
|
562
|
+
inputByteSize: aggregate.inputByteSize,
|
|
563
|
+
inputTokenCount: aggregate.inputTokenCount,
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
function applyGrownToolResultAccounting(args: GrownToolResultAccountingArgs): void {
|
|
568
|
+
if (args.byteSize <= args.aggregate.maxByteSize) return;
|
|
569
|
+
|
|
570
|
+
let tokenCount = args.knownTokenCount;
|
|
571
|
+
if (tokenCount !== undefined && args.tokenCacheKey) {
|
|
572
|
+
args.tokenCache?.set(args.tokenCacheKey, { text: args.text, tokenCount });
|
|
573
|
+
}
|
|
574
|
+
if (tokenCount === undefined) {
|
|
575
|
+
const tokenStart = args.timing ? performance.now() : 0;
|
|
576
|
+
tokenCount = getCachedToolPartTokenCount(
|
|
577
|
+
args.part,
|
|
578
|
+
args.text,
|
|
579
|
+
args.tokenCache,
|
|
580
|
+
args.tokenCacheKey,
|
|
581
|
+
);
|
|
582
|
+
if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
|
|
583
|
+
}
|
|
584
|
+
args.aggregate.maxByteSize = args.byteSize;
|
|
585
|
+
args.aggregate.maxTokenCount = tokenCount;
|
|
586
|
+
updateTagByteSize(args.db, args.sessionId, args.aggregate.tagId, args.byteSize);
|
|
587
|
+
updateTagTokenCount(args.db, args.sessionId, args.aggregate.tagId, tokenCount);
|
|
588
|
+
syncToolAggregateAccounting(args.tagger, args.sessionId, args.aggregate);
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
function readAggregateToolAccounting(
|
|
592
|
+
part: TranscriptPart,
|
|
593
|
+
timing: TagTranscriptTiming | undefined,
|
|
594
|
+
tokenCache?: Map<string, { text: string; tokenCount: number }>,
|
|
595
|
+
tokenCacheKey?: string,
|
|
596
|
+
): AggregateToolAccounting {
|
|
597
|
+
const text = part.getText() ?? "";
|
|
598
|
+
const byteSize = getToolPartByteSize(part, text);
|
|
599
|
+
const metadata = part.getToolMetadata();
|
|
600
|
+
let tokenCount = 0;
|
|
601
|
+
if (part.kind === "tool_result") {
|
|
602
|
+
const tokenStart = timing ? performance.now() : 0;
|
|
603
|
+
tokenCount = getCachedToolPartTokenCount(part, text, tokenCache, tokenCacheKey);
|
|
604
|
+
if (timing) timing.tokenCounting += performance.now() - tokenStart;
|
|
605
|
+
}
|
|
606
|
+
return {
|
|
607
|
+
text,
|
|
608
|
+
byteSize,
|
|
609
|
+
tokenCount,
|
|
610
|
+
toolName: metadata.toolName ?? null,
|
|
611
|
+
inputByteSize: metadata.inputByteSize,
|
|
612
|
+
inputTokenCount: metadata.inputTokenCount,
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
interface ApplyToolPrefixAndTargetArgs {
|
|
617
|
+
skipPrefixInjection: boolean;
|
|
618
|
+
part: TranscriptPart;
|
|
619
|
+
text: string;
|
|
620
|
+
tagId: number;
|
|
621
|
+
aggregate: ToolAggregate;
|
|
622
|
+
targets: Map<number, TagTarget>;
|
|
623
|
+
timing: TagTranscriptTiming | undefined;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
function applyToolPrefixAndTarget(args: ApplyToolPrefixAndTargetArgs): void {
|
|
627
|
+
if (!args.skipPrefixInjection && args.part.kind === "tool_result") {
|
|
628
|
+
const prefixStart = args.timing ? performance.now() : 0;
|
|
629
|
+
args.part.setText(prependTag(args.tagId, args.text));
|
|
630
|
+
if (args.timing) args.timing.prefix += performance.now() - prefixStart;
|
|
631
|
+
}
|
|
632
|
+
const targetStart = args.timing ? performance.now() : 0;
|
|
633
|
+
args.targets.set(args.tagId, buildAggregateTarget(args.tagId, args.aggregate.occurrences));
|
|
634
|
+
if (args.timing) args.timing.targets += performance.now() - targetStart;
|
|
635
|
+
}
|
|
636
|
+
|
|
401
637
|
function findLastUnresolvedToolAggregateKey(
|
|
402
638
|
pendingKeys: string[],
|
|
403
639
|
toolAggregates: Map<string, ToolAggregate & { tagId: number }>,
|
|
@@ -466,6 +702,19 @@ function getToolPartTokenCount(part: TranscriptPart, text: string): number {
|
|
|
466
702
|
return 0;
|
|
467
703
|
}
|
|
468
704
|
|
|
705
|
+
function getCachedToolPartTokenCount(
|
|
706
|
+
part: TranscriptPart,
|
|
707
|
+
text: string,
|
|
708
|
+
cache?: Map<string, { text: string; tokenCount: number }>,
|
|
709
|
+
cacheKey?: string,
|
|
710
|
+
): number {
|
|
711
|
+
const cached = cacheKey ? cache?.get(cacheKey) : undefined;
|
|
712
|
+
if (cached?.text === text) return cached.tokenCount;
|
|
713
|
+
const tokenCount = getToolPartTokenCount(part, text);
|
|
714
|
+
if (cacheKey) cache?.set(cacheKey, { text, tokenCount });
|
|
715
|
+
return tokenCount;
|
|
716
|
+
}
|
|
717
|
+
|
|
469
718
|
function getNonTextToolResultByteSize(part: TranscriptPart): number {
|
|
470
719
|
// Prefer the adapter's exact raw-payload size when available (Pi's
|
|
471
720
|
// tool_result proxy can serialize the real content array, incl. images).
|
|
@@ -508,11 +757,23 @@ interface TagTextPartArgs {
|
|
|
508
757
|
targets: Map<number, TagTarget>;
|
|
509
758
|
skipPrefixInjection: boolean;
|
|
510
759
|
entryFingerprint: string | null;
|
|
760
|
+
reuseIdentity: boolean;
|
|
761
|
+
timing?: TagTranscriptTiming;
|
|
762
|
+
textTokenCache?: Map<string, { text: string; tokenCount: number }>;
|
|
511
763
|
}
|
|
512
764
|
|
|
513
765
|
function tagTextPart(args: TagTextPartArgs): void {
|
|
766
|
+
const identityStart = args.timing ? performance.now() : 0;
|
|
514
767
|
const text = args.part.getText() ?? "";
|
|
515
768
|
const contentId = `${args.messageId}:p${args.textOrdinal}`;
|
|
769
|
+
const reusableTagId = args.reuseIdentity
|
|
770
|
+
? args.tagger.getTag(args.sessionId, contentId, "message")
|
|
771
|
+
: undefined;
|
|
772
|
+
if (reusableTagId !== undefined) {
|
|
773
|
+
if (args.timing) args.timing.identity += performance.now() - identityStart;
|
|
774
|
+
applyTextPrefixAndTarget(args, reusableTagId, text);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
516
777
|
const tagId = args.tagger.assignTag(
|
|
517
778
|
args.sessionId,
|
|
518
779
|
contentId,
|
|
@@ -525,11 +786,24 @@ function tagTextPart(args: TagTextPartArgs): void {
|
|
|
525
786
|
args.entryFingerprint,
|
|
526
787
|
// Lazy: fires only on fresh insert. Strip any §N§ prefix so a re-tag
|
|
527
788
|
// from already-prefixed text still tokenizes the pristine content.
|
|
528
|
-
() =>
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
789
|
+
() => {
|
|
790
|
+
const tokenStart = args.timing ? performance.now() : 0;
|
|
791
|
+
const cached = args.textTokenCache?.get(contentId);
|
|
792
|
+
const tokenCount =
|
|
793
|
+
cached?.text === text
|
|
794
|
+
? cached.tokenCount
|
|
795
|
+
: estimateTagTextTokens(stripTagPrefix(text));
|
|
796
|
+
if (cached?.text !== text) {
|
|
797
|
+
args.textTokenCache?.set(contentId, { text, tokenCount });
|
|
798
|
+
}
|
|
799
|
+
const counts = {
|
|
800
|
+
tokenCount,
|
|
801
|
+
inputTokenCount: null,
|
|
802
|
+
reasoningTokenCount: null,
|
|
803
|
+
};
|
|
804
|
+
if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
|
|
805
|
+
return counts;
|
|
806
|
+
},
|
|
533
807
|
);
|
|
534
808
|
|
|
535
809
|
// Persist the original (pre-tagged) source content so caveman
|
|
@@ -547,12 +821,20 @@ function tagTextPart(args: TagTextPartArgs): void {
|
|
|
547
821
|
if (sourceContent.trim().length > 0) {
|
|
548
822
|
saveSourceContent(args.db, args.sessionId, tagId, sourceContent);
|
|
549
823
|
}
|
|
824
|
+
if (args.timing) args.timing.identity += performance.now() - identityStart;
|
|
825
|
+
applyTextPrefixAndTarget(args, tagId, text);
|
|
826
|
+
}
|
|
550
827
|
|
|
828
|
+
function applyTextPrefixAndTarget(args: TagTextPartArgs, tagId: number, text: string): void {
|
|
551
829
|
if (!args.skipPrefixInjection) {
|
|
830
|
+
const prefixStart = args.timing ? performance.now() : 0;
|
|
552
831
|
args.part.setText(prependTag(tagId, text));
|
|
832
|
+
if (args.timing) args.timing.prefix += performance.now() - prefixStart;
|
|
553
833
|
}
|
|
554
834
|
|
|
835
|
+
const targetStart = args.timing ? performance.now() : 0;
|
|
555
836
|
args.targets.set(tagId, buildTextTarget(args.part, args.message));
|
|
837
|
+
if (args.timing) args.timing.targets += performance.now() - targetStart;
|
|
556
838
|
}
|
|
557
839
|
|
|
558
840
|
interface TagToolPartArgs {
|
|
@@ -566,9 +848,12 @@ interface TagToolPartArgs {
|
|
|
566
848
|
db: ContextDatabase;
|
|
567
849
|
targets: Map<number, TagTarget>;
|
|
568
850
|
skipPrefixInjection: boolean;
|
|
851
|
+
reuseIdentity: boolean;
|
|
852
|
+
timing?: TagTranscriptTiming;
|
|
569
853
|
}
|
|
570
854
|
|
|
571
855
|
function tagToolPart(args: TagToolPartArgs): void {
|
|
856
|
+
const identityStart = args.timing ? performance.now() : 0;
|
|
572
857
|
// Prefer the part's stable id (tool call id from Pi/OpenCode); fall
|
|
573
858
|
// back to a synthetic locator. Tool calls and their results MAY
|
|
574
859
|
// share an id (Pi sets toolCallId on ToolResultMessage to match the
|
|
@@ -577,10 +862,21 @@ function tagToolPart(args: TagToolPartArgs): void {
|
|
|
577
862
|
// target the call-id pair as a unit.
|
|
578
863
|
const stableId = args.part.id;
|
|
579
864
|
const contentId = stableId ?? `${args.messageId}:t${args.partIndex}`;
|
|
865
|
+
const reusableTagId = args.reuseIdentity
|
|
866
|
+
? args.tagger.getToolTag(args.sessionId, contentId, contentId)
|
|
867
|
+
: undefined;
|
|
868
|
+
if (reusableTagId !== undefined) {
|
|
869
|
+
const text = args.part.kind === "tool_result" ? (args.part.getText() ?? "") : "";
|
|
870
|
+
if (args.timing) args.timing.identity += performance.now() - identityStart;
|
|
871
|
+
applySingleToolPrefixAndTarget(args, reusableTagId, text);
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
580
874
|
const text = args.part.getText() ?? "";
|
|
581
875
|
const toolByteSize = getToolPartByteSize(args.part, text);
|
|
582
|
-
const toolTokenCount = getToolPartTokenCount(args.part, text);
|
|
583
876
|
const meta = args.part.getToolMetadata();
|
|
877
|
+
const tokenStart = args.timing ? performance.now() : 0;
|
|
878
|
+
const toolTokenCount = getToolPartTokenCount(args.part, text);
|
|
879
|
+
if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
|
|
584
880
|
// v3.3.1 Layer C: synthetic ownership for the no-callId Pi
|
|
585
881
|
// fallback. Owner == callId == contentId. The composite key
|
|
586
882
|
// collapses to a unique synthetic identifier per part, preserving
|
|
@@ -596,22 +892,34 @@ function tagToolPart(args: TagToolPartArgs): void {
|
|
|
596
892
|
0,
|
|
597
893
|
meta.toolName ?? null,
|
|
598
894
|
meta.inputByteSize,
|
|
599
|
-
() =>
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
895
|
+
() => {
|
|
896
|
+
const tokenStart = args.timing ? performance.now() : 0;
|
|
897
|
+
const counts = {
|
|
898
|
+
tokenCount: toolTokenCount,
|
|
899
|
+
inputTokenCount: meta.inputTokenCount,
|
|
900
|
+
reasoningTokenCount: null,
|
|
901
|
+
};
|
|
902
|
+
if (args.timing) args.timing.tokenCounting += performance.now() - tokenStart;
|
|
903
|
+
return counts;
|
|
904
|
+
},
|
|
604
905
|
);
|
|
906
|
+
if (args.timing) args.timing.identity += performance.now() - identityStart;
|
|
907
|
+
applySingleToolPrefixAndTarget(args, tagId, text);
|
|
908
|
+
}
|
|
605
909
|
|
|
910
|
+
function applySingleToolPrefixAndTarget(args: TagToolPartArgs, tagId: number, text: string): void {
|
|
606
911
|
// For tool parts, the visible payload is the tool result text. We
|
|
607
912
|
// can inject the tag prefix into it for in-text references; this
|
|
608
913
|
// matches the OpenCode behavior of tagging tool outputs.
|
|
609
914
|
if (!args.skipPrefixInjection && args.part.kind === "tool_result") {
|
|
610
|
-
const
|
|
611
|
-
args.part.setText(
|
|
915
|
+
const prefixStart = args.timing ? performance.now() : 0;
|
|
916
|
+
args.part.setText(prependTag(tagId, text));
|
|
917
|
+
if (args.timing) args.timing.prefix += performance.now() - prefixStart;
|
|
612
918
|
}
|
|
613
919
|
|
|
920
|
+
const targetStart = args.timing ? performance.now() : 0;
|
|
614
921
|
args.targets.set(tagId, buildToolTarget(args.part, args.message, tagId));
|
|
922
|
+
if (args.timing) args.timing.targets += performance.now() - targetStart;
|
|
615
923
|
}
|
|
616
924
|
|
|
617
925
|
function setToolContentOrText(part: TranscriptPart, content: string): boolean {
|