@bglocation/tune-context 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +115 -0
- package/README.md +83 -6
- package/cli/experiment.mjs +2 -2
- package/eval/adoption-report.mjs +1 -1
- package/eval/analyze.mjs +689 -0
- package/eval/arm.mjs +9 -5
- package/eval/pricing.mjs +32 -1
- package/eval/prompt-metrics.mjs +357 -29
- package/eval/report.mjs +870 -0
- package/eval/verdict.mjs +639 -0
- package/hooks/adoption-log.mjs +1 -1
- package/hooks/context-reminder.mjs +1 -1
- package/hooks/pre-compact-snapshot.mjs +4 -4
- package/hooks/session-start-reinject.mjs +1 -4
- package/package.json +7 -2
package/eval/arm.mjs
CHANGED
|
@@ -96,13 +96,17 @@ export function ledgerArmAt(entries, tsIso) {
|
|
|
96
96
|
* The row's timestamp: when the prompt was submitted, or the first **readable**
|
|
97
97
|
* one of the remaining fields.
|
|
98
98
|
*
|
|
99
|
-
* One definition for
|
|
100
|
-
*
|
|
99
|
+
* One definition for every reader — the ledger window, `pass_order`, and the
|
|
100
|
+
* natural-work attribution in analyze.mjs (TASK-089), which asks the same ledger
|
|
101
|
+
* the same question about a row that is NOT on an `exp/` branch. Exported for
|
|
102
|
+
* that reason: a second "when did this row happen" would be a second answer.
|
|
103
|
+
*
|
|
104
|
+
* Two of these readers already drifted apart once: `??` only falls through on null, so it
|
|
101
105
|
* stopped at a `ts_submit` that was a string but unparseable — and a row the
|
|
102
106
|
* ledger DID cover came out as `conflict:no-ledger-entry`, i.e. a run to be
|
|
103
107
|
* repeated (§7) because of a cosmetic defect in the data.
|
|
104
108
|
*/
|
|
105
|
-
function
|
|
109
|
+
export function rowTimestamp(row) {
|
|
106
110
|
for (const value of [row.ts_submit, row.ts_first_assistant, row.ts_end]) {
|
|
107
111
|
if (msOf(value) !== null) return value;
|
|
108
112
|
}
|
|
@@ -110,7 +114,7 @@ function rowTs(row) {
|
|
|
110
114
|
}
|
|
111
115
|
|
|
112
116
|
function rowMs(row) {
|
|
113
|
-
return msOf(
|
|
117
|
+
return msOf(rowTimestamp(row));
|
|
114
118
|
}
|
|
115
119
|
|
|
116
120
|
function armFields(row, { ledger, ledgerAvailable }) {
|
|
@@ -133,7 +137,7 @@ function armFields(row, { ledger, ledgerAvailable }) {
|
|
|
133
137
|
return { exp_task: parsed.task, exp_arm: parsed.arm, exp_arm_source: 'branch', exp_ledger_arm: '' };
|
|
134
138
|
}
|
|
135
139
|
|
|
136
|
-
const fromLedger = ledgerArmAt(ledger,
|
|
140
|
+
const fromLedger = ledgerArmAt(ledger, rowTimestamp(row));
|
|
137
141
|
if (fromLedger === null) {
|
|
138
142
|
return { exp_task: parsed.task, exp_arm: '', exp_arm_source: 'conflict:no-ledger-entry', exp_ledger_arm: '' };
|
|
139
143
|
}
|
package/eval/pricing.mjs
CHANGED
|
@@ -115,7 +115,29 @@ export const GEO_MULTIPLIERS = { global: 1, not_available: 1, us: 1.1 };
|
|
|
115
115
|
/** $10 per 1,000 searches. Web FETCH is free — only search is billed. */
|
|
116
116
|
export const WEB_SEARCH_USD = 0.01;
|
|
117
117
|
|
|
118
|
-
|
|
118
|
+
/**
|
|
119
|
+
* The five priced classes, paired with the field each one is read from on a
|
|
120
|
+
* prompt-metrics row. Two vocabularies for one fact — `input`/`output` are cost
|
|
121
|
+
* classes, `input_tokens`/`output_tokens` are row fields — and this is the one
|
|
122
|
+
* place that knows the translation, because costOfTokens() below is already its
|
|
123
|
+
* other reader. Anything that folds tokens up (eval/analyze.mjs) reads it from
|
|
124
|
+
* here rather than retyping the list: a sixth class added to the collector's
|
|
125
|
+
* emptyTokens() must not be able to fall out of a downstream total by being
|
|
126
|
+
* forgotten in a hand-written array.
|
|
127
|
+
*
|
|
128
|
+
* `cache_write_unsplit` is deliberately NOT here. It is not a priced class — it
|
|
129
|
+
* is the admission that a write's TTL is unknown, and a row carrying it is
|
|
130
|
+
* refused rather than priced.
|
|
131
|
+
*/
|
|
132
|
+
export const TOKEN_CLASS_FIELDS = [
|
|
133
|
+
['input', 'input_tokens'],
|
|
134
|
+
['cache_read', 'cache_read'],
|
|
135
|
+
['cache_write_5m', 'cache_write_5m'],
|
|
136
|
+
['cache_write_1h', 'cache_write_1h'],
|
|
137
|
+
['output', 'output_tokens'],
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
const TOKEN_CLASSES = TOKEN_CLASS_FIELDS.map(([className]) => className);
|
|
119
141
|
|
|
120
142
|
function emptyCost() {
|
|
121
143
|
return Object.fromEntries(TOKEN_CLASSES.map((c) => [c, 0]));
|
|
@@ -371,6 +393,15 @@ function usd(n) {
|
|
|
371
393
|
return `$${n.toFixed(4)}`;
|
|
372
394
|
}
|
|
373
395
|
|
|
396
|
+
/**
|
|
397
|
+
* An amount for human eyes, including the case where there is none. An
|
|
398
|
+
* uncomputable total prints as the word, never as `$0.0000` — the whole file
|
|
399
|
+
* exists to stop an unknown cost reading as a cheap one.
|
|
400
|
+
*/
|
|
401
|
+
export function formatUsd(n) {
|
|
402
|
+
return Number.isFinite(n) ? usd(n) : 'unknown (NOT $0)';
|
|
403
|
+
}
|
|
404
|
+
|
|
374
405
|
function count(n) {
|
|
375
406
|
return n.toLocaleString('en-US');
|
|
376
407
|
}
|
package/eval/prompt-metrics.mjs
CHANGED
|
@@ -20,11 +20,14 @@
|
|
|
20
20
|
// inside list_cost_usd. Web search costs $10/1000; web FETCH is free. Both
|
|
21
21
|
// counts ride on the row. Zero on all data measured so far.
|
|
22
22
|
// • Subagent turns are NOT in the parent transcript. Measured 2026-07-30:
|
|
23
|
-
// 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries.
|
|
24
|
-
//
|
|
25
|
-
// the
|
|
26
|
-
//
|
|
27
|
-
//
|
|
23
|
+
// 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries. But the
|
|
24
|
+
// RESULT of an `Agent` call carries its own `usage` and `resolvedModel`, so
|
|
25
|
+
// the delegation is measurable FROM BELOW — see applyDelegation(). The row
|
|
26
|
+
// therefore reports two figures that are never added together: the parent
|
|
27
|
+
// session's cost, and `delegations[].list_cost_usd`, a lower bound on what
|
|
28
|
+
// the subagents cost (COST_EVALUATION_PROCEDURE §11a). `agent_spawns` and
|
|
29
|
+
// `sidechain_turns` are still counted, so the size of the remaining blind
|
|
30
|
+
// spot stays visible (EPIC-009, error class 3).
|
|
28
31
|
// • `<synthetic>` assistant entries are generated by Claude Code, not billed
|
|
29
32
|
// by the API. They are excluded from tokens/turns and counted separately —
|
|
30
33
|
// an unbillable entry, not TASK-086's "unknown model".
|
|
@@ -42,7 +45,7 @@ import { parseArgs } from 'node:util';
|
|
|
42
45
|
import { claudeBaseDir } from '../hooks/config-dir.mjs';
|
|
43
46
|
import { experimentPaths, readLedger } from '../cli/experiment.mjs';
|
|
44
47
|
import { annotateRows, formatArmSummary } from './arm.mjs';
|
|
45
|
-
import { formatCostLines, priceRow, summarizeCost } from './pricing.mjs';
|
|
48
|
+
import { formatCostLines, formatUsd, priceRow, summarizeCost, webSearchCost } from './pricing.mjs';
|
|
46
49
|
|
|
47
50
|
// Tool names that spawn a subagent. `Agent` is what this Claude Code writes
|
|
48
51
|
// (measured); `Task` is the name the same tool carried in earlier versions.
|
|
@@ -86,6 +89,10 @@ leaves the arm unresolved rather than picking a winner.
|
|
|
86
89
|
These five fields are added here, in the CLI. Rows obtained through the module
|
|
87
90
|
(aggregate, collectFiles) carry the measurement only — call annotateRows from
|
|
88
91
|
eval/arm.mjs to add the arm.
|
|
92
|
+
|
|
93
|
+
Each row also carries delegations[]: one record per Agent call that returned its
|
|
94
|
+
own usage, priced with the same table. That figure is a LOWER BOUND on what the
|
|
95
|
+
subagents cost and is never added to the row's own cost.
|
|
89
96
|
`.trim();
|
|
90
97
|
|
|
91
98
|
/** Parse one transcript line. Returns null for blanks, junk, or non-objects. */
|
|
@@ -168,6 +175,33 @@ function addUsage(into, usage) {
|
|
|
168
175
|
into.output_tokens += usage.output_tokens || 0;
|
|
169
176
|
}
|
|
170
177
|
|
|
178
|
+
/** The `tool_use` blocks an assistant entry carries, in block order. One reader
|
|
179
|
+
* for the row path and the unattributed bucket both: what counts as a tool call
|
|
180
|
+
* must not quietly become two slightly different answers. */
|
|
181
|
+
function toolUseBlocks(entry) {
|
|
182
|
+
const content = Array.isArray(entry.message?.content) ? entry.message.content : [];
|
|
183
|
+
return content.filter((block) => block?.type === 'tool_use');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Server-side tool requests carried by a usage block, as plain counts.
|
|
187
|
+
*
|
|
188
|
+
* Deliberately NOT folded into addUsage(): every field that function touches is
|
|
189
|
+
* a token class priced PER TOKEN, and these are billed PER REQUEST, which is
|
|
190
|
+
* why they sit outside list_cost_by_class on a row. addUsage also feeds the
|
|
191
|
+
* usage.iterations comparison (where a request count would be compared against
|
|
192
|
+
* token counts) and the per-model buckets published as `by_model` — a field
|
|
193
|
+
* added there would surface in a row's token breakdown, which its own test
|
|
194
|
+
* pins to exactly the five priced classes plus the unsplit one. So the shape of
|
|
195
|
+
* `usage.server_tool_use` gets ONE reader instead, and the two accumulators
|
|
196
|
+
* that need it keep their own field names. */
|
|
197
|
+
function serverToolRequests(usage) {
|
|
198
|
+
const server = usage?.server_tool_use;
|
|
199
|
+
return {
|
|
200
|
+
search: server?.web_search_requests || 0,
|
|
201
|
+
fetch: server?.web_fetch_requests || 0,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
171
205
|
function newPrompt({ promptId, session }) {
|
|
172
206
|
return {
|
|
173
207
|
promptId,
|
|
@@ -207,6 +241,12 @@ function newPrompt({ promptId, session }) {
|
|
|
207
241
|
toolCalls: 0,
|
|
208
242
|
agentSpawns: 0,
|
|
209
243
|
sidechainTurns: 0,
|
|
244
|
+
// Keyed by agentId, because the same delegation result is written to
|
|
245
|
+
// SEVERAL transcript files (measured: 8 entries for 4 delegations). A list
|
|
246
|
+
// here would double the only figure we have for subagent cost.
|
|
247
|
+
delegations: new Map(),
|
|
248
|
+
delegationDuplicates: 0,
|
|
249
|
+
delegationsUnidentified: 0,
|
|
210
250
|
promptChars: null,
|
|
211
251
|
};
|
|
212
252
|
}
|
|
@@ -292,10 +332,86 @@ function applyUserEntry(p, entry) {
|
|
|
292
332
|
}
|
|
293
333
|
}
|
|
294
334
|
|
|
335
|
+
/**
|
|
336
|
+
* The subagent's own bill, off the `Agent` tool RESULT.
|
|
337
|
+
*
|
|
338
|
+
* A delegation leaves no turns in the parent transcript (0 `isSidechain`
|
|
339
|
+
* entries over the whole of `~/.claude/projects`), but the result entry carries
|
|
340
|
+
* `toolUseResult.usage` — the same structure as `message.usage`, cache split
|
|
341
|
+
* included — plus `resolvedModel`. So it is priced with the SAME `priceRow`
|
|
342
|
+
* that prices the parent turn: TASK-089 criterion 8 asks for no new pricing
|
|
343
|
+
* logic, and a second code path to dollars is a second set of prices.
|
|
344
|
+
*
|
|
345
|
+
* Three rules this obeys, all from COST_EVALUATION_PROCEDURE §11a:
|
|
346
|
+
*
|
|
347
|
+
* • **Deduplicated by `agentId`.** The result entry is written twice (8
|
|
348
|
+
* entries = 4 delegations, measured) — and, measured 2026-08-02, WITHIN one
|
|
349
|
+
* file rather than across files as first assumed, sometimes under a
|
|
350
|
+
* different `promptId`, which puts one delegation on two rows. The entry
|
|
351
|
+
* dedup in addEntry() removes the byte-identical copies; this keeps the
|
|
352
|
+
* guarantee for the ones it cannot see, i.e. copies with distinct uuids and
|
|
353
|
+
* rows collected from more than one file.
|
|
354
|
+
* • **A lower bound, not a cost.** `cache_read` grows LINEARLY with the
|
|
355
|
+
* subagent's tool-call count (13 → 37.6k, 19 → 59.5k, 26 → 76.5k), which is
|
|
356
|
+
* what one turn reading the whole accumulated context looks like — not a
|
|
357
|
+
* sum over turns, which would grow quadratically. So this is most likely
|
|
358
|
+
* the subagent's LAST turn. It rides on its own field and is never added to
|
|
359
|
+
* the row's cost.
|
|
360
|
+
* • **A result with no `agentId` is dropped, not guessed.** It cannot be
|
|
361
|
+
* deduplicated, and a double-counted lower bound is worse than a smaller
|
|
362
|
+
* one that is honest. Counted, so the omission is visible.
|
|
363
|
+
*/
|
|
364
|
+
function applyDelegation(p, entry, tsMs) {
|
|
365
|
+
const result = entry.toolUseResult;
|
|
366
|
+
if (!result || typeof result !== 'object' || Array.isArray(result)) return;
|
|
367
|
+
const usage = result.usage;
|
|
368
|
+
if (!usage || typeof usage !== 'object') return;
|
|
369
|
+
|
|
370
|
+
const id = typeof result.agentId === 'string' ? result.agentId : '';
|
|
371
|
+
if (id === '') {
|
|
372
|
+
p.delegationsUnidentified++;
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
if (p.delegations.has(id)) {
|
|
376
|
+
p.delegationDuplicates++;
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const tokens = emptyTokens();
|
|
381
|
+
addUsage(tokens, usage);
|
|
382
|
+
const model = typeof result.resolvedModel === 'string' ? result.resolvedModel : '';
|
|
383
|
+
// Shaped as a one-model row so priceRow answers about it unchanged — including
|
|
384
|
+
// the refusals: an unknown model, an unsplit cache write or a tier we cannot
|
|
385
|
+
// price yields `list_cost_usd: null` here for exactly the same reasons as on a
|
|
386
|
+
// parent turn, and lands in the same unpriced counter downstream.
|
|
387
|
+
const cost = priceRow(
|
|
388
|
+
{
|
|
389
|
+
by_model: { [model]: tokens },
|
|
390
|
+
speed: typeof usage.speed === 'string' ? usage.speed : null,
|
|
391
|
+
service_tier: usage.service_tier,
|
|
392
|
+
inference_geo: usage.inference_geo,
|
|
393
|
+
},
|
|
394
|
+
{ at: tsMs },
|
|
395
|
+
);
|
|
396
|
+
|
|
397
|
+
p.delegations.set(id, {
|
|
398
|
+
agent_id: id,
|
|
399
|
+
agent_type: typeof result.agentType === 'string' ? result.agentType : '',
|
|
400
|
+
model,
|
|
401
|
+
status: typeof result.status === 'string' ? result.status : '',
|
|
402
|
+
duration_ms: Number.isFinite(result.totalDurationMs) ? result.totalDurationMs : null,
|
|
403
|
+
// The subagent's own tool calls — the quantity `cache_read` tracks linearly,
|
|
404
|
+
// i.e. the evidence for reading the figure above as a lower bound.
|
|
405
|
+
tool_calls_total: Number.isFinite(result.totalToolUseCount) ? result.totalToolUseCount : null,
|
|
406
|
+
...tokens,
|
|
407
|
+
web_search_requests: usage.server_tool_use?.web_search_requests || 0,
|
|
408
|
+
...cost,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
|
|
295
412
|
/** Tools, delegations, model/effort, and the per-model token split. */
|
|
296
|
-
function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
297
|
-
for (const block of
|
|
298
|
-
if (block?.type !== 'tool_use') continue;
|
|
413
|
+
function applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse }) {
|
|
414
|
+
for (const block of toolUseBlocks(entry)) {
|
|
299
415
|
const name = typeof block.name === 'string' ? block.name : '(unknown)';
|
|
300
416
|
p.tools.set(name, (p.tools.get(name) ?? 0) + 1);
|
|
301
417
|
p.toolCalls++;
|
|
@@ -309,6 +425,26 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
|
309
425
|
return;
|
|
310
426
|
}
|
|
311
427
|
|
|
428
|
+
// `usage` describes the RESPONSE, not the entry. One API response reaches the
|
|
429
|
+
// transcript as several entries — one per content block (thinking / text /
|
|
430
|
+
// each tool_use) — and every one of them carries an identical copy of the same
|
|
431
|
+
// usage block. The uuid dedup in addEntry() cannot see this class: those uuids
|
|
432
|
+
// differ by definition. Measured over 19 transcripts: 11 295 entries for 5 954
|
|
433
|
+
// responses = 1.90x on api_turns, and 2.02x on the dollar figure, because the
|
|
434
|
+
// per-class inflation is not flat (cache-read 1.82x, output 2.17x) — a
|
|
435
|
+
// many-block response is not a scaled copy of a one-block response.
|
|
436
|
+
//
|
|
437
|
+
// Not a constant that cancels in a ratio of two arms: the factor IS the number
|
|
438
|
+
// of content blocks per turn, i.e. how many tools the model reached for, which
|
|
439
|
+
// is the variable the doctrine sets out to move. Per-session spread on one
|
|
440
|
+
// project: 1.66x to 2.24x (error class 1 — a figure that prints like a fact).
|
|
441
|
+
//
|
|
442
|
+
// The tool_use loop above runs BEFORE this guard on purpose: tool_use blocks
|
|
443
|
+
// are NOT duplicated across a response's entries (6 403 blocks, 6 403 distinct
|
|
444
|
+
// ids, 0 repeats), so skipping the whole entry would drop ~2/3 of all tool
|
|
445
|
+
// calls and gut the adoption half of the analysis.
|
|
446
|
+
if (repeatOfResponse) return;
|
|
447
|
+
|
|
312
448
|
p.apiTurns++;
|
|
313
449
|
if (model) p.models.add(model);
|
|
314
450
|
if (typeof entry.effort === 'string') p.efforts.add(entry.effort);
|
|
@@ -329,11 +465,9 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
|
329
465
|
if (usageUnderstated(usage)) p.understatedTurns++;
|
|
330
466
|
// Billed per request, not per token, so they sit outside list_cost_by_class.
|
|
331
467
|
// Counted so the gap announces itself instead of hiding in a clean total.
|
|
332
|
-
const server = usage
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
p.webFetchRequests += server.web_fetch_requests || 0;
|
|
336
|
-
}
|
|
468
|
+
const server = serverToolRequests(usage);
|
|
469
|
+
p.webSearchRequests += server.search;
|
|
470
|
+
p.webFetchRequests += server.fetch;
|
|
337
471
|
}
|
|
338
472
|
|
|
339
473
|
/**
|
|
@@ -365,8 +499,34 @@ export function createCollector() {
|
|
|
365
499
|
const prompts = new Map(); // key -> accumulator, in first-seen order
|
|
366
500
|
const groups = new Map(); // key -> {key, promptId, session}, shared by memoized uuids
|
|
367
501
|
let uuidGroup = new Map(); // uuid -> group | null; reset per file (see beginFile)
|
|
368
|
-
const unattributed = {
|
|
369
|
-
|
|
502
|
+
const unattributed = {
|
|
503
|
+
entries: 0,
|
|
504
|
+
turns: 0,
|
|
505
|
+
// Counted for the same reason the rows count them: tool_calls is an
|
|
506
|
+
// ADOPTION metric — how often the model reached for a tool, i.e. the very
|
|
507
|
+
// variable the doctrine sets out to move — so an entry that lands outside
|
|
508
|
+
// every row must still show up in that denominator. `web_search_requests`
|
|
509
|
+
// is a different kind of thing: it is MONEY ($10/1000, billed per request),
|
|
510
|
+
// and money that counting the entry's tokens does not cover, because no
|
|
511
|
+
// token class prices it (TASK-093).
|
|
512
|
+
tool_calls: 0,
|
|
513
|
+
// A strict subset of tool_calls, tracked separately for the same reason the
|
|
514
|
+
// rows track it: agent_spawns is one half of how big the subagent blind spot
|
|
515
|
+
// is (error class 3), and a spawn nobody counted is a whole delegation whose
|
|
516
|
+
// cost we do not merely under-report — we never learn it happened.
|
|
517
|
+
agent_spawns: 0,
|
|
518
|
+
web_search_requests: 0,
|
|
519
|
+
web_fetch_requests: 0,
|
|
520
|
+
...emptyTokens(),
|
|
521
|
+
};
|
|
522
|
+
// Identity of every API response already billed. NOT reset in beginFile, and
|
|
523
|
+
// deliberately unlike uuidGroup: a uuid chain cannot cross a file, but nothing
|
|
524
|
+
// promises the same about a response id, and a global set costs one string per
|
|
525
|
+
// response. Measured across the 19 guidetrackee transcripts: 0 of 5 954 ids
|
|
526
|
+
// appear in more than one file, so today the scope changes no figure — it only
|
|
527
|
+
// removes a way for a future transcript layout to reopen the bug silently.
|
|
528
|
+
const seenResponses = new Set();
|
|
529
|
+
const totals = { malformed: 0, entries: 0, duplicates: 0, responseRepeats: 0, responsesUnidentified: 0, files: 0 };
|
|
370
530
|
|
|
371
531
|
function groupFor(entry) {
|
|
372
532
|
// Direct: `user` entries (prompt AND tool results) carry promptId.
|
|
@@ -389,6 +549,28 @@ export function createCollector() {
|
|
|
389
549
|
|
|
390
550
|
function addEntry(entry) {
|
|
391
551
|
totals.entries++;
|
|
552
|
+
|
|
553
|
+
// A transcript repeats entries VERBATIM, and not rarely: 11 296 of 36 463
|
|
554
|
+
// local entries (31%) carry a uuid already written earlier IN THE SAME FILE,
|
|
555
|
+
// 6 739 of them assistant turns with a usage block. The copies are identical
|
|
556
|
+
// apart from a cosmetic `slug` — same tokens, same model, same timestamp —
|
|
557
|
+
// so summing both charged 45% of all cache-read tokens and 42% of all output
|
|
558
|
+
// tokens twice. That is not a rounding error, it is the epic's whole effect
|
|
559
|
+
// size several times over (error class 1: a figure that prints like a fact).
|
|
560
|
+
//
|
|
561
|
+
// Scoped to the file on purpose: across the 34 local transcripts ZERO uuids
|
|
562
|
+
// are shared between files, so the index that already exists per file
|
|
563
|
+
// (uuidGroup, dropped in beginFile) is exactly the right one to ask. First
|
|
564
|
+
// occurrence wins — file order is event order, and a later copy carrying a
|
|
565
|
+
// different promptId is the copy, not the event.
|
|
566
|
+
//
|
|
567
|
+
// Counted, never silent: a run over a transcript that suddenly stops
|
|
568
|
+
// repeating itself must show that too.
|
|
569
|
+
if (typeof entry.uuid === 'string' && uuidGroup.has(entry.uuid)) {
|
|
570
|
+
totals.duplicates++;
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
|
|
392
574
|
const group = groupFor(entry);
|
|
393
575
|
// Memoize BEFORE any type filtering: unknown types are links in the chain
|
|
394
576
|
// even when they carry no metric of their own.
|
|
@@ -405,11 +587,64 @@ export function createCollector() {
|
|
|
405
587
|
const model = entry.message?.model ?? null;
|
|
406
588
|
const synthetic = isAssistant && model === SYNTHETIC_MODEL;
|
|
407
589
|
|
|
590
|
+
// Decided here, not inside applyAssistantEntry, because the unattributed
|
|
591
|
+
// branch below bills its own turns and tokens and never reaches that
|
|
592
|
+
// function — the same response split across blocks would inflate the one
|
|
593
|
+
// counter whose job is to report what fell outside the rows.
|
|
594
|
+
//
|
|
595
|
+
// `message.id` is the identity the API gives a response; `requestId` is the
|
|
596
|
+
// fallback for an entry that lacks it. Measured: the two are 1:1 (no id maps
|
|
597
|
+
// to two of the other, in either direction) and no entry has neither, so the
|
|
598
|
+
// fallback is belt-and-braces rather than load-bearing. An entry with NO id
|
|
599
|
+
// at all is counted rather than assumed unique — an unidentifiable response
|
|
600
|
+
// is not silently a new one — and reported via `responsesUnidentified`.
|
|
601
|
+
//
|
|
602
|
+
// Synthetic turns never enter the set: they are excluded from every billing
|
|
603
|
+
// figure anyway, and 8 of them carry a message.id (0 colliding with a real
|
|
604
|
+
// response) that would otherwise poison it.
|
|
605
|
+
let repeatOfResponse = false;
|
|
606
|
+
if (isAssistant && !synthetic) {
|
|
607
|
+
// Non-empty strings only, the same test groupFor() applies to promptId.
|
|
608
|
+
// `??` alone would accept `id: ''`, and an empty id shared by every entry
|
|
609
|
+
// in a file would collapse the whole file into ONE response — the bill
|
|
610
|
+
// would fall to near zero and read as a clean result. Under-counting is
|
|
611
|
+
// the worse direction: an over-count announces itself as implausible.
|
|
612
|
+
const id = entry.message?.id;
|
|
613
|
+
const req = entry.requestId;
|
|
614
|
+
const responseId =
|
|
615
|
+
(typeof id === 'string' && id !== '' && id) || (typeof req === 'string' && req !== '' && req) || null;
|
|
616
|
+
if (responseId === null) totals.responsesUnidentified++;
|
|
617
|
+
else if (seenResponses.has(responseId)) {
|
|
618
|
+
repeatOfResponse = true;
|
|
619
|
+
totals.responseRepeats++;
|
|
620
|
+
} else seenResponses.add(responseId);
|
|
621
|
+
}
|
|
622
|
+
|
|
408
623
|
if (group === null) {
|
|
409
624
|
unattributed.entries++;
|
|
410
|
-
|
|
625
|
+
// Per ENTRY, above the repeat guard and outside the synthetic test —
|
|
626
|
+
// exactly where the row path counts them, and for its reasons: tool_use
|
|
627
|
+
// blocks are NOT duplicated across the entries of one response
|
|
628
|
+
// (TASK-092), and a synthetic turn's tool call still happened even though
|
|
629
|
+
// nobody billed for it.
|
|
630
|
+
if (isAssistant) {
|
|
631
|
+
for (const block of toolUseBlocks(entry)) {
|
|
632
|
+
unattributed.tool_calls++;
|
|
633
|
+
if (AGENT_TOOLS.has(block.name)) unattributed.agent_spawns++;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (isAssistant && !synthetic && !repeatOfResponse) {
|
|
411
637
|
unattributed.turns++;
|
|
412
|
-
if (usage)
|
|
638
|
+
if (usage) {
|
|
639
|
+
addUsage(unattributed, usage);
|
|
640
|
+
// Under the guard, the mirror image of the tool calls above:
|
|
641
|
+
// server_tool_use is part of `usage`, so it IS copied onto every entry
|
|
642
|
+
// of a split response, and counting it per entry would re-open
|
|
643
|
+
// TASK-092 in the one place we just closed it.
|
|
644
|
+
const server = serverToolRequests(usage);
|
|
645
|
+
unattributed.web_search_requests += server.search;
|
|
646
|
+
unattributed.web_fetch_requests += server.fetch;
|
|
647
|
+
}
|
|
413
648
|
}
|
|
414
649
|
return;
|
|
415
650
|
}
|
|
@@ -424,9 +659,21 @@ export function createCollector() {
|
|
|
424
659
|
|
|
425
660
|
applyIdentity(p, entry);
|
|
426
661
|
applyTiming(p, entry, isAssistant);
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
662
|
+
// Guarded like api_turns, and for the same reason: this is a TURN counter —
|
|
663
|
+
// it is the figure that says how big the subagent blind spot is (error class
|
|
664
|
+
// 3), paired with agent_spawns. A subagent response splits across content
|
|
665
|
+
// blocks like any other, and being tool-heavy it splits MORE, so counting it
|
|
666
|
+
// per entry would inflate the very number that measures what we cannot see.
|
|
667
|
+
// Zero isSidechain entries in all data measured so far, so this moves no
|
|
668
|
+
// current figure — it stops the bug from arriving with the first one.
|
|
669
|
+
if (entry.isSidechain === true && isAssistant && !repeatOfResponse) p.sidechainTurns++;
|
|
670
|
+
if (isAssistant) applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse });
|
|
671
|
+
else {
|
|
672
|
+
applyUserEntry(p, entry);
|
|
673
|
+
// Tool results come back as `user` entries; the delegation's bill rides on
|
|
674
|
+
// the one belonging to an `Agent` call, priced at the moment it happened.
|
|
675
|
+
applyDelegation(p, entry, msOf(entry));
|
|
676
|
+
}
|
|
430
677
|
}
|
|
431
678
|
|
|
432
679
|
function addLine(line) {
|
|
@@ -486,6 +733,13 @@ export function createCollector() {
|
|
|
486
733
|
tool_calls_total: p.toolCalls,
|
|
487
734
|
agent_spawns: p.agentSpawns,
|
|
488
735
|
sidechain_turns: p.sidechainTurns,
|
|
736
|
+
// Deliberately NOT folded into the row's tokens or cost: what the
|
|
737
|
+
// subagents cost is a lower bound (see applyDelegation), and a lower bound
|
|
738
|
+
// added into a total stops being labelled as one. Sorted so two runs over
|
|
739
|
+
// the same transcripts produce byte-identical rows.
|
|
740
|
+
delegations: [...p.delegations.values()].sort((a, b) => a.agent_id.localeCompare(b.agent_id)),
|
|
741
|
+
delegation_duplicates_dropped: p.delegationDuplicates,
|
|
742
|
+
delegations_unidentified: p.delegationsUnidentified,
|
|
489
743
|
multi_attempt_turns: p.multiAttemptTurns,
|
|
490
744
|
understated_turns: p.understatedTurns,
|
|
491
745
|
web_search_requests: p.webSearchRequests,
|
|
@@ -524,10 +778,18 @@ export function createCollector() {
|
|
|
524
778
|
unattributed,
|
|
525
779
|
tokens,
|
|
526
780
|
cost: summarizeCost(rows),
|
|
781
|
+
// Rolled up with the SAME summarizer as the rows — a delegation record
|
|
782
|
+
// carries the cost fields summarizeCost reads, so the lower bound is
|
|
783
|
+
// built by the code that already knows how to refuse an unpriced row.
|
|
784
|
+
// Kept beside `cost`, never inside it (TASK-089 criterion 8b).
|
|
785
|
+
delegationCost: summarizeCost(rows.flatMap((r) => r.delegations)),
|
|
527
786
|
totals: {
|
|
528
787
|
...totals,
|
|
529
788
|
prompts: rows.length,
|
|
530
789
|
agent_spawns: sum('agent_spawns'),
|
|
790
|
+
delegations_measured: rows.reduce((n, r) => n + r.delegations.length, 0),
|
|
791
|
+
delegation_duplicates_dropped: sum('delegation_duplicates_dropped'),
|
|
792
|
+
delegations_unidentified: sum('delegations_unidentified'),
|
|
531
793
|
sidechain_turns: sum('sidechain_turns'),
|
|
532
794
|
synthetic_turns: sum('synthetic_turns'),
|
|
533
795
|
multi_attempt_turns: sum('multi_attempt_turns'),
|
|
@@ -557,23 +819,89 @@ export function formatRow(row) {
|
|
|
557
819
|
* the uncaptured cost of delegation — because a summary that only reports what
|
|
558
820
|
* it measured is how a blind spot becomes a clean-looking result.
|
|
559
821
|
*/
|
|
560
|
-
export function formatSummary({ totals, unattributed, cost, tokens }) {
|
|
822
|
+
export function formatSummary({ totals, unattributed, cost, tokens, delegationCost }) {
|
|
561
823
|
const L = [];
|
|
562
824
|
L.push(
|
|
563
825
|
`prompt-metrics: ${totals.prompts} prompt(s) from ${totals.entries} entries` +
|
|
564
826
|
` in ${totals.files} transcript(s)` +
|
|
565
827
|
(totals.malformed > 0 ? `; ${totals.malformed} malformed line(s) skipped` : ''),
|
|
566
828
|
);
|
|
829
|
+
if (totals.duplicates > 0) {
|
|
830
|
+
// Loud, because the figure below is ~1.8x smaller than it used to be and a
|
|
831
|
+
// reader comparing two runs of this tool deserves to know why.
|
|
832
|
+
L.push(
|
|
833
|
+
` repeated entries skipped: ${totals.duplicates} (same uuid, already seen in this file)` +
|
|
834
|
+
' — counting them charged the same tokens twice',
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
if (totals.responseRepeats > 0) {
|
|
838
|
+
// Its own line, never folded into the uuid count above: the two are
|
|
839
|
+
// different defects with different fixes, and this one halves the bill. A
|
|
840
|
+
// reader comparing this run against a pre-TASK-092 run must be able to see
|
|
841
|
+
// which of the two moved the number.
|
|
842
|
+
L.push(
|
|
843
|
+
` extra entries of the same response: ${totals.responseRepeats}` +
|
|
844
|
+
' (same message.id — one API response split across content blocks; usage counted once)',
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
if (totals.responsesUnidentified > 0) {
|
|
848
|
+
// Counted, not assumed unique: an entry with neither id could be a fresh
|
|
849
|
+
// response or the fourth copy of one already billed. We charge it (an
|
|
850
|
+
// unknown response is never priced at zero) and say so.
|
|
851
|
+
L.push(
|
|
852
|
+
` assistant entries with no message.id and no requestId: ${totals.responsesUnidentified}` +
|
|
853
|
+
' — billed as distinct responses; may double-count if they are split blocks',
|
|
854
|
+
);
|
|
855
|
+
}
|
|
567
856
|
L.push(
|
|
568
857
|
` unattributed entries: ${unattributed.entries}` +
|
|
569
858
|
` (${unattributed.turns} API turn(s), ${unattributed.output_tokens} output tokens) — not in any row`,
|
|
570
859
|
);
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
860
|
+
// Its own line, and only the parts that are non-zero — a clean run stays
|
|
861
|
+
// clean, and so does a run whose only gap is one of the three. Split from the
|
|
862
|
+
// line above because the halves are different kinds of gap: the tool calls are
|
|
863
|
+
// missing from an adoption DENOMINATOR, the searches are missing DOLLARS, and
|
|
864
|
+
// dollars that appear in no cost figure this tool prints, here or below.
|
|
865
|
+
const outside = [];
|
|
866
|
+
if (unattributed.tool_calls > 0) {
|
|
867
|
+
outside.push(
|
|
868
|
+
`${unattributed.tool_calls} tool call(s)` +
|
|
869
|
+
(unattributed.agent_spawns > 0
|
|
870
|
+
? `, ${unattributed.agent_spawns} of them subagent spawn(s) whose cost is UNCAPTURED (class 3)`
|
|
871
|
+
: ''),
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
if (unattributed.web_search_requests > 0) {
|
|
875
|
+
outside.push(
|
|
876
|
+
`${unattributed.web_search_requests} web search(es) =` +
|
|
877
|
+
` ${formatUsd(webSearchCost(unattributed.web_search_requests))} NOT in any cost figure below`,
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
if (unattributed.web_fetch_requests > 0) outside.push(`${unattributed.web_fetch_requests} web fetch(es) (free)`);
|
|
881
|
+
if (outside.length > 0) L.push(` also outside every row: ${outside.join(', ')}`);
|
|
882
|
+
L.push(` subagent spawns: ${totals.agent_spawns}; sidechain turns in transcript: ${totals.sidechain_turns}`);
|
|
883
|
+
if (totals.delegations_measured > 0) {
|
|
884
|
+
// Two numbers, never one: what we could price, and how far the spawns
|
|
885
|
+
// outrun it. A spawn with no result entry is cost we have NO figure for.
|
|
886
|
+
const missing = Math.max(0, totals.agent_spawns - totals.delegations_measured);
|
|
887
|
+
L.push(
|
|
888
|
+
` delegations priced from their own usage: ${totals.delegations_measured}` +
|
|
889
|
+
` (${totals.delegation_duplicates_dropped} duplicate entr(ies) dropped` +
|
|
890
|
+
`, ${totals.delegations_unidentified} without an agentId, not counted)`,
|
|
891
|
+
);
|
|
892
|
+
if (delegationCost) {
|
|
893
|
+
L.push(
|
|
894
|
+
` subagent LOWER BOUND: ${formatUsd(delegationCost.list_cost_usd)} over` +
|
|
895
|
+
` ${delegationCost.priced_rows} of ${delegationCost.total_rows} delegation(s)` +
|
|
896
|
+
' — separate from the figure above, NEVER added to it (procedure §11a)',
|
|
897
|
+
);
|
|
898
|
+
}
|
|
899
|
+
if (missing > 0) {
|
|
900
|
+
L.push(` ${missing} spawn(s) left no priced result — their cost is UNCAPTURED, not zero (EPIC-009 class 3)`);
|
|
901
|
+
}
|
|
902
|
+
} else if (totals.agent_spawns > 0) {
|
|
903
|
+
L.push(' subagent cost is UNCAPTURED, not zero (EPIC-009 class 3)');
|
|
904
|
+
}
|
|
577
905
|
if (totals.synthetic_turns > 0) {
|
|
578
906
|
L.push(` <synthetic> turns excluded from tokens/turns: ${totals.synthetic_turns}`);
|
|
579
907
|
}
|
|
@@ -713,6 +1041,6 @@ async function main() {
|
|
|
713
1041
|
if (arms !== null) process.stderr.write(`${arms}\n`);
|
|
714
1042
|
}
|
|
715
1043
|
|
|
716
|
-
if (process.argv[1]
|
|
1044
|
+
if (process.argv[1]?.endsWith('prompt-metrics.mjs')) {
|
|
717
1045
|
await main();
|
|
718
1046
|
}
|