@bglocation/tune-context 2.4.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +195 -0
- package/README.md +211 -18
- package/bin/tune-context +1 -1
- package/bin/tune-context.mjs +167 -51
- package/cli/archive.mjs +322 -0
- package/cli/entry-point.mjs +73 -0
- package/cli/experiment.mjs +35 -3
- package/cli/version.mjs +118 -0
- package/eval/adoption-report.mjs +11 -5
- package/eval/analyze.mjs +792 -0
- package/eval/arm.mjs +9 -5
- package/eval/pricing.mjs +32 -1
- package/eval/prompt-metrics.mjs +395 -46
- package/eval/report.mjs +881 -0
- package/eval/verdict.mjs +963 -0
- package/hooks/adoption-log.mjs +1 -1
- package/hooks/context-reminder.mjs +1 -1
- package/hooks/pre-compact-snapshot.mjs +4 -4
- package/hooks/session-start-reinject.mjs +1 -4
- package/package.json +7 -2
- package/skills/token-efficiency/SKILL.md +1 -1
- package/skills/tune-context/SKILL.md +4 -2
package/eval/prompt-metrics.mjs
CHANGED
|
@@ -20,11 +20,14 @@
|
|
|
20
20
|
// inside list_cost_usd. Web search costs $10/1000; web FETCH is free. Both
|
|
21
21
|
// counts ride on the row. Zero on all data measured so far.
|
|
22
22
|
// • Subagent turns are NOT in the parent transcript. Measured 2026-07-30:
|
|
23
|
-
// 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries.
|
|
24
|
-
//
|
|
25
|
-
// the
|
|
26
|
-
//
|
|
27
|
-
//
|
|
23
|
+
// 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries. But the
|
|
24
|
+
// RESULT of an `Agent` call carries its own `usage` and `resolvedModel`, so
|
|
25
|
+
// the delegation is measurable FROM BELOW — see applyDelegation(). The row
|
|
26
|
+
// therefore reports two figures that are never added together: the parent
|
|
27
|
+
// session's cost, and `delegations[].list_cost_usd`, a lower bound on what
|
|
28
|
+
// the subagents cost (COST_EVALUATION_PROCEDURE §11a). `agent_spawns` and
|
|
29
|
+
// `sidechain_turns` are still counted, so the size of the remaining blind
|
|
30
|
+
// spot stays visible (EPIC-009, error class 3).
|
|
28
31
|
// • `<synthetic>` assistant entries are generated by Claude Code, not billed
|
|
29
32
|
// by the API. They are excluded from tokens/turns and counted separately —
|
|
30
33
|
// an unbillable entry, not TASK-086's "unknown model".
|
|
@@ -40,9 +43,11 @@ import { createInterface } from 'node:readline';
|
|
|
40
43
|
import { join } from 'node:path';
|
|
41
44
|
import { parseArgs } from 'node:util';
|
|
42
45
|
import { claudeBaseDir } from '../hooks/config-dir.mjs';
|
|
43
|
-
import {
|
|
46
|
+
import { endQuietlyOnClosedPipe, isEntryPoint } from '../cli/entry-point.mjs';
|
|
47
|
+
import { readStepLedger } from '../cli/experiment.mjs';
|
|
48
|
+
import { meterStamp } from '../cli/version.mjs';
|
|
44
49
|
import { annotateRows, formatArmSummary } from './arm.mjs';
|
|
45
|
-
import { formatCostLines, priceRow, summarizeCost } from './pricing.mjs';
|
|
50
|
+
import { formatCostLines, formatUsd, priceRow, summarizeCost, webSearchCost } from './pricing.mjs';
|
|
46
51
|
|
|
47
52
|
// Tool names that spawn a subagent. `Agent` is what this Claude Code writes
|
|
48
53
|
// (measured); `Task` is the name the same tool carried in earlier versions.
|
|
@@ -58,7 +63,7 @@ const USAGE = `
|
|
|
58
63
|
prompt-metrics — per-prompt time/token/tool metrics from Claude Code transcripts.
|
|
59
64
|
|
|
60
65
|
Usage:
|
|
61
|
-
prompt-metrics [--transcript <path>]... [--dir <path>]
|
|
66
|
+
prompt-metrics [--transcript <path>]... [--dir <path>] [--ledger <path>]
|
|
62
67
|
|
|
63
68
|
Options:
|
|
64
69
|
--transcript <path> A single .jsonl transcript. Repeatable.
|
|
@@ -67,6 +72,12 @@ Options:
|
|
|
67
72
|
(default when neither is given: <config dir>/projects,
|
|
68
73
|
where <config dir> is $CLAUDE_CONFIG_DIR if set,
|
|
69
74
|
else ~/.claude)
|
|
75
|
+
--ledger <path> The experiment ledger to assign arms by (default:
|
|
76
|
+
<config dir>/tune-context/experiment.jsonl). For
|
|
77
|
+
recomputing an archive — \`tune-context experiment
|
|
78
|
+
archive\` keeps the ledger next to the transcripts. A
|
|
79
|
+
path that does not exist, or is not a file, is an
|
|
80
|
+
error.
|
|
70
81
|
-h, --help Show this help
|
|
71
82
|
|
|
72
83
|
Writes JSONL to stdout, one row per prompt. A run summary — including how many
|
|
@@ -86,6 +97,16 @@ leaves the arm unresolved rather than picking a winner.
|
|
|
86
97
|
These five fields are added here, in the CLI. Rows obtained through the module
|
|
87
98
|
(aggregate, collectFiles) carry the measurement only — call annotateRows from
|
|
88
99
|
eval/arm.mjs to add the arm.
|
|
100
|
+
|
|
101
|
+
Every row also carries meter_version: the code that computed it — the bare
|
|
102
|
+
release number for an npm install, number+<sha> for a checkout, .dirty for
|
|
103
|
+
uncommitted changes in the meter's paths (EPIC-010). The first line of the
|
|
104
|
+
summary on stderr is that stamp, as \`meter <stamp>\`: it goes into the run
|
|
105
|
+
journal before the first pair. analyze refuses rows without it.
|
|
106
|
+
|
|
107
|
+
Each row also carries delegations[]: one record per Agent call that returned its
|
|
108
|
+
own usage, priced with the same table. That figure is a LOWER BOUND on what the
|
|
109
|
+
subagents cost and is never added to the row's own cost.
|
|
89
110
|
`.trim();
|
|
90
111
|
|
|
91
112
|
/** Parse one transcript line. Returns null for blanks, junk, or non-objects. */
|
|
@@ -168,6 +189,33 @@ function addUsage(into, usage) {
|
|
|
168
189
|
into.output_tokens += usage.output_tokens || 0;
|
|
169
190
|
}
|
|
170
191
|
|
|
192
|
+
/** The `tool_use` blocks an assistant entry carries, in block order. One reader
|
|
193
|
+
* for the row path and the unattributed bucket both: what counts as a tool call
|
|
194
|
+
* must not quietly become two slightly different answers. */
|
|
195
|
+
function toolUseBlocks(entry) {
|
|
196
|
+
const content = Array.isArray(entry.message?.content) ? entry.message.content : [];
|
|
197
|
+
return content.filter((block) => block?.type === 'tool_use');
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Server-side tool requests carried by a usage block, as plain counts.
|
|
201
|
+
*
|
|
202
|
+
* Deliberately NOT folded into addUsage(): every field that function touches is
|
|
203
|
+
* a token class priced PER TOKEN, and these are billed PER REQUEST, which is
|
|
204
|
+
* why they sit outside list_cost_by_class on a row. addUsage also feeds the
|
|
205
|
+
* usage.iterations comparison (where a request count would be compared against
|
|
206
|
+
* token counts) and the per-model buckets published as `by_model` — a field
|
|
207
|
+
* added there would surface in a row's token breakdown, which its own test
|
|
208
|
+
* pins to exactly the five priced classes plus the unsplit one. So the shape of
|
|
209
|
+
* `usage.server_tool_use` gets ONE reader instead, and the two accumulators
|
|
210
|
+
* that need it keep their own field names. */
|
|
211
|
+
function serverToolRequests(usage) {
|
|
212
|
+
const server = usage?.server_tool_use;
|
|
213
|
+
return {
|
|
214
|
+
search: server?.web_search_requests || 0,
|
|
215
|
+
fetch: server?.web_fetch_requests || 0,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
171
219
|
function newPrompt({ promptId, session }) {
|
|
172
220
|
return {
|
|
173
221
|
promptId,
|
|
@@ -207,6 +255,12 @@ function newPrompt({ promptId, session }) {
|
|
|
207
255
|
toolCalls: 0,
|
|
208
256
|
agentSpawns: 0,
|
|
209
257
|
sidechainTurns: 0,
|
|
258
|
+
// Keyed by agentId, because the same delegation result is written to
|
|
259
|
+
// SEVERAL transcript files (measured: 8 entries for 4 delegations). A list
|
|
260
|
+
// here would double the only figure we have for subagent cost.
|
|
261
|
+
delegations: new Map(),
|
|
262
|
+
delegationDuplicates: 0,
|
|
263
|
+
delegationsUnidentified: 0,
|
|
210
264
|
promptChars: null,
|
|
211
265
|
};
|
|
212
266
|
}
|
|
@@ -292,10 +346,86 @@ function applyUserEntry(p, entry) {
|
|
|
292
346
|
}
|
|
293
347
|
}
|
|
294
348
|
|
|
349
|
+
/**
|
|
350
|
+
* The subagent's own bill, off the `Agent` tool RESULT.
|
|
351
|
+
*
|
|
352
|
+
* A delegation leaves no turns in the parent transcript (0 `isSidechain`
|
|
353
|
+
* entries over the whole of `~/.claude/projects`), but the result entry carries
|
|
354
|
+
* `toolUseResult.usage` — the same structure as `message.usage`, cache split
|
|
355
|
+
* included — plus `resolvedModel`. So it is priced with the SAME `priceRow`
|
|
356
|
+
* that prices the parent turn: TASK-089 criterion 8 asks for no new pricing
|
|
357
|
+
* logic, and a second code path to dollars is a second set of prices.
|
|
358
|
+
*
|
|
359
|
+
* Three rules this obeys, all from COST_EVALUATION_PROCEDURE §11a:
|
|
360
|
+
*
|
|
361
|
+
* • **Deduplicated by `agentId`.** The result entry is written twice (8
|
|
362
|
+
* entries = 4 delegations, measured) — and, measured 2026-08-02, WITHIN one
|
|
363
|
+
* file rather than across files as first assumed, sometimes under a
|
|
364
|
+
* different `promptId`, which puts one delegation on two rows. The entry
|
|
365
|
+
* dedup in addEntry() removes the byte-identical copies; this keeps the
|
|
366
|
+
* guarantee for the ones it cannot see, i.e. copies with distinct uuids and
|
|
367
|
+
* rows collected from more than one file.
|
|
368
|
+
* • **A lower bound, not a cost.** `cache_read` grows LINEARLY with the
|
|
369
|
+
* subagent's tool-call count (13 → 37.6k, 19 → 59.5k, 26 → 76.5k), which is
|
|
370
|
+
* what one turn reading the whole accumulated context looks like — not a
|
|
371
|
+
* sum over turns, which would grow quadratically. So this is most likely
|
|
372
|
+
* the subagent's LAST turn. It rides on its own field and is never added to
|
|
373
|
+
* the row's cost.
|
|
374
|
+
* • **A result with no `agentId` is dropped, not guessed.** It cannot be
|
|
375
|
+
* deduplicated, and a double-counted lower bound is worse than a smaller
|
|
376
|
+
* one that is honest. Counted, so the omission is visible.
|
|
377
|
+
*/
|
|
378
|
+
function applyDelegation(p, entry, tsMs) {
|
|
379
|
+
const result = entry.toolUseResult;
|
|
380
|
+
if (!result || typeof result !== 'object' || Array.isArray(result)) return;
|
|
381
|
+
const usage = result.usage;
|
|
382
|
+
if (!usage || typeof usage !== 'object') return;
|
|
383
|
+
|
|
384
|
+
const id = typeof result.agentId === 'string' ? result.agentId : '';
|
|
385
|
+
if (id === '') {
|
|
386
|
+
p.delegationsUnidentified++;
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (p.delegations.has(id)) {
|
|
390
|
+
p.delegationDuplicates++;
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const tokens = emptyTokens();
|
|
395
|
+
addUsage(tokens, usage);
|
|
396
|
+
const model = typeof result.resolvedModel === 'string' ? result.resolvedModel : '';
|
|
397
|
+
// Shaped as a one-model row so priceRow answers about it unchanged — including
|
|
398
|
+
// the refusals: an unknown model, an unsplit cache write or a tier we cannot
|
|
399
|
+
// price yields `list_cost_usd: null` here for exactly the same reasons as on a
|
|
400
|
+
// parent turn, and lands in the same unpriced counter downstream.
|
|
401
|
+
const cost = priceRow(
|
|
402
|
+
{
|
|
403
|
+
by_model: { [model]: tokens },
|
|
404
|
+
speed: typeof usage.speed === 'string' ? usage.speed : null,
|
|
405
|
+
service_tier: usage.service_tier,
|
|
406
|
+
inference_geo: usage.inference_geo,
|
|
407
|
+
},
|
|
408
|
+
{ at: tsMs },
|
|
409
|
+
);
|
|
410
|
+
|
|
411
|
+
p.delegations.set(id, {
|
|
412
|
+
agent_id: id,
|
|
413
|
+
agent_type: typeof result.agentType === 'string' ? result.agentType : '',
|
|
414
|
+
model,
|
|
415
|
+
status: typeof result.status === 'string' ? result.status : '',
|
|
416
|
+
duration_ms: Number.isFinite(result.totalDurationMs) ? result.totalDurationMs : null,
|
|
417
|
+
// The subagent's own tool calls — the quantity `cache_read` tracks linearly,
|
|
418
|
+
// i.e. the evidence for reading the figure above as a lower bound.
|
|
419
|
+
tool_calls_total: Number.isFinite(result.totalToolUseCount) ? result.totalToolUseCount : null,
|
|
420
|
+
...tokens,
|
|
421
|
+
web_search_requests: usage.server_tool_use?.web_search_requests || 0,
|
|
422
|
+
...cost,
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
|
|
295
426
|
/** Tools, delegations, model/effort, and the per-model token split. */
|
|
296
|
-
function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
297
|
-
for (const block of
|
|
298
|
-
if (block?.type !== 'tool_use') continue;
|
|
427
|
+
function applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse }) {
|
|
428
|
+
for (const block of toolUseBlocks(entry)) {
|
|
299
429
|
const name = typeof block.name === 'string' ? block.name : '(unknown)';
|
|
300
430
|
p.tools.set(name, (p.tools.get(name) ?? 0) + 1);
|
|
301
431
|
p.toolCalls++;
|
|
@@ -309,6 +439,26 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
|
309
439
|
return;
|
|
310
440
|
}
|
|
311
441
|
|
|
442
|
+
// `usage` describes the RESPONSE, not the entry. One API response reaches the
|
|
443
|
+
// transcript as several entries — one per content block (thinking / text /
|
|
444
|
+
// each tool_use) — and every one of them carries an identical copy of the same
|
|
445
|
+
// usage block. The uuid dedup in addEntry() cannot see this class: those uuids
|
|
446
|
+
// differ by definition. Measured over 19 transcripts: 11 295 entries for 5 954
|
|
447
|
+
// responses = 1.90x on api_turns, and 2.02x on the dollar figure, because the
|
|
448
|
+
// per-class inflation is not flat (cache-read 1.82x, output 2.17x) — a
|
|
449
|
+
// many-block response is not a scaled copy of a one-block response.
|
|
450
|
+
//
|
|
451
|
+
// Not a constant that cancels in a ratio of two arms: the factor IS the number
|
|
452
|
+
// of content blocks per turn, i.e. how many tools the model reached for, which
|
|
453
|
+
// is the variable the doctrine sets out to move. Per-session spread on one
|
|
454
|
+
// project: 1.66x to 2.24x (error class 1 — a figure that prints like a fact).
|
|
455
|
+
//
|
|
456
|
+
// The tool_use loop above runs BEFORE this guard on purpose: tool_use blocks
|
|
457
|
+
// are NOT duplicated across a response's entries (6 403 blocks, 6 403 distinct
|
|
458
|
+
// ids, 0 repeats), so skipping the whole entry would drop ~2/3 of all tool
|
|
459
|
+
// calls and gut the adoption half of the analysis.
|
|
460
|
+
if (repeatOfResponse) return;
|
|
461
|
+
|
|
312
462
|
p.apiTurns++;
|
|
313
463
|
if (model) p.models.add(model);
|
|
314
464
|
if (typeof entry.effort === 'string') p.efforts.add(entry.effort);
|
|
@@ -329,11 +479,9 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
|
|
|
329
479
|
if (usageUnderstated(usage)) p.understatedTurns++;
|
|
330
480
|
// Billed per request, not per token, so they sit outside list_cost_by_class.
|
|
331
481
|
// Counted so the gap announces itself instead of hiding in a clean total.
|
|
332
|
-
const server = usage
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
p.webFetchRequests += server.web_fetch_requests || 0;
|
|
336
|
-
}
|
|
482
|
+
const server = serverToolRequests(usage);
|
|
483
|
+
p.webSearchRequests += server.search;
|
|
484
|
+
p.webFetchRequests += server.fetch;
|
|
337
485
|
}
|
|
338
486
|
|
|
339
487
|
/**
|
|
@@ -365,8 +513,34 @@ export function createCollector() {
|
|
|
365
513
|
const prompts = new Map(); // key -> accumulator, in first-seen order
|
|
366
514
|
const groups = new Map(); // key -> {key, promptId, session}, shared by memoized uuids
|
|
367
515
|
let uuidGroup = new Map(); // uuid -> group | null; reset per file (see beginFile)
|
|
368
|
-
const unattributed = {
|
|
369
|
-
|
|
516
|
+
const unattributed = {
|
|
517
|
+
entries: 0,
|
|
518
|
+
turns: 0,
|
|
519
|
+
// Counted for the same reason the rows count them: tool_calls is an
|
|
520
|
+
// ADOPTION metric — how often the model reached for a tool, i.e. the very
|
|
521
|
+
// variable the doctrine sets out to move — so an entry that lands outside
|
|
522
|
+
// every row must still show up in that denominator. `web_search_requests`
|
|
523
|
+
// is a different kind of thing: it is MONEY ($10/1000, billed per request),
|
|
524
|
+
// and money that counting the entry's tokens does not cover, because no
|
|
525
|
+
// token class prices it (TASK-093).
|
|
526
|
+
tool_calls: 0,
|
|
527
|
+
// A strict subset of tool_calls, tracked separately for the same reason the
|
|
528
|
+
// rows track it: agent_spawns is one half of how big the subagent blind spot
|
|
529
|
+
// is (error class 3), and a spawn nobody counted is a whole delegation whose
|
|
530
|
+
// cost we do not merely under-report — we never learn it happened.
|
|
531
|
+
agent_spawns: 0,
|
|
532
|
+
web_search_requests: 0,
|
|
533
|
+
web_fetch_requests: 0,
|
|
534
|
+
...emptyTokens(),
|
|
535
|
+
};
|
|
536
|
+
// Identity of every API response already billed. NOT reset in beginFile, and
|
|
537
|
+
// deliberately unlike uuidGroup: a uuid chain cannot cross a file, but nothing
|
|
538
|
+
// promises the same about a response id, and a global set costs one string per
|
|
539
|
+
// response. Measured across the 19 guidetrackee transcripts: 0 of 5 954 ids
|
|
540
|
+
// appear in more than one file, so today the scope changes no figure — it only
|
|
541
|
+
// removes a way for a future transcript layout to reopen the bug silently.
|
|
542
|
+
const seenResponses = new Set();
|
|
543
|
+
const totals = { malformed: 0, entries: 0, duplicates: 0, responseRepeats: 0, responsesUnidentified: 0, files: 0 };
|
|
370
544
|
|
|
371
545
|
function groupFor(entry) {
|
|
372
546
|
// Direct: `user` entries (prompt AND tool results) carry promptId.
|
|
@@ -389,6 +563,28 @@ export function createCollector() {
|
|
|
389
563
|
|
|
390
564
|
function addEntry(entry) {
|
|
391
565
|
totals.entries++;
|
|
566
|
+
|
|
567
|
+
// A transcript repeats entries VERBATIM, and not rarely: 11 296 of 36 463
|
|
568
|
+
// local entries (31%) carry a uuid already written earlier IN THE SAME FILE,
|
|
569
|
+
// 6 739 of them assistant turns with a usage block. The copies are identical
|
|
570
|
+
// apart from a cosmetic `slug` — same tokens, same model, same timestamp —
|
|
571
|
+
// so summing both charged 45% of all cache-read tokens and 42% of all output
|
|
572
|
+
// tokens twice. That is not a rounding error, it is the epic's whole effect
|
|
573
|
+
// size several times over (error class 1: a figure that prints like a fact).
|
|
574
|
+
//
|
|
575
|
+
// Scoped to the file on purpose: across the 34 local transcripts ZERO uuids
|
|
576
|
+
// are shared between files, so the index that already exists per file
|
|
577
|
+
// (uuidGroup, dropped in beginFile) is exactly the right one to ask. First
|
|
578
|
+
// occurrence wins — file order is event order, and a later copy carrying a
|
|
579
|
+
// different promptId is the copy, not the event.
|
|
580
|
+
//
|
|
581
|
+
// Counted, never silent: a run over a transcript that suddenly stops
|
|
582
|
+
// repeating itself must show that too.
|
|
583
|
+
if (typeof entry.uuid === 'string' && uuidGroup.has(entry.uuid)) {
|
|
584
|
+
totals.duplicates++;
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
|
|
392
588
|
const group = groupFor(entry);
|
|
393
589
|
// Memoize BEFORE any type filtering: unknown types are links in the chain
|
|
394
590
|
// even when they carry no metric of their own.
|
|
@@ -405,11 +601,64 @@ export function createCollector() {
|
|
|
405
601
|
const model = entry.message?.model ?? null;
|
|
406
602
|
const synthetic = isAssistant && model === SYNTHETIC_MODEL;
|
|
407
603
|
|
|
604
|
+
// Decided here, not inside applyAssistantEntry, because the unattributed
|
|
605
|
+
// branch below bills its own turns and tokens and never reaches that
|
|
606
|
+
// function — the same response split across blocks would inflate the one
|
|
607
|
+
// counter whose job is to report what fell outside the rows.
|
|
608
|
+
//
|
|
609
|
+
// `message.id` is the identity the API gives a response; `requestId` is the
|
|
610
|
+
// fallback for an entry that lacks it. Measured: the two are 1:1 (no id maps
|
|
611
|
+
// to two of the other, in either direction) and no entry has neither, so the
|
|
612
|
+
// fallback is belt-and-braces rather than load-bearing. An entry with NO id
|
|
613
|
+
// at all is counted rather than assumed unique — an unidentifiable response
|
|
614
|
+
// is not silently a new one — and reported via `responsesUnidentified`.
|
|
615
|
+
//
|
|
616
|
+
// Synthetic turns never enter the set: they are excluded from every billing
|
|
617
|
+
// figure anyway, and 8 of them carry a message.id (0 colliding with a real
|
|
618
|
+
// response) that would otherwise poison it.
|
|
619
|
+
let repeatOfResponse = false;
|
|
620
|
+
if (isAssistant && !synthetic) {
|
|
621
|
+
// Non-empty strings only, the same test groupFor() applies to promptId.
|
|
622
|
+
// `??` alone would accept `id: ''`, and an empty id shared by every entry
|
|
623
|
+
// in a file would collapse the whole file into ONE response — the bill
|
|
624
|
+
// would fall to near zero and read as a clean result. Under-counting is
|
|
625
|
+
// the worse direction: an over-count announces itself as implausible.
|
|
626
|
+
const id = entry.message?.id;
|
|
627
|
+
const req = entry.requestId;
|
|
628
|
+
const responseId =
|
|
629
|
+
(typeof id === 'string' && id !== '' && id) || (typeof req === 'string' && req !== '' && req) || null;
|
|
630
|
+
if (responseId === null) totals.responsesUnidentified++;
|
|
631
|
+
else if (seenResponses.has(responseId)) {
|
|
632
|
+
repeatOfResponse = true;
|
|
633
|
+
totals.responseRepeats++;
|
|
634
|
+
} else seenResponses.add(responseId);
|
|
635
|
+
}
|
|
636
|
+
|
|
408
637
|
if (group === null) {
|
|
409
638
|
unattributed.entries++;
|
|
410
|
-
|
|
639
|
+
// Per ENTRY, above the repeat guard and outside the synthetic test —
|
|
640
|
+
// exactly where the row path counts them, and for its reasons: tool_use
|
|
641
|
+
// blocks are NOT duplicated across the entries of one response
|
|
642
|
+
// (TASK-092), and a synthetic turn's tool call still happened even though
|
|
643
|
+
// nobody billed for it.
|
|
644
|
+
if (isAssistant) {
|
|
645
|
+
for (const block of toolUseBlocks(entry)) {
|
|
646
|
+
unattributed.tool_calls++;
|
|
647
|
+
if (AGENT_TOOLS.has(block.name)) unattributed.agent_spawns++;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
if (isAssistant && !synthetic && !repeatOfResponse) {
|
|
411
651
|
unattributed.turns++;
|
|
412
|
-
if (usage)
|
|
652
|
+
if (usage) {
|
|
653
|
+
addUsage(unattributed, usage);
|
|
654
|
+
// Under the guard, the mirror image of the tool calls above:
|
|
655
|
+
// server_tool_use is part of `usage`, so it IS copied onto every entry
|
|
656
|
+
// of a split response, and counting it per entry would re-open
|
|
657
|
+
// TASK-092 in the one place we just closed it.
|
|
658
|
+
const server = serverToolRequests(usage);
|
|
659
|
+
unattributed.web_search_requests += server.search;
|
|
660
|
+
unattributed.web_fetch_requests += server.fetch;
|
|
661
|
+
}
|
|
413
662
|
}
|
|
414
663
|
return;
|
|
415
664
|
}
|
|
@@ -424,9 +673,21 @@ export function createCollector() {
|
|
|
424
673
|
|
|
425
674
|
applyIdentity(p, entry);
|
|
426
675
|
applyTiming(p, entry, isAssistant);
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
676
|
+
// Guarded like api_turns, and for the same reason: this is a TURN counter —
|
|
677
|
+
// it is the figure that says how big the subagent blind spot is (error class
|
|
678
|
+
// 3), paired with agent_spawns. A subagent response splits across content
|
|
679
|
+
// blocks like any other, and being tool-heavy it splits MORE, so counting it
|
|
680
|
+
// per entry would inflate the very number that measures what we cannot see.
|
|
681
|
+
// Zero isSidechain entries in all data measured so far, so this moves no
|
|
682
|
+
// current figure — it stops the bug from arriving with the first one.
|
|
683
|
+
if (entry.isSidechain === true && isAssistant && !repeatOfResponse) p.sidechainTurns++;
|
|
684
|
+
if (isAssistant) applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse });
|
|
685
|
+
else {
|
|
686
|
+
applyUserEntry(p, entry);
|
|
687
|
+
// Tool results come back as `user` entries; the delegation's bill rides on
|
|
688
|
+
// the one belonging to an `Agent` call, priced at the moment it happened.
|
|
689
|
+
applyDelegation(p, entry, msOf(entry));
|
|
690
|
+
}
|
|
430
691
|
}
|
|
431
692
|
|
|
432
693
|
function addLine(line) {
|
|
@@ -486,6 +747,13 @@ export function createCollector() {
|
|
|
486
747
|
tool_calls_total: p.toolCalls,
|
|
487
748
|
agent_spawns: p.agentSpawns,
|
|
488
749
|
sidechain_turns: p.sidechainTurns,
|
|
750
|
+
// Deliberately NOT folded into the row's tokens or cost: what the
|
|
751
|
+
// subagents cost is a lower bound (see applyDelegation), and a lower bound
|
|
752
|
+
// added into a total stops being labelled as one. Sorted so two runs over
|
|
753
|
+
// the same transcripts produce byte-identical rows.
|
|
754
|
+
delegations: [...p.delegations.values()].sort((a, b) => a.agent_id.localeCompare(b.agent_id)),
|
|
755
|
+
delegation_duplicates_dropped: p.delegationDuplicates,
|
|
756
|
+
delegations_unidentified: p.delegationsUnidentified,
|
|
489
757
|
multi_attempt_turns: p.multiAttemptTurns,
|
|
490
758
|
understated_turns: p.understatedTurns,
|
|
491
759
|
web_search_requests: p.webSearchRequests,
|
|
@@ -524,10 +792,18 @@ export function createCollector() {
|
|
|
524
792
|
unattributed,
|
|
525
793
|
tokens,
|
|
526
794
|
cost: summarizeCost(rows),
|
|
795
|
+
// Rolled up with the SAME summarizer as the rows — a delegation record
|
|
796
|
+
// carries the cost fields summarizeCost reads, so the lower bound is
|
|
797
|
+
// built by the code that already knows how to refuse an unpriced row.
|
|
798
|
+
// Kept beside `cost`, never inside it (TASK-089 criterion 8b).
|
|
799
|
+
delegationCost: summarizeCost(rows.flatMap((r) => r.delegations)),
|
|
527
800
|
totals: {
|
|
528
801
|
...totals,
|
|
529
802
|
prompts: rows.length,
|
|
530
803
|
agent_spawns: sum('agent_spawns'),
|
|
804
|
+
delegations_measured: rows.reduce((n, r) => n + r.delegations.length, 0),
|
|
805
|
+
delegation_duplicates_dropped: sum('delegation_duplicates_dropped'),
|
|
806
|
+
delegations_unidentified: sum('delegations_unidentified'),
|
|
531
807
|
sidechain_turns: sum('sidechain_turns'),
|
|
532
808
|
synthetic_turns: sum('synthetic_turns'),
|
|
533
809
|
multi_attempt_turns: sum('multi_attempt_turns'),
|
|
@@ -557,23 +833,89 @@ export function formatRow(row) {
|
|
|
557
833
|
* the uncaptured cost of delegation — because a summary that only reports what
|
|
558
834
|
* it measured is how a blind spot becomes a clean-looking result.
|
|
559
835
|
*/
|
|
560
|
-
export function formatSummary({ totals, unattributed, cost, tokens }) {
|
|
836
|
+
export function formatSummary({ totals, unattributed, cost, tokens, delegationCost }) {
|
|
561
837
|
const L = [];
|
|
562
838
|
L.push(
|
|
563
839
|
`prompt-metrics: ${totals.prompts} prompt(s) from ${totals.entries} entries` +
|
|
564
840
|
` in ${totals.files} transcript(s)` +
|
|
565
841
|
(totals.malformed > 0 ? `; ${totals.malformed} malformed line(s) skipped` : ''),
|
|
566
842
|
);
|
|
843
|
+
if (totals.duplicates > 0) {
|
|
844
|
+
// Loud, because the figure below is ~1.8x smaller than it used to be and a
|
|
845
|
+
// reader comparing two runs of this tool deserves to know why.
|
|
846
|
+
L.push(
|
|
847
|
+
` repeated entries skipped: ${totals.duplicates} (same uuid, already seen in this file)` +
|
|
848
|
+
' — counting them charged the same tokens twice',
|
|
849
|
+
);
|
|
850
|
+
}
|
|
851
|
+
if (totals.responseRepeats > 0) {
|
|
852
|
+
// Its own line, never folded into the uuid count above: the two are
|
|
853
|
+
// different defects with different fixes, and this one halves the bill. A
|
|
854
|
+
// reader comparing this run against a pre-TASK-092 run must be able to see
|
|
855
|
+
// which of the two moved the number.
|
|
856
|
+
L.push(
|
|
857
|
+
` extra entries of the same response: ${totals.responseRepeats}` +
|
|
858
|
+
' (same message.id — one API response split across content blocks; usage counted once)',
|
|
859
|
+
);
|
|
860
|
+
}
|
|
861
|
+
if (totals.responsesUnidentified > 0) {
|
|
862
|
+
// Counted, not assumed unique: an entry with neither id could be a fresh
|
|
863
|
+
// response or the fourth copy of one already billed. We charge it (an
|
|
864
|
+
// unknown response is never priced at zero) and say so.
|
|
865
|
+
L.push(
|
|
866
|
+
` assistant entries with no message.id and no requestId: ${totals.responsesUnidentified}` +
|
|
867
|
+
' — billed as distinct responses; may double-count if they are split blocks',
|
|
868
|
+
);
|
|
869
|
+
}
|
|
567
870
|
L.push(
|
|
568
871
|
` unattributed entries: ${unattributed.entries}` +
|
|
569
872
|
` (${unattributed.turns} API turn(s), ${unattributed.output_tokens} output tokens) — not in any row`,
|
|
570
873
|
);
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
874
|
+
// Its own line, and only the parts that are non-zero — a clean run stays
|
|
875
|
+
// clean, and so does a run whose only gap is one of the three. Split from the
|
|
876
|
+
// line above because the halves are different kinds of gap: the tool calls are
|
|
877
|
+
// missing from an adoption DENOMINATOR, the searches are missing DOLLARS, and
|
|
878
|
+
// dollars that appear in no cost figure this tool prints, here or below.
|
|
879
|
+
const outside = [];
|
|
880
|
+
if (unattributed.tool_calls > 0) {
|
|
881
|
+
outside.push(
|
|
882
|
+
`${unattributed.tool_calls} tool call(s)` +
|
|
883
|
+
(unattributed.agent_spawns > 0
|
|
884
|
+
? `, ${unattributed.agent_spawns} of them subagent spawn(s) whose cost is UNCAPTURED (class 3)`
|
|
885
|
+
: ''),
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
if (unattributed.web_search_requests > 0) {
|
|
889
|
+
outside.push(
|
|
890
|
+
`${unattributed.web_search_requests} web search(es) =` +
|
|
891
|
+
` ${formatUsd(webSearchCost(unattributed.web_search_requests))} NOT in any cost figure below`,
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
if (unattributed.web_fetch_requests > 0) outside.push(`${unattributed.web_fetch_requests} web fetch(es) (free)`);
|
|
895
|
+
if (outside.length > 0) L.push(` also outside every row: ${outside.join(', ')}`);
|
|
896
|
+
L.push(` subagent spawns: ${totals.agent_spawns}; sidechain turns in transcript: ${totals.sidechain_turns}`);
|
|
897
|
+
if (totals.delegations_measured > 0) {
|
|
898
|
+
// Two numbers, never one: what we could price, and how far the spawns
|
|
899
|
+
// outrun it. A spawn with no result entry is cost we have NO figure for.
|
|
900
|
+
const missing = Math.max(0, totals.agent_spawns - totals.delegations_measured);
|
|
901
|
+
L.push(
|
|
902
|
+
` delegations priced from their own usage: ${totals.delegations_measured}` +
|
|
903
|
+
` (${totals.delegation_duplicates_dropped} duplicate entr(ies) dropped` +
|
|
904
|
+
`, ${totals.delegations_unidentified} without an agentId, not counted)`,
|
|
905
|
+
);
|
|
906
|
+
if (delegationCost) {
|
|
907
|
+
L.push(
|
|
908
|
+
` subagent LOWER BOUND: ${formatUsd(delegationCost.list_cost_usd)} over` +
|
|
909
|
+
` ${delegationCost.priced_rows} of ${delegationCost.total_rows} delegation(s)` +
|
|
910
|
+
' — separate from the figure above, NEVER added to it (procedure §11a)',
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
if (missing > 0) {
|
|
914
|
+
L.push(` ${missing} spawn(s) left no priced result — their cost is UNCAPTURED, not zero (EPIC-009 class 3)`);
|
|
915
|
+
}
|
|
916
|
+
} else if (totals.agent_spawns > 0) {
|
|
917
|
+
L.push(' subagent cost is UNCAPTURED, not zero (EPIC-009 class 3)');
|
|
918
|
+
}
|
|
577
919
|
if (totals.synthetic_turns > 0) {
|
|
578
920
|
L.push(` <synthetic> turns excluded from tokens/turns: ${totals.synthetic_turns}`);
|
|
579
921
|
}
|
|
@@ -645,14 +987,21 @@ function defaultDir() {
|
|
|
645
987
|
return join(claudeBaseDir(), 'projects');
|
|
646
988
|
}
|
|
647
989
|
|
|
648
|
-
|
|
990
|
+
/**
|
|
991
|
+
* The CLI. `args` is everything after the script name, or after
|
|
992
|
+
* `tune-context eval prompt-metrics` — one parser behind both doors (EPIC-010).
|
|
993
|
+
*/
|
|
994
|
+
export async function main(args = process.argv.slice(2)) {
|
|
995
|
+
endQuietlyOnClosedPipe();
|
|
649
996
|
let paths;
|
|
997
|
+
let ledger;
|
|
650
998
|
try {
|
|
651
999
|
const { values } = parseArgs({
|
|
652
|
-
args
|
|
1000
|
+
args,
|
|
653
1001
|
options: {
|
|
654
1002
|
transcript: { type: 'string', multiple: true },
|
|
655
1003
|
dir: { type: 'string' },
|
|
1004
|
+
ledger: { type: 'string' },
|
|
656
1005
|
help: { type: 'boolean', short: 'h', default: false },
|
|
657
1006
|
},
|
|
658
1007
|
strict: true,
|
|
@@ -661,6 +1010,9 @@ async function main() {
|
|
|
661
1010
|
process.stdout.write(`${USAGE}\n`);
|
|
662
1011
|
return;
|
|
663
1012
|
}
|
|
1013
|
+
// Read now, before the transcripts: a --ledger that is not a file stops the
|
|
1014
|
+
// run here, with the same message analyze gives (TASK-098).
|
|
1015
|
+
ledger = readStepLedger(values.ledger, claudeBaseDir());
|
|
664
1016
|
// Both flags together are a UNION, deduplicated, not a precedence rule:
|
|
665
1017
|
// silently ignoring one of two flags the user passed is how a run measures
|
|
666
1018
|
// something other than what was asked for.
|
|
@@ -688,11 +1040,6 @@ async function main() {
|
|
|
688
1040
|
return;
|
|
689
1041
|
}
|
|
690
1042
|
|
|
691
|
-
// Piping into `head` closes stdout early; that is a normal end, not a crash.
|
|
692
|
-
process.stdout.on('error', (e) => {
|
|
693
|
-
if (e.code !== 'EPIPE') throw e;
|
|
694
|
-
});
|
|
695
|
-
|
|
696
1043
|
const collector = await collectFiles(paths);
|
|
697
1044
|
const result = collector.result();
|
|
698
1045
|
|
|
@@ -700,19 +1047,21 @@ async function main() {
|
|
|
700
1047
|
// transcript into rows and has no business knowing about the experiment, and
|
|
701
1048
|
// the ledger is machine state rather than transcript content. The split also
|
|
702
1049
|
// lets the same rules run over rows collected elsewhere (TASK-088).
|
|
703
|
-
const
|
|
704
|
-
const ledgerAvailable = existsSync(ledgerPath);
|
|
705
|
-
const rows = annotateRows(result.rows, {
|
|
706
|
-
ledger: ledgerAvailable ? readLedger(claudeBaseDir()) : [],
|
|
707
|
-
ledgerAvailable,
|
|
708
|
-
});
|
|
1050
|
+
const rows = annotateRows(result.rows, ledger);
|
|
709
1051
|
|
|
710
|
-
|
|
711
|
-
process
|
|
1052
|
+
// Which code computed the rows rides on every row — added here, with the arm,
|
|
1053
|
+
// because it is a property of the process that ran, not of the transcript
|
|
1054
|
+
// (TASK-097). Last on the row, so nothing that already read a row by position
|
|
1055
|
+
// in a diff moves.
|
|
1056
|
+
const stamp = meterStamp();
|
|
1057
|
+
for (const row of rows) process.stdout.write(`${formatRow({ ...row, meter_version: stamp })}\n`);
|
|
1058
|
+
// First line of the summary, on its own: this is what the operator copies
|
|
1059
|
+
// into the run journal before the first pair.
|
|
1060
|
+
process.stderr.write(`meter ${stamp}\n${formatSummary(result)}\n`);
|
|
712
1061
|
const arms = formatArmSummary(rows);
|
|
713
1062
|
if (arms !== null) process.stderr.write(`${arms}\n`);
|
|
714
1063
|
}
|
|
715
1064
|
|
|
716
|
-
if (
|
|
1065
|
+
if (isEntryPoint(import.meta.url)) {
|
|
717
1066
|
await main();
|
|
718
1067
|
}
|