@bglocation/tune-context 2.4.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,11 +20,14 @@
20
20
  // inside list_cost_usd. Web search costs $10/1000; web FETCH is free. Both
21
21
  // counts ride on the row. Zero on all data measured so far.
22
22
  // • Subagent turns are NOT in the parent transcript. Measured 2026-07-30:
23
- // 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries. Their
24
- // cost is UNCAPTURED, not zero. `agent_spawns` is the only trace of it in
25
- // the data, so it is always counted; `sidechain_turns` is counted too, so
26
- // the blind spot closes by itself if a future Claude Code starts writing
27
- // those entries (EPIC-009, error class 3).
23
+ // 29 transcripts, 21 `Agent` tool calls, ZERO isSidechain entries. But the
24
+ // RESULT of an `Agent` call carries its own `usage` and `resolvedModel`, so
25
+ // the delegation is measurable FROM BELOW — see applyDelegation(). The row
26
+ // therefore reports two figures that are never added together: the parent
27
+ // session's cost, and `delegations[].list_cost_usd`, a lower bound on what
28
+ // the subagents cost (COST_EVALUATION_PROCEDURE §11a). `agent_spawns` and
29
+ // `sidechain_turns` are still counted, so the size of the remaining blind
30
+ // spot stays visible (EPIC-009, error class 3).
28
31
  // • `<synthetic>` assistant entries are generated by Claude Code, not billed
29
32
  // by the API. They are excluded from tokens/turns and counted separately —
30
33
  // an unbillable entry, not TASK-086's "unknown model".
@@ -40,9 +43,11 @@ import { createInterface } from 'node:readline';
40
43
  import { join } from 'node:path';
41
44
  import { parseArgs } from 'node:util';
42
45
  import { claudeBaseDir } from '../hooks/config-dir.mjs';
43
- import { experimentPaths, readLedger } from '../cli/experiment.mjs';
46
+ import { endQuietlyOnClosedPipe, isEntryPoint } from '../cli/entry-point.mjs';
47
+ import { readStepLedger } from '../cli/experiment.mjs';
48
+ import { meterStamp } from '../cli/version.mjs';
44
49
  import { annotateRows, formatArmSummary } from './arm.mjs';
45
- import { formatCostLines, priceRow, summarizeCost } from './pricing.mjs';
50
+ import { formatCostLines, formatUsd, priceRow, summarizeCost, webSearchCost } from './pricing.mjs';
46
51
 
47
52
  // Tool names that spawn a subagent. `Agent` is what this Claude Code writes
48
53
  // (measured); `Task` is the name the same tool carried in earlier versions.
@@ -58,7 +63,7 @@ const USAGE = `
58
63
  prompt-metrics — per-prompt time/token/tool metrics from Claude Code transcripts.
59
64
 
60
65
  Usage:
61
- prompt-metrics [--transcript <path>]... [--dir <path>]
66
+ prompt-metrics [--transcript <path>]... [--dir <path>] [--ledger <path>]
62
67
 
63
68
  Options:
64
69
  --transcript <path> A single .jsonl transcript. Repeatable.
@@ -67,6 +72,12 @@ Options:
67
72
  (default when neither is given: <config dir>/projects,
68
73
  where <config dir> is $CLAUDE_CONFIG_DIR if set,
69
74
  else ~/.claude)
75
+ --ledger <path> The experiment ledger to assign arms by (default:
76
+ <config dir>/tune-context/experiment.jsonl). For
77
+ recomputing an archive — \`tune-context experiment
78
+ archive\` keeps the ledger next to the transcripts. A
79
+ path that does not exist, or is not a file, is an
80
+ error.
70
81
  -h, --help Show this help
71
82
 
72
83
  Writes JSONL to stdout, one row per prompt. A run summary — including how many
@@ -86,6 +97,16 @@ leaves the arm unresolved rather than picking a winner.
86
97
  These five fields are added here, in the CLI. Rows obtained through the module
87
98
  (aggregate, collectFiles) carry the measurement only — call annotateRows from
88
99
  eval/arm.mjs to add the arm.
100
+
101
+ Every row also carries meter_version: the code that computed it — the bare
102
+ release number for an npm install, number+<sha> for a checkout, .dirty for
103
+ uncommitted changes in the meter's paths (EPIC-010). The first line of the
104
+ summary on stderr is that stamp, as \`meter <stamp>\`: it goes into the run
105
+ journal before the first pair. analyze refuses rows without it.
106
+
107
+ Each row also carries delegations[]: one record per Agent call that returned its
108
+ own usage, priced with the same table. That figure is a LOWER BOUND on what the
109
+ subagents cost and is never added to the row's own cost.
89
110
  `.trim();
90
111
 
91
112
  /** Parse one transcript line. Returns null for blanks, junk, or non-objects. */
@@ -168,6 +189,33 @@ function addUsage(into, usage) {
168
189
  into.output_tokens += usage.output_tokens || 0;
169
190
  }
170
191
 
192
+ /** The `tool_use` blocks an assistant entry carries, in block order. One reader
193
+ * for the row path and the unattributed bucket both: what counts as a tool call
194
+ * must not quietly become two slightly different answers. */
195
+ function toolUseBlocks(entry) {
196
+ const content = Array.isArray(entry.message?.content) ? entry.message.content : [];
197
+ return content.filter((block) => block?.type === 'tool_use');
198
+ }
199
+
200
+ /** Server-side tool requests carried by a usage block, as plain counts.
201
+ *
202
+ * Deliberately NOT folded into addUsage(): every field that function touches is
203
+ * a token class priced PER TOKEN, and these are billed PER REQUEST, which is
204
+ * why they sit outside list_cost_by_class on a row. addUsage also feeds the
205
+ * usage.iterations comparison (where a request count would be compared against
206
+ * token counts) and the per-model buckets published as `by_model` — a field
207
+ * added there would surface in a row's token breakdown, which its own test
208
+ * pins to exactly the five priced classes plus the unsplit one. So the shape of
209
+ * `usage.server_tool_use` gets ONE reader instead, and the two accumulators
210
+ * that need it keep their own field names. */
211
+ function serverToolRequests(usage) {
212
+ const server = usage?.server_tool_use;
213
+ return {
214
+ search: server?.web_search_requests || 0,
215
+ fetch: server?.web_fetch_requests || 0,
216
+ };
217
+ }
218
+
171
219
  function newPrompt({ promptId, session }) {
172
220
  return {
173
221
  promptId,
@@ -207,6 +255,12 @@ function newPrompt({ promptId, session }) {
207
255
  toolCalls: 0,
208
256
  agentSpawns: 0,
209
257
  sidechainTurns: 0,
258
+ // Keyed by agentId, because the same delegation result is written to
259
+ // SEVERAL transcript files (measured: 8 entries for 4 delegations). A list
260
+ // here would double the only figure we have for subagent cost.
261
+ delegations: new Map(),
262
+ delegationDuplicates: 0,
263
+ delegationsUnidentified: 0,
210
264
  promptChars: null,
211
265
  };
212
266
  }
@@ -292,10 +346,86 @@ function applyUserEntry(p, entry) {
292
346
  }
293
347
  }
294
348
 
349
+ /**
350
+ * The subagent's own bill, off the `Agent` tool RESULT.
351
+ *
352
+ * A delegation leaves no turns in the parent transcript (0 `isSidechain`
353
+ * entries over the whole of `~/.claude/projects`), but the result entry carries
354
+ * `toolUseResult.usage` — the same structure as `message.usage`, cache split
355
+ * included — plus `resolvedModel`. So it is priced with the SAME `priceRow`
356
+ * that prices the parent turn: TASK-089 criterion 8 asks for no new pricing
357
+ * logic, and a second code path to dollars is a second set of prices.
358
+ *
359
+ * Three rules this obeys, all from COST_EVALUATION_PROCEDURE §11a:
360
+ *
361
+ * • **Deduplicated by `agentId`.** The result entry is written twice (8
362
+ * entries = 4 delegations, measured) — and, measured 2026-08-02, WITHIN one
363
+ * file rather than across files as first assumed, sometimes under a
364
+ * different `promptId`, which puts one delegation on two rows. The entry
365
+ * dedup in addEntry() removes the byte-identical copies; this keeps the
366
+ * guarantee for the ones it cannot see, i.e. copies with distinct uuids and
367
+ * rows collected from more than one file.
368
+ * • **A lower bound, not a cost.** `cache_read` grows LINEARLY with the
369
+ * subagent's tool-call count (13 → 37.6k, 19 → 59.5k, 26 → 76.5k), which is
370
+ * what one turn reading the whole accumulated context looks like — not a
371
+ * sum over turns, which would grow quadratically. So this is most likely
372
+ * the subagent's LAST turn. It rides on its own field and is never added to
373
+ * the row's cost.
374
+ * • **A result with no `agentId` is dropped, not guessed.** It cannot be
375
+ * deduplicated, and a double-counted lower bound is worse than a smaller
376
+ * one that is honest. Counted, so the omission is visible.
377
+ */
378
+ function applyDelegation(p, entry, tsMs) {
379
+ const result = entry.toolUseResult;
380
+ if (!result || typeof result !== 'object' || Array.isArray(result)) return;
381
+ const usage = result.usage;
382
+ if (!usage || typeof usage !== 'object') return;
383
+
384
+ const id = typeof result.agentId === 'string' ? result.agentId : '';
385
+ if (id === '') {
386
+ p.delegationsUnidentified++;
387
+ return;
388
+ }
389
+ if (p.delegations.has(id)) {
390
+ p.delegationDuplicates++;
391
+ return;
392
+ }
393
+
394
+ const tokens = emptyTokens();
395
+ addUsage(tokens, usage);
396
+ const model = typeof result.resolvedModel === 'string' ? result.resolvedModel : '';
397
+ // Shaped as a one-model row so priceRow answers about it unchanged — including
398
+ // the refusals: an unknown model, an unsplit cache write or a tier we cannot
399
+ // price yields `list_cost_usd: null` here for exactly the same reasons as on a
400
+ // parent turn, and lands in the same unpriced counter downstream.
401
+ const cost = priceRow(
402
+ {
403
+ by_model: { [model]: tokens },
404
+ speed: typeof usage.speed === 'string' ? usage.speed : null,
405
+ service_tier: usage.service_tier,
406
+ inference_geo: usage.inference_geo,
407
+ },
408
+ { at: tsMs },
409
+ );
410
+
411
+ p.delegations.set(id, {
412
+ agent_id: id,
413
+ agent_type: typeof result.agentType === 'string' ? result.agentType : '',
414
+ model,
415
+ status: typeof result.status === 'string' ? result.status : '',
416
+ duration_ms: Number.isFinite(result.totalDurationMs) ? result.totalDurationMs : null,
417
+ // The subagent's own tool calls — the quantity `cache_read` tracks linearly,
418
+ // i.e. the evidence for reading the figure above as a lower bound.
419
+ tool_calls_total: Number.isFinite(result.totalToolUseCount) ? result.totalToolUseCount : null,
420
+ ...tokens,
421
+ web_search_requests: usage.server_tool_use?.web_search_requests || 0,
422
+ ...cost,
423
+ });
424
+ }
425
+
295
426
  /** Tools, delegations, model/effort, and the per-model token split. */
296
- function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
297
- for (const block of Array.isArray(entry.message?.content) ? entry.message.content : []) {
298
- if (block?.type !== 'tool_use') continue;
427
+ function applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse }) {
428
+ for (const block of toolUseBlocks(entry)) {
299
429
  const name = typeof block.name === 'string' ? block.name : '(unknown)';
300
430
  p.tools.set(name, (p.tools.get(name) ?? 0) + 1);
301
431
  p.toolCalls++;
@@ -309,6 +439,26 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
309
439
  return;
310
440
  }
311
441
 
442
+ // `usage` describes the RESPONSE, not the entry. One API response reaches the
443
+ // transcript as several entries — one per content block (thinking / text /
444
+ // each tool_use) — and every one of them carries an identical copy of the same
445
+ // usage block. The uuid dedup in addEntry() cannot see this class: those uuids
446
+ // differ by definition. Measured over 19 transcripts: 11 295 entries for 5 954
447
+ // responses = 1.90x on api_turns, and 2.02x on the dollar figure, because the
448
+ // per-class inflation is not flat (cache-read 1.82x, output 2.17x) — a
449
+ // many-block response is not a scaled copy of a one-block response.
450
+ //
451
+ // Not a constant that cancels in a ratio of two arms: the factor IS the number
452
+ // of content blocks per turn, i.e. how many tools the model reached for, which
453
+ // is the variable the doctrine sets out to move. Per-session spread on one
454
+ // project: 1.66x to 2.24x (error class 1 — a figure that prints like a fact).
455
+ //
456
+ // The tool_use loop above runs BEFORE this guard on purpose: tool_use blocks
457
+ // are NOT duplicated across a response's entries (6 403 blocks, 6 403 distinct
458
+ // ids, 0 repeats), so skipping the whole entry would drop ~2/3 of all tool
459
+ // calls and gut the adoption half of the analysis.
460
+ if (repeatOfResponse) return;
461
+
312
462
  p.apiTurns++;
313
463
  if (model) p.models.add(model);
314
464
  if (typeof entry.effort === 'string') p.efforts.add(entry.effort);
@@ -329,11 +479,9 @@ function applyAssistantEntry(p, entry, { usage, model, synthetic }) {
329
479
  if (usageUnderstated(usage)) p.understatedTurns++;
330
480
  // Billed per request, not per token, so they sit outside list_cost_by_class.
331
481
  // Counted so the gap announces itself instead of hiding in a clean total.
332
- const server = usage.server_tool_use;
333
- if (server) {
334
- p.webSearchRequests += server.web_search_requests || 0;
335
- p.webFetchRequests += server.web_fetch_requests || 0;
336
- }
482
+ const server = serverToolRequests(usage);
483
+ p.webSearchRequests += server.search;
484
+ p.webFetchRequests += server.fetch;
337
485
  }
338
486
 
339
487
  /**
@@ -365,8 +513,34 @@ export function createCollector() {
365
513
  const prompts = new Map(); // key -> accumulator, in first-seen order
366
514
  const groups = new Map(); // key -> {key, promptId, session}, shared by memoized uuids
367
515
  let uuidGroup = new Map(); // uuid -> group | null; reset per file (see beginFile)
368
- const unattributed = { entries: 0, turns: 0, ...emptyTokens() };
369
- const totals = { malformed: 0, entries: 0, files: 0 };
516
+ const unattributed = {
517
+ entries: 0,
518
+ turns: 0,
519
+ // Counted for the same reason the rows count them: tool_calls is an
520
+ // ADOPTION metric — how often the model reached for a tool, i.e. the very
521
+ // variable the doctrine sets out to move — so an entry that lands outside
522
+ // every row must still show up in that denominator. `web_search_requests`
523
+ // is a different kind of thing: it is MONEY ($10/1000, billed per request),
524
+ // and money that counting the entry's tokens does not cover, because no
525
+ // token class prices it (TASK-093).
526
+ tool_calls: 0,
527
+ // A strict subset of tool_calls, tracked separately for the same reason the
528
+ // rows track it: agent_spawns is one half of how big the subagent blind spot
529
+ // is (error class 3), and a spawn nobody counted is a whole delegation whose
530
+ // cost we do not merely under-report — we never learn it happened.
531
+ agent_spawns: 0,
532
+ web_search_requests: 0,
533
+ web_fetch_requests: 0,
534
+ ...emptyTokens(),
535
+ };
536
+ // Identity of every API response already billed. NOT reset in beginFile, and
537
+ // deliberately unlike uuidGroup: a uuid chain cannot cross a file, but nothing
538
+ // promises the same about a response id, and a global set costs one string per
539
+ // response. Measured across the 19 guidetrackee transcripts: 0 of 5 954 ids
540
+ // appear in more than one file, so today the scope changes no figure — it only
541
+ // removes a way for a future transcript layout to reopen the bug silently.
542
+ const seenResponses = new Set();
543
+ const totals = { malformed: 0, entries: 0, duplicates: 0, responseRepeats: 0, responsesUnidentified: 0, files: 0 };
370
544
 
371
545
  function groupFor(entry) {
372
546
  // Direct: `user` entries (prompt AND tool results) carry promptId.
@@ -389,6 +563,28 @@ export function createCollector() {
389
563
 
390
564
  function addEntry(entry) {
391
565
  totals.entries++;
566
+
567
+ // A transcript repeats entries VERBATIM, and not rarely: 11 296 of 36 463
568
+ // local entries (31%) carry a uuid already written earlier IN THE SAME FILE,
569
+ // 6 739 of them assistant turns with a usage block. The copies are identical
570
+ // apart from a cosmetic `slug` — same tokens, same model, same timestamp —
571
+ // so summing both charged 45% of all cache-read tokens and 42% of all output
572
+ // tokens twice. That is not a rounding error, it is the epic's whole effect
573
+ // size several times over (error class 1: a figure that prints like a fact).
574
+ //
575
+ // Scoped to the file on purpose: across the 34 local transcripts ZERO uuids
576
+ // are shared between files, so the index that already exists per file
577
+ // (uuidGroup, dropped in beginFile) is exactly the right one to ask. First
578
+ // occurrence wins — file order is event order, and a later copy carrying a
579
+ // different promptId is the copy, not the event.
580
+ //
581
+ // Counted, never silent: a run over a transcript that suddenly stops
582
+ // repeating itself must show that too.
583
+ if (typeof entry.uuid === 'string' && uuidGroup.has(entry.uuid)) {
584
+ totals.duplicates++;
585
+ return;
586
+ }
587
+
392
588
  const group = groupFor(entry);
393
589
  // Memoize BEFORE any type filtering: unknown types are links in the chain
394
590
  // even when they carry no metric of their own.
@@ -405,11 +601,64 @@ export function createCollector() {
405
601
  const model = entry.message?.model ?? null;
406
602
  const synthetic = isAssistant && model === SYNTHETIC_MODEL;
407
603
 
604
+ // Decided here, not inside applyAssistantEntry, because the unattributed
605
+ // branch below bills its own turns and tokens and never reaches that
606
+ // function — the same response split across blocks would inflate the one
607
+ // counter whose job is to report what fell outside the rows.
608
+ //
609
+ // `message.id` is the identity the API gives a response; `requestId` is the
610
+ // fallback for an entry that lacks it. Measured: the two are 1:1 (no id maps
611
+ // to two of the other, in either direction) and no entry has neither, so the
612
+ // fallback is belt-and-braces rather than load-bearing. An entry with NO id
613
+ // at all is counted rather than assumed unique — an unidentifiable response
614
+ // is not silently a new one — and reported via `responsesUnidentified`.
615
+ //
616
+ // Synthetic turns never enter the set: they are excluded from every billing
617
+ // figure anyway, and 8 of them carry a message.id (0 colliding with a real
618
+ // response) that would otherwise poison it.
619
+ let repeatOfResponse = false;
620
+ if (isAssistant && !synthetic) {
621
+ // Non-empty strings only, the same test groupFor() applies to promptId.
622
+ // `??` alone would accept `id: ''`, and an empty id shared by every entry
623
+ // in a file would collapse the whole file into ONE response — the bill
624
+ // would fall to near zero and read as a clean result. Under-counting is
625
+ // the worse direction: an over-count announces itself as implausible.
626
+ const id = entry.message?.id;
627
+ const req = entry.requestId;
628
+ const responseId =
629
+ (typeof id === 'string' && id !== '' && id) || (typeof req === 'string' && req !== '' && req) || null;
630
+ if (responseId === null) totals.responsesUnidentified++;
631
+ else if (seenResponses.has(responseId)) {
632
+ repeatOfResponse = true;
633
+ totals.responseRepeats++;
634
+ } else seenResponses.add(responseId);
635
+ }
636
+
408
637
  if (group === null) {
409
638
  unattributed.entries++;
410
- if (isAssistant && !synthetic) {
639
+ // Per ENTRY, above the repeat guard and outside the synthetic test —
640
+ // exactly where the row path counts them, and for its reasons: tool_use
641
+ // blocks are NOT duplicated across the entries of one response
642
+ // (TASK-092), and a synthetic turn's tool call still happened even though
643
+ // nobody billed for it.
644
+ if (isAssistant) {
645
+ for (const block of toolUseBlocks(entry)) {
646
+ unattributed.tool_calls++;
647
+ if (AGENT_TOOLS.has(block.name)) unattributed.agent_spawns++;
648
+ }
649
+ }
650
+ if (isAssistant && !synthetic && !repeatOfResponse) {
411
651
  unattributed.turns++;
412
- if (usage) addUsage(unattributed, usage);
652
+ if (usage) {
653
+ addUsage(unattributed, usage);
654
+ // Under the guard, the mirror image of the tool calls above:
655
+ // server_tool_use is part of `usage`, so it IS copied onto every entry
656
+ // of a split response, and counting it per entry would re-open
657
+ // TASK-092 in the one place we just closed it.
658
+ const server = serverToolRequests(usage);
659
+ unattributed.web_search_requests += server.search;
660
+ unattributed.web_fetch_requests += server.fetch;
661
+ }
413
662
  }
414
663
  return;
415
664
  }
@@ -424,9 +673,21 @@ export function createCollector() {
424
673
 
425
674
  applyIdentity(p, entry);
426
675
  applyTiming(p, entry, isAssistant);
427
- if (entry.isSidechain === true && isAssistant) p.sidechainTurns++;
428
- if (isAssistant) applyAssistantEntry(p, entry, { usage, model, synthetic });
429
- else applyUserEntry(p, entry);
676
+ // Guarded like api_turns, and for the same reason: this is a TURN counter —
677
+ // it is the figure that says how big the subagent blind spot is (error class
678
+ // 3), paired with agent_spawns. A subagent response splits across content
679
+ // blocks like any other, and being tool-heavy it splits MORE, so counting it
680
+ // per entry would inflate the very number that measures what we cannot see.
681
+ // Zero isSidechain entries in all data measured so far, so this moves no
682
+ // current figure — it stops the bug from arriving with the first one.
683
+ if (entry.isSidechain === true && isAssistant && !repeatOfResponse) p.sidechainTurns++;
684
+ if (isAssistant) applyAssistantEntry(p, entry, { usage, model, synthetic, repeatOfResponse });
685
+ else {
686
+ applyUserEntry(p, entry);
687
+ // Tool results come back as `user` entries; the delegation's bill rides on
688
+ // the one belonging to an `Agent` call, priced at the moment it happened.
689
+ applyDelegation(p, entry, msOf(entry));
690
+ }
430
691
  }
431
692
 
432
693
  function addLine(line) {
@@ -486,6 +747,13 @@ export function createCollector() {
486
747
  tool_calls_total: p.toolCalls,
487
748
  agent_spawns: p.agentSpawns,
488
749
  sidechain_turns: p.sidechainTurns,
750
+ // Deliberately NOT folded into the row's tokens or cost: what the
751
+ // subagents cost is a lower bound (see applyDelegation), and a lower bound
752
+ // added into a total stops being labelled as one. Sorted so two runs over
753
+ // the same transcripts produce byte-identical rows.
754
+ delegations: [...p.delegations.values()].sort((a, b) => a.agent_id.localeCompare(b.agent_id)),
755
+ delegation_duplicates_dropped: p.delegationDuplicates,
756
+ delegations_unidentified: p.delegationsUnidentified,
489
757
  multi_attempt_turns: p.multiAttemptTurns,
490
758
  understated_turns: p.understatedTurns,
491
759
  web_search_requests: p.webSearchRequests,
@@ -524,10 +792,18 @@ export function createCollector() {
524
792
  unattributed,
525
793
  tokens,
526
794
  cost: summarizeCost(rows),
795
+ // Rolled up with the SAME summarizer as the rows — a delegation record
796
+ // carries the cost fields summarizeCost reads, so the lower bound is
797
+ // built by the code that already knows how to refuse an unpriced row.
798
+ // Kept beside `cost`, never inside it (TASK-089 criterion 8b).
799
+ delegationCost: summarizeCost(rows.flatMap((r) => r.delegations)),
527
800
  totals: {
528
801
  ...totals,
529
802
  prompts: rows.length,
530
803
  agent_spawns: sum('agent_spawns'),
804
+ delegations_measured: rows.reduce((n, r) => n + r.delegations.length, 0),
805
+ delegation_duplicates_dropped: sum('delegation_duplicates_dropped'),
806
+ delegations_unidentified: sum('delegations_unidentified'),
531
807
  sidechain_turns: sum('sidechain_turns'),
532
808
  synthetic_turns: sum('synthetic_turns'),
533
809
  multi_attempt_turns: sum('multi_attempt_turns'),
@@ -557,23 +833,89 @@ export function formatRow(row) {
557
833
  * the uncaptured cost of delegation — because a summary that only reports what
558
834
  * it measured is how a blind spot becomes a clean-looking result.
559
835
  */
560
- export function formatSummary({ totals, unattributed, cost, tokens }) {
836
+ export function formatSummary({ totals, unattributed, cost, tokens, delegationCost }) {
561
837
  const L = [];
562
838
  L.push(
563
839
  `prompt-metrics: ${totals.prompts} prompt(s) from ${totals.entries} entries` +
564
840
  ` in ${totals.files} transcript(s)` +
565
841
  (totals.malformed > 0 ? `; ${totals.malformed} malformed line(s) skipped` : ''),
566
842
  );
843
+ if (totals.duplicates > 0) {
844
+ // Loud, because the figure below is ~1.8x smaller than it used to be and a
845
+ // reader comparing two runs of this tool deserves to know why.
846
+ L.push(
847
+ ` repeated entries skipped: ${totals.duplicates} (same uuid, already seen in this file)` +
848
+ ' — counting them charged the same tokens twice',
849
+ );
850
+ }
851
+ if (totals.responseRepeats > 0) {
852
+ // Its own line, never folded into the uuid count above: the two are
853
+ // different defects with different fixes, and this one halves the bill. A
854
+ // reader comparing this run against a pre-TASK-092 run must be able to see
855
+ // which of the two moved the number.
856
+ L.push(
857
+ ` extra entries of the same response: ${totals.responseRepeats}` +
858
+ ' (same message.id — one API response split across content blocks; usage counted once)',
859
+ );
860
+ }
861
+ if (totals.responsesUnidentified > 0) {
862
+ // Counted, not assumed unique: an entry with neither id could be a fresh
863
+ // response or the fourth copy of one already billed. We charge it (an
864
+ // unknown response is never priced at zero) and say so.
865
+ L.push(
866
+ ` assistant entries with no message.id and no requestId: ${totals.responsesUnidentified}` +
867
+ ' — billed as distinct responses; may double-count if they are split blocks',
868
+ );
869
+ }
567
870
  L.push(
568
871
  ` unattributed entries: ${unattributed.entries}` +
569
872
  ` (${unattributed.turns} API turn(s), ${unattributed.output_tokens} output tokens) — not in any row`,
570
873
  );
571
- L.push(
572
- ` subagent spawns: ${totals.agent_spawns}; sidechain turns in transcript: ${totals.sidechain_turns}` +
573
- (totals.agent_spawns > 0 && totals.sidechain_turns === 0
574
- ? ' — subagent cost is UNCAPTURED, not zero (EPIC-009 class 3)'
575
- : ''),
576
- );
874
+ // Its own line, and only the parts that are non-zero — a clean run stays
875
+ // clean, and so does a run whose only gap is one of the three. Split from the
876
+ // line above because the halves are different kinds of gap: the tool calls are
877
+ // missing from an adoption DENOMINATOR, the searches are missing DOLLARS, and
878
+ // dollars that appear in no cost figure this tool prints, here or below.
879
+ const outside = [];
880
+ if (unattributed.tool_calls > 0) {
881
+ outside.push(
882
+ `${unattributed.tool_calls} tool call(s)` +
883
+ (unattributed.agent_spawns > 0
884
+ ? `, ${unattributed.agent_spawns} of them subagent spawn(s) whose cost is UNCAPTURED (class 3)`
885
+ : ''),
886
+ );
887
+ }
888
+ if (unattributed.web_search_requests > 0) {
889
+ outside.push(
890
+ `${unattributed.web_search_requests} web search(es) =` +
891
+ ` ${formatUsd(webSearchCost(unattributed.web_search_requests))} NOT in any cost figure below`,
892
+ );
893
+ }
894
+ if (unattributed.web_fetch_requests > 0) outside.push(`${unattributed.web_fetch_requests} web fetch(es) (free)`);
895
+ if (outside.length > 0) L.push(` also outside every row: ${outside.join(', ')}`);
896
+ L.push(` subagent spawns: ${totals.agent_spawns}; sidechain turns in transcript: ${totals.sidechain_turns}`);
897
+ if (totals.delegations_measured > 0) {
898
+ // Two numbers, never one: what we could price, and how far the spawns
899
+ // outrun it. A spawn with no result entry is cost we have NO figure for.
900
+ const missing = Math.max(0, totals.agent_spawns - totals.delegations_measured);
901
+ L.push(
902
+ ` delegations priced from their own usage: ${totals.delegations_measured}` +
903
+ ` (${totals.delegation_duplicates_dropped} duplicate entr(ies) dropped` +
904
+ `, ${totals.delegations_unidentified} without an agentId, not counted)`,
905
+ );
906
+ if (delegationCost) {
907
+ L.push(
908
+ ` subagent LOWER BOUND: ${formatUsd(delegationCost.list_cost_usd)} over` +
909
+ ` ${delegationCost.priced_rows} of ${delegationCost.total_rows} delegation(s)` +
910
+ ' — separate from the figure above, NEVER added to it (procedure §11a)',
911
+ );
912
+ }
913
+ if (missing > 0) {
914
+ L.push(` ${missing} spawn(s) left no priced result — their cost is UNCAPTURED, not zero (EPIC-009 class 3)`);
915
+ }
916
+ } else if (totals.agent_spawns > 0) {
917
+ L.push(' subagent cost is UNCAPTURED, not zero (EPIC-009 class 3)');
918
+ }
577
919
  if (totals.synthetic_turns > 0) {
578
920
  L.push(` <synthetic> turns excluded from tokens/turns: ${totals.synthetic_turns}`);
579
921
  }
@@ -645,14 +987,21 @@ function defaultDir() {
645
987
  return join(claudeBaseDir(), 'projects');
646
988
  }
647
989
 
648
- async function main() {
990
+ /**
991
+ * The CLI. `args` is everything after the script name, or after
992
+ * `tune-context eval prompt-metrics` — one parser behind both doors (EPIC-010).
993
+ */
994
+ export async function main(args = process.argv.slice(2)) {
995
+ endQuietlyOnClosedPipe();
649
996
  let paths;
997
+ let ledger;
650
998
  try {
651
999
  const { values } = parseArgs({
652
- args: process.argv.slice(2),
1000
+ args,
653
1001
  options: {
654
1002
  transcript: { type: 'string', multiple: true },
655
1003
  dir: { type: 'string' },
1004
+ ledger: { type: 'string' },
656
1005
  help: { type: 'boolean', short: 'h', default: false },
657
1006
  },
658
1007
  strict: true,
@@ -661,6 +1010,9 @@ async function main() {
661
1010
  process.stdout.write(`${USAGE}\n`);
662
1011
  return;
663
1012
  }
1013
+ // Read now, before the transcripts: a --ledger that is not a file stops the
1014
+ // run here, with the same message analyze gives (TASK-098).
1015
+ ledger = readStepLedger(values.ledger, claudeBaseDir());
664
1016
  // Both flags together are a UNION, deduplicated, not a precedence rule:
665
1017
  // silently ignoring one of two flags the user passed is how a run measures
666
1018
  // something other than what was asked for.
@@ -688,11 +1040,6 @@ async function main() {
688
1040
  return;
689
1041
  }
690
1042
 
691
- // Piping into `head` closes stdout early; that is a normal end, not a crash.
692
- process.stdout.on('error', (e) => {
693
- if (e.code !== 'EPIPE') throw e;
694
- });
695
-
696
1043
  const collector = await collectFiles(paths);
697
1044
  const result = collector.result();
698
1045
 
@@ -700,19 +1047,21 @@ async function main() {
700
1047
  // transcript into rows and has no business knowing about the experiment, and
701
1048
  // the ledger is machine state rather than transcript content. The split also
702
1049
  // lets the same rules run over rows collected elsewhere (TASK-088).
703
- const ledgerPath = experimentPaths(claudeBaseDir()).ledgerPath;
704
- const ledgerAvailable = existsSync(ledgerPath);
705
- const rows = annotateRows(result.rows, {
706
- ledger: ledgerAvailable ? readLedger(claudeBaseDir()) : [],
707
- ledgerAvailable,
708
- });
1050
+ const rows = annotateRows(result.rows, ledger);
709
1051
 
710
- for (const row of rows) process.stdout.write(`${formatRow(row)}\n`);
711
- process.stderr.write(`${formatSummary(result)}\n`);
1052
+ // Which code computed the rows rides on every row — added here, with the arm,
1053
+ // because it is a property of the process that ran, not of the transcript
1054
+ // (TASK-097). Last on the row, so nothing that already read a row by position
1055
+ // in a diff moves.
1056
+ const stamp = meterStamp();
1057
+ for (const row of rows) process.stdout.write(`${formatRow({ ...row, meter_version: stamp })}\n`);
1058
+ // First line of the summary, on its own: this is what the operator copies
1059
+ // into the run journal before the first pair.
1060
+ process.stderr.write(`meter ${stamp}\n${formatSummary(result)}\n`);
712
1061
  const arms = formatArmSummary(rows);
713
1062
  if (arms !== null) process.stderr.write(`${arms}\n`);
714
1063
  }
715
1064
 
716
- if (process.argv[1] && process.argv[1].endsWith('prompt-metrics.mjs')) {
1065
+ if (isEntryPoint(import.meta.url)) {
717
1066
  await main();
718
1067
  }