@cat-factory/executor-harness 1.116.0 → 1.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/agent-runner.js +68 -74
- package/dist/claude-call-aggregator.d.ts +9 -0
- package/dist/claude-call-aggregator.js +10 -1
- package/dist/inline.js +20 -6
- package/dist/job.d.ts +8 -1
- package/dist/pi.d.ts +17 -24
- package/dist/pi.js +7 -41
- package/dist/usage-attribution.d.ts +56 -0
- package/dist/usage-attribution.js +96 -0
- package/package.json +4 -4
- package/src/agent-runner.ts +89 -80
- package/src/claude-call-aggregator.ts +15 -2
- package/src/inline.ts +20 -7
- package/src/job.ts +8 -1
- package/src/pi.ts +17 -51
- package/src/usage-attribution.ts +102 -0
package/README.md
CHANGED
|
@@ -335,6 +335,7 @@ Kimi / DeepSeek) and meters spend. The provider key never enters the container.
|
|
|
335
335
|
| `src/package-registries.ts` | Private-registry (npm) auth: renders the job's allowlisted entries into an npmrc; the user `~/.npmrc` in a container, a per-job file pointed at by `npm_config_userconfig` for a native job. |
|
|
336
336
|
| `src/agent-runner.ts` | The subscription-harness runners (`runClaudeCode` / `runCodex`): talk direct to the vendor with a leased OAuth token, lift per-turn usage/telemetry off the CLI event stream. |
|
|
337
337
|
| `src/claude-call-aggregator.ts` | Folds Claude Code's per-CONTENT-BLOCK `stream-json` envelopes back into the model calls they belong to (by `message.id`), reconstructs each call's request transcript, and routes subagent turns off the parent's chain. **Exported as the `./claude-call-aggregator` subpath and driven by the BACKEND too** (`runtimes/local`, for an inline step running on the developer's host `claude`), so it stays the ONE implementation: the per-envelope over-count it fixes inflated a measured 1.47M tokens to 5.53M, and both drivers have to learn that only once. That second driver is why the transcript is retained only to `MAX_TRANSCRIPT_CHARS` (stating what it stopped retaining) and why assembling bodies at all is a `bodies` switch: in a container the reconstruction is one job's memory in a box sized for it, in the backend it is per concurrent inline step in the orchestrator process. Unlike the compile-only `./embed`, this subpath is a `dist` import, which is why the package emits declarations, and why a consumer's typecheck depends on Turbo's `^build` edge having built this package first (see `tsconfig.json`'s `comment:buildOrder`). |
|
|
338
|
+
| `src/usage-attribution.ts` | Reconciles a subscription CLI's TWO token channels: the per-turn usage its stream narrates and the cumulative total its terminal event reports. They disagree routinely and in one direction (Claude Code's per-turn `output_tokens` is the message-START snapshot, single digits), so whatever the turns did not account for becomes ONE extra metric standing for the job (`standsForJob`, filed with a null turn index) rather than tokens grafted onto a real turn, which would make a derived number read as a measured one. Reconciled against the PARENT loop's calls alone, since the terminal cumulative covers only that conversation. |
|
|
338
339
|
| `src/transcript-retention.ts` | Lifts the CLI session transcripts (`projects/` / `sessions/`) out of the isolated, credential-bearing config home before it is deleted, and prunes them on a TTL (debugging artifact retention). |
|
|
339
340
|
| `src/captured-command.ts` | The one way the harness runs a declared shell command on its own behalf: `sh -c` with a per-command watchdog, abort handling, conventional exit codes (124/127/130) and a scrub-then-bound output capture. Shared by both pre-PR verification phases so a fix to one cannot miss the other. |
|
|
340
341
|
| `src/dependency-install.ts` | Dependency prepopulation: `prepopulateDependencies` is the ONE seam every checkout-having mode calls; it runs the service's install command before the agent's first turn, excludes what the install materialised from git so no `git add -A` can sweep a dependency tree into the PR, and builds the prompt note describing the outcome. Best-effort: every failure shape becomes a note, never a failed job. Generic: keyed off the job body, never the agent kind. |
|
package/dist/agent-runner.js
CHANGED
|
@@ -3,11 +3,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js';
|
|
6
|
-
import {
|
|
6
|
+
import { claudeUsage, unaccountedUsageCall } from './usage-attribution.js';
|
|
7
|
+
import { createClaudeRunTelemetry, subagentDispatchId, } from './claude-call-aggregator.js';
|
|
7
8
|
import { ToolCallTracker, recordClaudeToolResults, } from './tool-trajectory.js';
|
|
8
9
|
import { log } from './logger.js';
|
|
9
10
|
import { NO_TOOL_WINDOW } from './tool-silence.js';
|
|
10
|
-
import {
|
|
11
|
+
import { publishCallMetric, } from './pi.js';
|
|
11
12
|
import { claudeAllowedToolPatterns, mcpServerSecretValues, observeClaudeMcpInit, writeClaudeMcpConfig, } from './agent-capabilities.js';
|
|
12
13
|
import { codexImageGapNote, createCodexHome, disposeCodexHome } from './codex-home.js';
|
|
13
14
|
import { ProgressGuard } from './progress-guard.js';
|
|
@@ -19,22 +20,6 @@ import { createSliceTracker, startSubagentWatcher } from './subagents.js';
|
|
|
19
20
|
import { createTaskPlanTracker, mergeProgress, normalizeStatus, pickProgress, toProgress, todosToProgress, } from './progress.js';
|
|
20
21
|
import { assertOnboardingKeysCurrent, writeOnboardingPreseed } from './onboarding-preseed.js';
|
|
21
22
|
import { retainSessionTranscripts } from './transcript-retention.js';
|
|
22
|
-
/**
|
|
23
|
-
* Fallback token attribution: if a CLI reported a cumulative total but no per-turn
|
|
24
|
-
* usage (so every captured call has zero tokens), pin the whole total onto the LAST
|
|
25
|
-
* call rather than dropping it — the run's tokens are still accounted, just not split
|
|
26
|
-
* per turn. A no-op when the calls already carry per-turn tokens.
|
|
27
|
-
*/
|
|
28
|
-
function attributeCumulativeUsage(calls, usage) {
|
|
29
|
-
if (!usage || calls.length === 0)
|
|
30
|
-
return;
|
|
31
|
-
const anyTokens = calls.some((c) => c.inputTokens > 0 || c.outputTokens > 0);
|
|
32
|
-
if (anyTokens)
|
|
33
|
-
return;
|
|
34
|
-
const last = calls[calls.length - 1];
|
|
35
|
-
last.inputTokens = usage.inputTokens;
|
|
36
|
-
last.outputTokens = usage.outputTokens;
|
|
37
|
-
}
|
|
38
23
|
/**
|
|
39
24
|
* Drive one CLI subprocess to completion, streaming LF-framed JSONL from stdout
|
|
40
25
|
* through `onEvent`. Mirrors `runPi`'s lifecycle: prompt over stdin (out-of-band,
|
|
@@ -472,6 +457,59 @@ function openToolWindow(opts) {
|
|
|
472
457
|
function carriesToolResult(content) {
|
|
473
458
|
return content.some((block) => isObject(block) && block.type === 'tool_result');
|
|
474
459
|
}
|
|
460
|
+
/**
|
|
461
|
+
* Open the per-call telemetry capture for one claude-code run.
|
|
462
|
+
*
|
|
463
|
+
* It reconstructs the full per-call request/response bodies from the stream.
|
|
464
|
+
* `--output-format stream-json --verbose` emits a near-verbatim Anthropic Messages envelope per
|
|
465
|
+
* response CONTENT BLOCK (not per call), so the aggregator folds the envelopes sharing a
|
|
466
|
+
* `message.id` back into one call and buffers that call's `user` tool_result turns — together the
|
|
467
|
+
* growing prompt transcript, in the shape the model was actually sent. It is SEEDED with the inputs
|
|
468
|
+
* the harness supplies (they never appear in the stream): the system + first user message when the
|
|
469
|
+
* prompt rides argv, or a single folded user turn when it doesn't, so the reconstruction never shows
|
|
470
|
+
* a system turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
471
|
+
*
|
|
472
|
+
* The parent loop's calls are tracked SEPARATELY, by reference into the same list, because the
|
|
473
|
+
* terminal `result` event's cumulative usage covers only the parent conversation. In `ambientAuth`
|
|
474
|
+
* mode there is no transcript watcher, so the CLI's tagged subagent turns are captured here too and
|
|
475
|
+
* `calls` holds both; reconciling against that mixed list is what once billed a subagent for the
|
|
476
|
+
* parent's whole output shortfall.
|
|
477
|
+
*/
|
|
478
|
+
function openClaudeCallCapture(opts, stream) {
|
|
479
|
+
const calls = [];
|
|
480
|
+
const parentCalls = [];
|
|
481
|
+
const publish = (metric) => publishCallMetric(calls, metric, opts.onCallMetric);
|
|
482
|
+
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring in the caller: it is started
|
|
483
|
+
// only when the CLI has an isolated config home to watch, which an `ambientAuth` run does not
|
|
484
|
+
// have. The telemetry routes the CLI's tagged subagent turns accordingly — see
|
|
485
|
+
// `createClaudeRunTelemetry`.
|
|
486
|
+
const telemetry = createClaudeRunTelemetry({
|
|
487
|
+
seed: stream.folded
|
|
488
|
+
? [{ role: 'user', content: stream.prompt }]
|
|
489
|
+
: [
|
|
490
|
+
{ role: 'system', content: opts.systemPrompt },
|
|
491
|
+
{ role: 'user', content: opts.userPrompt },
|
|
492
|
+
],
|
|
493
|
+
secrets: stream.secrets,
|
|
494
|
+
watcherOwnsSubagents: !opts.ambientAuth,
|
|
495
|
+
publish: (metric) => {
|
|
496
|
+
parentCalls.push(metric);
|
|
497
|
+
publish(metric);
|
|
498
|
+
},
|
|
499
|
+
publishSubagent: publish,
|
|
500
|
+
});
|
|
501
|
+
return {
|
|
502
|
+
calls,
|
|
503
|
+
telemetry,
|
|
504
|
+
settleUsage: (usage) => {
|
|
505
|
+
// Published like any other call so the live drain records it too, which is also what stamps
|
|
506
|
+
// its `seq` and therefore its stable row id.
|
|
507
|
+
const remainder = unaccountedUsageCall(parentCalls, usage);
|
|
508
|
+
if (remainder)
|
|
509
|
+
publish(remainder);
|
|
510
|
+
},
|
|
511
|
+
};
|
|
512
|
+
}
|
|
475
513
|
export async function runClaudeCode(opts) {
|
|
476
514
|
const stats = { toolCalls: 0, assistantChars: 0 };
|
|
477
515
|
let summary = '';
|
|
@@ -488,34 +526,9 @@ export async function runClaudeCode(opts) {
|
|
|
488
526
|
bytes: Buffer.byteLength(opts.systemPrompt, 'utf8'),
|
|
489
527
|
});
|
|
490
528
|
}
|
|
491
|
-
// Reconstruct the full per-call request/response bodies for telemetry from the
|
|
492
|
-
// stream. `--output-format stream-json --verbose` emits a near-verbatim Anthropic
|
|
493
|
-
// Messages envelope per response CONTENT BLOCK (not per call), so the aggregator below
|
|
494
|
-
// folds the envelopes sharing a `message.id` back into one call and buffers that call's
|
|
495
|
-
// `user` tool_result turns — together the growing prompt transcript, in the shape the
|
|
496
|
-
// model was actually sent. We seed it with the inputs the harness supplies (they never
|
|
497
|
-
// appear in the stream): the system + first user message when the prompt rides argv, or
|
|
498
|
-
// a single folded user turn when it doesn't — so the reconstruction never shows a system
|
|
499
|
-
// turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
500
529
|
const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [];
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
// may still rewrite below (a published call must be final — see the publisher).
|
|
504
|
-
const publisher = createCallMetricPublisher(calls, opts.onCallMetric);
|
|
505
|
-
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring below: it is started only when
|
|
506
|
-
// the CLI has an isolated config home to watch, which an `ambientAuth` run does not have. The
|
|
507
|
-
// telemetry routes the CLI's tagged subagent turns accordingly — see `createClaudeRunTelemetry`.
|
|
508
|
-
const telemetry = createClaudeRunTelemetry({
|
|
509
|
-
seed: folded
|
|
510
|
-
? [{ role: 'user', content: prompt }]
|
|
511
|
-
: [
|
|
512
|
-
{ role: 'system', content: opts.systemPrompt },
|
|
513
|
-
{ role: 'user', content: opts.userPrompt },
|
|
514
|
-
],
|
|
515
|
-
secrets,
|
|
516
|
-
watcherOwnsSubagents: !opts.ambientAuth,
|
|
517
|
-
publish: (metric) => publisher.publish(metric),
|
|
518
|
-
});
|
|
530
|
+
const capture = openClaudeCallCapture(opts, { prompt, folded, secrets });
|
|
531
|
+
const telemetry = capture.telemetry;
|
|
519
532
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from the two views the run
|
|
520
533
|
// produces of the SAME slicing. The parent's subagent dispatches + their terminal tool_results
|
|
521
534
|
// appear on this stream (as do the subagents' own intermediate turns, tagged with the dispatch
|
|
@@ -668,8 +681,7 @@ export async function runClaudeCode(opts) {
|
|
|
668
681
|
summary,
|
|
669
682
|
stats,
|
|
670
683
|
stderrTail,
|
|
671
|
-
|
|
672
|
-
publisher,
|
|
684
|
+
capture,
|
|
673
685
|
usage,
|
|
674
686
|
subagents,
|
|
675
687
|
expectSubagentCalls: telemetry.expectsWatcherCalls(),
|
|
@@ -678,11 +690,11 @@ export async function runClaudeCode(opts) {
|
|
|
678
690
|
}
|
|
679
691
|
catch (err) {
|
|
680
692
|
// The stream ended abnormally (guard trip, watchdog kill, CLI crash). Complete the call in
|
|
681
|
-
// flight anyway
|
|
682
|
-
//
|
|
683
|
-
//
|
|
693
|
+
// flight anyway: a killed run never returns an outcome, so the live channel is the ONLY record
|
|
694
|
+
// of what it spent, and dropping its last turn is what the streaming exists to avoid. No
|
|
695
|
+
// terminal `result` event arrived, so there is no cumulative total to reconcile against and no
|
|
696
|
+
// remainder row to file — every captured turn already streamed as it was completed.
|
|
684
697
|
telemetry.flush();
|
|
685
|
-
publisher.flush();
|
|
686
698
|
// A tripped no-progress guard aborted the CLI; streamCli rejects with its generic abort
|
|
687
699
|
// message, so replace it with the guard's actionable diagnostic — carrying the stderr tail it
|
|
688
700
|
// attached, since that is usually the only evidence of what the CLI was doing when it was
|
|
@@ -808,14 +820,11 @@ function buildClaudeEnv(opts, configHome) {
|
|
|
808
820
|
* loop's telemetry must filter them (`subagentDispatchId`) for this concatenation to hold.
|
|
809
821
|
*/
|
|
810
822
|
async function assembleClaudeOutcome(args) {
|
|
811
|
-
const { summary, stats, stderrTail,
|
|
812
|
-
|
|
813
|
-
//
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
// alongside the result, and the backend records the attributed numbers rather than the zeros
|
|
817
|
-
// they carried while the run was in flight.
|
|
818
|
-
publisher.flush();
|
|
823
|
+
const { summary, stats, stderrTail, capture, usage, subagents } = args;
|
|
824
|
+
const calls = capture.calls;
|
|
825
|
+
// What the parent's narrated turns did not account for, as its OWN row (never tokens grafted onto
|
|
826
|
+
// a real turn).
|
|
827
|
+
capture.settleUsage(usage);
|
|
819
828
|
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
820
829
|
// fold the subagents' usage + per-call telemetry into the run's outcome.
|
|
821
830
|
await subagents?.stop();
|
|
@@ -840,21 +849,6 @@ async function assembleClaudeOutcome(args) {
|
|
|
840
849
|
...(mergedCalls.length ? { callMetrics: mergedCalls } : {}),
|
|
841
850
|
};
|
|
842
851
|
}
|
|
843
|
-
function claudeUsage(raw) {
|
|
844
|
-
if (!isObject(raw))
|
|
845
|
-
return undefined;
|
|
846
|
-
// Count every input bucket Anthropic bills: fresh input plus BOTH cache reads and
|
|
847
|
-
// cache writes (cache_creation_input_tokens), which are real consumed tokens — and
|
|
848
|
-
// are the dominant share on a long agent run. Omitting them under-weights a token's
|
|
849
|
-
// true load in the usage-aware rotation window.
|
|
850
|
-
const input = numberOf(raw.input_tokens) +
|
|
851
|
-
numberOf(raw.cache_read_input_tokens) +
|
|
852
|
-
numberOf(raw.cache_creation_input_tokens);
|
|
853
|
-
const output = numberOf(raw.output_tokens);
|
|
854
|
-
if (input === 0 && output === 0)
|
|
855
|
-
return undefined;
|
|
856
|
-
return { inputTokens: input, outputTokens: output };
|
|
857
|
-
}
|
|
858
852
|
// ---------------------------------------------------------------------------
|
|
859
853
|
// Codex
|
|
860
854
|
// ---------------------------------------------------------------------------
|
|
@@ -157,8 +157,17 @@ export interface ClaudeRunTelemetry {
|
|
|
157
157
|
* `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
|
|
158
158
|
* instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
|
|
159
159
|
* billed by neither channel, and an under-count reads as a cheap run rather than as an error.
|
|
160
|
+
*
|
|
161
|
+
* Which is also why the two are published through SEPARATE callbacks. A caller reconciling the
|
|
162
|
+
* parent's terminal cumulative usage needs the parent's calls alone, and with one shared callback
|
|
163
|
+
* the fallback channel silently mixed subagent turns into that list — where they both understated
|
|
164
|
+
* the shortfall and, being last, attracted it (`unaccountedUsageCall`). `publishSubagent` is
|
|
165
|
+
* optional so a caller that draws no distinction (a test, the settled-transcript path where nothing
|
|
166
|
+
* arrives here anyway) keeps one sink.
|
|
160
167
|
*/
|
|
161
168
|
export declare function createClaudeRunTelemetry(opts: ClaudeStreamTelemetryOptions & {
|
|
162
169
|
watcherOwnsSubagents: boolean;
|
|
170
|
+
/** Where a SUBAGENT conversation's call goes. Absent ⇒ `publish`, the parent's sink. */
|
|
171
|
+
publishSubagent?: (metric: HarnessCallMetric) => void;
|
|
163
172
|
}): ClaudeRunTelemetry;
|
|
164
173
|
export {};
|
|
@@ -311,10 +311,19 @@ function createSubagentStreamTelemetry(opts) {
|
|
|
311
311
|
* `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
|
|
312
312
|
* instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
|
|
313
313
|
* billed by neither channel, and an under-count reads as a cheap run rather than as an error.
|
|
314
|
+
*
|
|
315
|
+
* Which is also why the two are published through SEPARATE callbacks. A caller reconciling the
|
|
316
|
+
* parent's terminal cumulative usage needs the parent's calls alone, and with one shared callback
|
|
317
|
+
* the fallback channel silently mixed subagent turns into that list — where they both understated
|
|
318
|
+
* the shortfall and, being last, attracted it (`unaccountedUsageCall`). `publishSubagent` is
|
|
319
|
+
* optional so a caller that draws no distinction (a test, the settled-transcript path where nothing
|
|
320
|
+
* arrives here anyway) keeps one sink.
|
|
314
321
|
*/
|
|
315
322
|
export function createClaudeRunTelemetry(opts) {
|
|
316
323
|
const parent = createClaudeStreamTelemetry(opts);
|
|
317
|
-
const subagents = opts.watcherOwnsSubagents
|
|
324
|
+
const subagents = opts.watcherOwnsSubagents
|
|
325
|
+
? undefined
|
|
326
|
+
: createSubagentStreamTelemetry({ ...opts, publish: opts.publishSubagent ?? opts.publish });
|
|
318
327
|
let sawSubagentTurn = false;
|
|
319
328
|
return {
|
|
320
329
|
onAssistant(dispatchId, message) {
|
package/dist/inline.js
CHANGED
|
@@ -13,14 +13,27 @@ import { runSubscriptionHarness } from './agent-runner.js';
|
|
|
13
13
|
// `auth.json`), so the container and coding paths can never disagree on how a credential
|
|
14
14
|
// is injected.
|
|
15
15
|
/**
|
|
16
|
-
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the inline
|
|
17
|
+
* `finishReason` the reviewer keys off, or `undefined` when the CLI reported NONE.
|
|
18
|
+
*
|
|
19
|
+
* Undefined rather than `stop`, which is what this returned for years on the strength of a
|
|
20
|
+
* comment claiming Claude Code reports the reason. It does not: its `stream-json` `assistant`
|
|
21
|
+
* envelopes carry the message-START snapshot, whose `stop_reason` is null, so every call metric
|
|
22
|
+
* a claude-code or codex run produces has a null reason and this answered `stop` for all of them.
|
|
23
|
+
* `stop` is a positive claim that the model finished of its own accord, and it is the exact claim
|
|
24
|
+
* a truncation check is trying to disprove — so the one caller keyed off it
|
|
25
|
+
* (`finishReason === 'length'`) could never fire, and every store that kept the row recorded a
|
|
26
|
+
* clean stop nobody observed.
|
|
27
|
+
*
|
|
28
|
+
* Kept as a MAPPING rather than deleted because the field remains reachable: a subagent turn is
|
|
29
|
+
* read from a completed JSONL transcript, which does carry `stop_reason`, and a future CLI build
|
|
30
|
+
* (or `--include-partial-messages`) would restore it on the parent stream too.
|
|
20
31
|
*/
|
|
21
32
|
function deriveFinishReason(calls) {
|
|
22
33
|
const last = calls?.[calls.length - 1];
|
|
23
|
-
const reason = last?.finishReason?.toLowerCase()
|
|
34
|
+
const reason = last?.finishReason?.toLowerCase();
|
|
35
|
+
if (!reason)
|
|
36
|
+
return undefined;
|
|
24
37
|
return reason === 'max_tokens' || reason === 'length' ? 'length' : 'stop';
|
|
25
38
|
}
|
|
26
39
|
/**
|
|
@@ -51,9 +64,10 @@ export async function handleInline(job, opts) {
|
|
|
51
64
|
...(opts.signal ? { signal: opts.signal } : {}),
|
|
52
65
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
53
66
|
});
|
|
67
|
+
const finishReason = deriveFinishReason(outcome.callMetrics);
|
|
54
68
|
return {
|
|
55
69
|
text: outcome.summary,
|
|
56
|
-
finishReason:
|
|
70
|
+
...(finishReason ? { finishReason } : {}),
|
|
57
71
|
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
58
72
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
59
73
|
};
|
package/dist/job.d.ts
CHANGED
|
@@ -673,7 +673,14 @@ export interface InlineJob extends HarnessAuthFields {
|
|
|
673
673
|
/** The inline completion result: the reply text plus lifted token usage / per-call telemetry. */
|
|
674
674
|
export interface InlineResult {
|
|
675
675
|
text: string;
|
|
676
|
-
/**
|
|
676
|
+
/**
|
|
677
|
+
* `length` when the model hit its output cap (the reviewer rejects a truncated doc), `stop`
|
|
678
|
+
* when it finished of its own accord, ABSENT when the CLI reported no stop reason at all.
|
|
679
|
+
*
|
|
680
|
+
* Absent is the normal case today: neither subscription CLI exposes a per-call stop reason on
|
|
681
|
+
* its parent stream, and the three states must stay distinct because a reader that takes
|
|
682
|
+
* absent for `stop` is asserting the one thing a truncation check exists to disprove.
|
|
683
|
+
*/
|
|
677
684
|
finishReason?: 'stop' | 'length';
|
|
678
685
|
/**
|
|
679
686
|
* The job's token usage with the input side split into its three ORTHOGONAL classes:
|
package/dist/pi.d.ts
CHANGED
|
@@ -304,6 +304,16 @@ export interface HarnessCallMetric {
|
|
|
304
304
|
* never disagree about which phase billed a call.
|
|
305
305
|
*/
|
|
306
306
|
phase?: string;
|
|
307
|
+
/**
|
|
308
|
+
* This row is not a TURN: it stands for the job as a whole, carrying the spend the CLI reported
|
|
309
|
+
* in its terminal cumulative total and did not attribute to any turn it narrated (see
|
|
310
|
+
* {@link unaccountedUsageCall}). It has no bodies, because there was no request to capture.
|
|
311
|
+
*
|
|
312
|
+
* The backend files it with a NULL turn index for that reason, while still deriving its row id
|
|
313
|
+
* from {@link seq} so a replayed poll re-records instead of duplicating. Absent on every real
|
|
314
|
+
* turn. `CliInlineLanguageModel`'s step-level row is the same idea on the inline path.
|
|
315
|
+
*/
|
|
316
|
+
standsForJob?: boolean;
|
|
307
317
|
}
|
|
308
318
|
/**
|
|
309
319
|
* Publish one captured model call: append it to the run's list (which becomes the terminal
|
|
@@ -318,32 +328,15 @@ export interface HarnessCallMetric {
|
|
|
318
328
|
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
319
329
|
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
320
330
|
* the chain tip it was written against), which means a field mutated after publishing never
|
|
321
|
-
* reaches the store.
|
|
322
|
-
* whose totals arrive with the CLI's terminal `result` event) publishes through
|
|
323
|
-
* {@link createCallMetricPublisher} instead, which withholds exactly those.
|
|
324
|
-
*/
|
|
325
|
-
export declare function publishCallMetric(calls: HarnessCallMetric[], call: HarnessCallMetric, onCallMetric?: (call: HarnessCallMetric) => void): void;
|
|
326
|
-
/** Appends captured calls to a run's list, streaming each one as soon as it is final. */
|
|
327
|
-
export interface CallMetricPublisher {
|
|
328
|
-
/** Append a captured call, streaming it now unless its tokens can still be rewritten. */
|
|
329
|
-
publish(call: HarnessCallMetric): void;
|
|
330
|
-
/** Stream whatever is still withheld. Call once the run's totals are attributed. */
|
|
331
|
-
flush(): void;
|
|
332
|
-
}
|
|
333
|
-
/**
|
|
334
|
-
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
335
|
-
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
336
|
-
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
337
|
-
* arrives.
|
|
331
|
+
* reaches the store.
|
|
338
332
|
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
333
|
+
* That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
|
|
334
|
+
* shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
|
|
335
|
+
* last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
|
|
336
|
+
* could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
|
|
337
|
+
* the wrapper had no reason left to exist.
|
|
345
338
|
*/
|
|
346
|
-
export declare function
|
|
339
|
+
export declare function publishCallMetric(calls: HarnessCallMetric[], call: HarnessCallMetric, onCallMetric?: (call: HarnessCallMetric) => void): void;
|
|
347
340
|
/** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
|
|
348
341
|
export interface PiRunOutcome {
|
|
349
342
|
summary: string;
|
package/dist/pi.js
CHANGED
|
@@ -436,52 +436,18 @@ export async function writeWebToolsConfig(config) {
|
|
|
436
436
|
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
437
437
|
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
438
438
|
* the chain tip it was written against), which means a field mutated after publishing never
|
|
439
|
-
* reaches the store.
|
|
440
|
-
*
|
|
441
|
-
*
|
|
439
|
+
* reaches the store.
|
|
440
|
+
*
|
|
441
|
+
* That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
|
|
442
|
+
* shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
|
|
443
|
+
* last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
|
|
444
|
+
* could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
|
|
445
|
+
* the wrapper had no reason left to exist.
|
|
442
446
|
*/
|
|
443
447
|
export function publishCallMetric(calls, call, onCallMetric) {
|
|
444
448
|
calls.push(call);
|
|
445
449
|
onCallMetric?.(call);
|
|
446
450
|
}
|
|
447
|
-
/**
|
|
448
|
-
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
449
|
-
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
450
|
-
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
451
|
-
* arrives.
|
|
452
|
-
*
|
|
453
|
-
* Since a published call must be final (the backend stores it on the drain and ignores the
|
|
454
|
-
* terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
|
|
455
|
-
* live stream — otherwise it records as a zero-token row and the attributed numbers never land.
|
|
456
|
-
* The withholding window closes the moment any call IS costed: attribution can no longer fire, so
|
|
457
|
-
* everything held is final and released at once, in capture order, and every later call streams
|
|
458
|
-
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
459
|
-
*/
|
|
460
|
-
export function createCallMetricPublisher(calls, onCallMetric) {
|
|
461
|
-
const withheld = [];
|
|
462
|
-
let anyCosted = false;
|
|
463
|
-
const flush = () => {
|
|
464
|
-
for (const call of withheld)
|
|
465
|
-
onCallMetric?.(call);
|
|
466
|
-
withheld.length = 0;
|
|
467
|
-
};
|
|
468
|
-
return {
|
|
469
|
-
publish(call) {
|
|
470
|
-
const costed = call.inputTokens > 0 || call.outputTokens > 0;
|
|
471
|
-
if (!costed && !anyCosted) {
|
|
472
|
-
publishCallMetric(calls, call);
|
|
473
|
-
withheld.push(call);
|
|
474
|
-
return;
|
|
475
|
-
}
|
|
476
|
-
if (costed)
|
|
477
|
-
anyCosted = true;
|
|
478
|
-
// Released BEFORE this call so the live sequence stays in capture order.
|
|
479
|
-
flush();
|
|
480
|
-
publishCallMetric(calls, call, onCallMetric);
|
|
481
|
-
},
|
|
482
|
-
flush,
|
|
483
|
-
};
|
|
484
|
-
}
|
|
485
451
|
/**
|
|
486
452
|
* Pull the `todo` tool's result `details` out of a Pi `--mode json` event, or
|
|
487
453
|
* undefined if the event isn't a successful `todo` tool result.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { HarnessCallMetric } from './pi.js';
|
|
2
|
+
/**
|
|
3
|
+
* Read Claude Code's terminal cumulative usage.
|
|
4
|
+
*
|
|
5
|
+
* Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
|
|
6
|
+
* (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
|
|
7
|
+
* long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
|
|
8
|
+
* window. `undefined` when the event carried no usage at all, so a caller can tell that from a
|
|
9
|
+
* genuine zero.
|
|
10
|
+
*/
|
|
11
|
+
export declare function claudeUsage(raw: unknown): {
|
|
12
|
+
inputTokens: number;
|
|
13
|
+
outputTokens: number;
|
|
14
|
+
} | undefined;
|
|
15
|
+
/**
|
|
16
|
+
* The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
|
|
17
|
+
* usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
|
|
18
|
+
* add up, so nothing is double counted.
|
|
19
|
+
*
|
|
20
|
+
* The per-side part is why this exists at all, and it replaced an all-or-nothing guard
|
|
21
|
+
* (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
|
|
22
|
+
* CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
|
|
23
|
+
* carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
|
|
24
|
+
* `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
|
|
25
|
+
* turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
|
|
26
|
+
* board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
|
|
27
|
+
* `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
|
|
28
|
+
* figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
|
|
29
|
+
* tokens had been reported.
|
|
30
|
+
*
|
|
31
|
+
* **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
|
|
32
|
+
* thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
|
|
33
|
+
* measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
|
|
34
|
+
* step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
|
|
35
|
+
* inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
|
|
36
|
+
* step-level row; this is the same answer for the channel that has a call list. {@link
|
|
37
|
+
* HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
|
|
38
|
+
*
|
|
39
|
+
* **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
|
|
40
|
+
* parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
|
|
41
|
+
* subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
|
|
42
|
+
* would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
|
|
43
|
+
* CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
|
|
44
|
+
* turns are captured through the same publisher as the parent's.
|
|
45
|
+
*
|
|
46
|
+
* {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
|
|
47
|
+
* all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
|
|
48
|
+
* lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
|
|
49
|
+
*
|
|
50
|
+
* Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
|
|
51
|
+
* reported the two inconsistently, and negative spend is not a thing to record.
|
|
52
|
+
*/
|
|
53
|
+
export declare function unaccountedUsageCall(parentCalls: readonly HarnessCallMetric[], usage: {
|
|
54
|
+
inputTokens: number;
|
|
55
|
+
outputTokens: number;
|
|
56
|
+
} | undefined): HarnessCallMetric | undefined;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { isObject, numberOf } from './claude-stream.js';
|
|
2
|
+
// How a subscription CLI's TWO token channels are reconciled into the per-call rows the backend
|
|
3
|
+
// stores: the per-turn usage the stream narrates, and the cumulative total the terminal `result`
|
|
4
|
+
// event reports. They disagree routinely and in a specific direction, so the reconciliation is a
|
|
5
|
+
// concern of its own rather than a helper beside the stream reader that happens to need it.
|
|
6
|
+
//
|
|
7
|
+
// Split out of `agent-runner.ts` when it hit its size budget.
|
|
8
|
+
/**
|
|
9
|
+
* Read Claude Code's terminal cumulative usage.
|
|
10
|
+
*
|
|
11
|
+
* Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
|
|
12
|
+
* (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
|
|
13
|
+
* long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
|
|
14
|
+
* window. `undefined` when the event carried no usage at all, so a caller can tell that from a
|
|
15
|
+
* genuine zero.
|
|
16
|
+
*/
|
|
17
|
+
export function claudeUsage(raw) {
|
|
18
|
+
if (!isObject(raw))
|
|
19
|
+
return undefined;
|
|
20
|
+
const input = numberOf(raw.input_tokens) +
|
|
21
|
+
numberOf(raw.cache_read_input_tokens) +
|
|
22
|
+
numberOf(raw.cache_creation_input_tokens);
|
|
23
|
+
const output = numberOf(raw.output_tokens);
|
|
24
|
+
if (input === 0 && output === 0)
|
|
25
|
+
return undefined;
|
|
26
|
+
return { inputTokens: input, outputTokens: output };
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
|
|
30
|
+
* usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
|
|
31
|
+
* add up, so nothing is double counted.
|
|
32
|
+
*
|
|
33
|
+
* The per-side part is why this exists at all, and it replaced an all-or-nothing guard
|
|
34
|
+
* (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
|
|
35
|
+
* CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
|
|
36
|
+
* carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
|
|
37
|
+
* `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
|
|
38
|
+
* turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
|
|
39
|
+
* board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
|
|
40
|
+
* `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
|
|
41
|
+
* figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
|
|
42
|
+
* tokens had been reported.
|
|
43
|
+
*
|
|
44
|
+
* **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
|
|
45
|
+
* thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
|
|
46
|
+
* measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
|
|
47
|
+
* step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
|
|
48
|
+
* inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
|
|
49
|
+
* step-level row; this is the same answer for the channel that has a call list. {@link
|
|
50
|
+
* HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
|
|
51
|
+
*
|
|
52
|
+
* **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
|
|
53
|
+
* parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
|
|
54
|
+
* subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
|
|
55
|
+
* would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
|
|
56
|
+
* CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
|
|
57
|
+
* turns are captured through the same publisher as the parent's.
|
|
58
|
+
*
|
|
59
|
+
* {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
|
|
60
|
+
* all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
|
|
61
|
+
* lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
|
|
62
|
+
*
|
|
63
|
+
* Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
|
|
64
|
+
* reported the two inconsistently, and negative spend is not a thing to record.
|
|
65
|
+
*/
|
|
66
|
+
export function unaccountedUsageCall(parentCalls, usage) {
|
|
67
|
+
if (!usage)
|
|
68
|
+
return undefined;
|
|
69
|
+
let accountedInput = 0;
|
|
70
|
+
let accountedOutput = 0;
|
|
71
|
+
for (const call of parentCalls) {
|
|
72
|
+
accountedInput += call.inputTokens + call.cacheReadTokens + call.cacheWriteTokens;
|
|
73
|
+
accountedOutput += call.outputTokens;
|
|
74
|
+
}
|
|
75
|
+
const inputTokens = Math.max(0, usage.inputTokens - accountedInput);
|
|
76
|
+
const outputTokens = Math.max(0, usage.outputTokens - accountedOutput);
|
|
77
|
+
if (!inputTokens && !outputTokens)
|
|
78
|
+
return undefined;
|
|
79
|
+
return {
|
|
80
|
+
// No `model`: the terminal event names none, and the recorder then files the row under the
|
|
81
|
+
// model the step DISPATCHED, which is the same answer without this claiming to have observed
|
|
82
|
+
// it. (Claude Code serves some turns with a different model, so a guess here misprices.)
|
|
83
|
+
promptText: '',
|
|
84
|
+
messageCount: 0,
|
|
85
|
+
responseText: '',
|
|
86
|
+
reasoningText: '',
|
|
87
|
+
inputTokens,
|
|
88
|
+
// Both 0 rather than a split of `inputTokens`: the terminal figure is one number and says
|
|
89
|
+
// nothing about which input class the remainder belonged to.
|
|
90
|
+
cacheReadTokens: 0,
|
|
91
|
+
cacheWriteTokens: 0,
|
|
92
|
+
outputTokens,
|
|
93
|
+
finishReason: null,
|
|
94
|
+
standsForJob: true,
|
|
95
|
+
};
|
|
96
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.120.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"hono": "^4.13.1",
|
|
31
31
|
"typescript": "7.0.2",
|
|
32
32
|
"vitest": "^4.1.10",
|
|
33
|
-
"@cat-factory/kernel": "0.
|
|
34
|
-
"@cat-factory/server": "0.
|
|
35
|
-
"@cat-factory/spend": "0.15.
|
|
33
|
+
"@cat-factory/kernel": "0.299.1",
|
|
34
|
+
"@cat-factory/server": "0.286.0",
|
|
35
|
+
"@cat-factory/spend": "0.15.93"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.json",
|
package/src/agent-runner.ts
CHANGED
|
@@ -3,7 +3,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
|
3
3
|
import { tmpdir } from 'node:os'
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js'
|
|
6
|
-
import {
|
|
6
|
+
import { claudeUsage, unaccountedUsageCall } from './usage-attribution.js'
|
|
7
|
+
import {
|
|
8
|
+
createClaudeRunTelemetry,
|
|
9
|
+
subagentDispatchId,
|
|
10
|
+
type ClaudeRunTelemetry,
|
|
11
|
+
} from './claude-call-aggregator.js'
|
|
7
12
|
import {
|
|
8
13
|
ToolCallTracker,
|
|
9
14
|
type TrackedToolCall,
|
|
@@ -12,9 +17,7 @@ import {
|
|
|
12
17
|
import { log, type Logger } from './logger.js'
|
|
13
18
|
import { NO_TOOL_WINDOW, type ToolProgressWindow } from './tool-silence.js'
|
|
14
19
|
import {
|
|
15
|
-
createCallMetricPublisher,
|
|
16
20
|
publishCallMetric,
|
|
17
|
-
type CallMetricPublisher,
|
|
18
21
|
type HarnessCallMetric,
|
|
19
22
|
type PiRunOutcome,
|
|
20
23
|
type TodoProgress,
|
|
@@ -202,24 +205,6 @@ export interface SubscriptionRunOptions {
|
|
|
202
205
|
log?: Logger
|
|
203
206
|
}
|
|
204
207
|
|
|
205
|
-
/**
|
|
206
|
-
* Fallback token attribution: if a CLI reported a cumulative total but no per-turn
|
|
207
|
-
* usage (so every captured call has zero tokens), pin the whole total onto the LAST
|
|
208
|
-
* call rather than dropping it — the run's tokens are still accounted, just not split
|
|
209
|
-
* per turn. A no-op when the calls already carry per-turn tokens.
|
|
210
|
-
*/
|
|
211
|
-
function attributeCumulativeUsage(
|
|
212
|
-
calls: HarnessCallMetric[],
|
|
213
|
-
usage: { inputTokens: number; outputTokens: number } | undefined,
|
|
214
|
-
): void {
|
|
215
|
-
if (!usage || calls.length === 0) return
|
|
216
|
-
const anyTokens = calls.some((c) => c.inputTokens > 0 || c.outputTokens > 0)
|
|
217
|
-
if (anyTokens) return
|
|
218
|
-
const last = calls[calls.length - 1]!
|
|
219
|
-
last.inputTokens = usage.inputTokens
|
|
220
|
-
last.outputTokens = usage.outputTokens
|
|
221
|
-
}
|
|
222
|
-
|
|
223
208
|
/**
|
|
224
209
|
* Drive one CLI subprocess to completion, streaming LF-framed JSONL from stdout
|
|
225
210
|
* through `onEvent`. Mirrors `runPi`'s lifecycle: prompt over stdin (out-of-band,
|
|
@@ -711,6 +696,75 @@ function carriesToolResult(content: unknown[]): boolean {
|
|
|
711
696
|
return content.some((block) => isObject(block) && block.type === 'tool_result')
|
|
712
697
|
}
|
|
713
698
|
|
|
699
|
+
/** One claude-code run's per-call telemetry: what was captured, and how it is settled. */
|
|
700
|
+
interface ClaudeCallCapture {
|
|
701
|
+
/** Every captured call, terminal-result order — the parent's, the subagents', the remainder. */
|
|
702
|
+
calls: HarnessCallMetric[]
|
|
703
|
+
telemetry: ClaudeRunTelemetry
|
|
704
|
+
/**
|
|
705
|
+
* File whatever the parent's narrated turns did not account for, once its terminal cumulative
|
|
706
|
+
* usage is known. A no-op when they add up. See {@link unaccountedUsageCall}.
|
|
707
|
+
*/
|
|
708
|
+
settleUsage: (usage: { inputTokens: number; outputTokens: number } | undefined) => void
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/**
|
|
712
|
+
* Open the per-call telemetry capture for one claude-code run.
|
|
713
|
+
*
|
|
714
|
+
* It reconstructs the full per-call request/response bodies from the stream.
|
|
715
|
+
* `--output-format stream-json --verbose` emits a near-verbatim Anthropic Messages envelope per
|
|
716
|
+
* response CONTENT BLOCK (not per call), so the aggregator folds the envelopes sharing a
|
|
717
|
+
* `message.id` back into one call and buffers that call's `user` tool_result turns — together the
|
|
718
|
+
* growing prompt transcript, in the shape the model was actually sent. It is SEEDED with the inputs
|
|
719
|
+
* the harness supplies (they never appear in the stream): the system + first user message when the
|
|
720
|
+
* prompt rides argv, or a single folded user turn when it doesn't, so the reconstruction never shows
|
|
721
|
+
* a system turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
722
|
+
*
|
|
723
|
+
* The parent loop's calls are tracked SEPARATELY, by reference into the same list, because the
|
|
724
|
+
* terminal `result` event's cumulative usage covers only the parent conversation. In `ambientAuth`
|
|
725
|
+
* mode there is no transcript watcher, so the CLI's tagged subagent turns are captured here too and
|
|
726
|
+
* `calls` holds both; reconciling against that mixed list is what once billed a subagent for the
|
|
727
|
+
* parent's whole output shortfall.
|
|
728
|
+
*/
|
|
729
|
+
function openClaudeCallCapture(
|
|
730
|
+
opts: SubscriptionRunOptions,
|
|
731
|
+
stream: { prompt: string; folded: boolean; secrets: string[] },
|
|
732
|
+
): ClaudeCallCapture {
|
|
733
|
+
const calls: HarnessCallMetric[] = []
|
|
734
|
+
const parentCalls: HarnessCallMetric[] = []
|
|
735
|
+
const publish = (metric: HarnessCallMetric): void =>
|
|
736
|
+
publishCallMetric(calls, metric, opts.onCallMetric)
|
|
737
|
+
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring in the caller: it is started
|
|
738
|
+
// only when the CLI has an isolated config home to watch, which an `ambientAuth` run does not
|
|
739
|
+
// have. The telemetry routes the CLI's tagged subagent turns accordingly — see
|
|
740
|
+
// `createClaudeRunTelemetry`.
|
|
741
|
+
const telemetry = createClaudeRunTelemetry({
|
|
742
|
+
seed: stream.folded
|
|
743
|
+
? [{ role: 'user', content: stream.prompt }]
|
|
744
|
+
: [
|
|
745
|
+
{ role: 'system', content: opts.systemPrompt },
|
|
746
|
+
{ role: 'user', content: opts.userPrompt },
|
|
747
|
+
],
|
|
748
|
+
secrets: stream.secrets,
|
|
749
|
+
watcherOwnsSubagents: !opts.ambientAuth,
|
|
750
|
+
publish: (metric) => {
|
|
751
|
+
parentCalls.push(metric)
|
|
752
|
+
publish(metric)
|
|
753
|
+
},
|
|
754
|
+
publishSubagent: publish,
|
|
755
|
+
})
|
|
756
|
+
return {
|
|
757
|
+
calls,
|
|
758
|
+
telemetry,
|
|
759
|
+
settleUsage: (usage) => {
|
|
760
|
+
// Published like any other call so the live drain records it too, which is also what stamps
|
|
761
|
+
// its `seq` and therefore its stable row id.
|
|
762
|
+
const remainder = unaccountedUsageCall(parentCalls, usage)
|
|
763
|
+
if (remainder) publish(remainder)
|
|
764
|
+
},
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
714
768
|
export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
715
769
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
716
770
|
let summary = ''
|
|
@@ -729,34 +783,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
729
783
|
})
|
|
730
784
|
}
|
|
731
785
|
|
|
732
|
-
// Reconstruct the full per-call request/response bodies for telemetry from the
|
|
733
|
-
// stream. `--output-format stream-json --verbose` emits a near-verbatim Anthropic
|
|
734
|
-
// Messages envelope per response CONTENT BLOCK (not per call), so the aggregator below
|
|
735
|
-
// folds the envelopes sharing a `message.id` back into one call and buffers that call's
|
|
736
|
-
// `user` tool_result turns — together the growing prompt transcript, in the shape the
|
|
737
|
-
// model was actually sent. We seed it with the inputs the harness supplies (they never
|
|
738
|
-
// appear in the stream): the system + first user message when the prompt rides argv, or
|
|
739
|
-
// a single folded user turn when it doesn't — so the reconstruction never shows a system
|
|
740
|
-
// turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
741
786
|
const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : []
|
|
742
|
-
const
|
|
743
|
-
|
|
744
|
-
// may still rewrite below (a published call must be final — see the publisher).
|
|
745
|
-
const publisher = createCallMetricPublisher(calls, opts.onCallMetric)
|
|
746
|
-
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring below: it is started only when
|
|
747
|
-
// the CLI has an isolated config home to watch, which an `ambientAuth` run does not have. The
|
|
748
|
-
// telemetry routes the CLI's tagged subagent turns accordingly — see `createClaudeRunTelemetry`.
|
|
749
|
-
const telemetry = createClaudeRunTelemetry({
|
|
750
|
-
seed: folded
|
|
751
|
-
? [{ role: 'user', content: prompt }]
|
|
752
|
-
: [
|
|
753
|
-
{ role: 'system', content: opts.systemPrompt },
|
|
754
|
-
{ role: 'user', content: opts.userPrompt },
|
|
755
|
-
],
|
|
756
|
-
secrets,
|
|
757
|
-
watcherOwnsSubagents: !opts.ambientAuth,
|
|
758
|
-
publish: (metric) => publisher.publish(metric),
|
|
759
|
-
})
|
|
787
|
+
const capture = openClaudeCallCapture(opts, { prompt, folded, secrets })
|
|
788
|
+
const telemetry = capture.telemetry
|
|
760
789
|
|
|
761
790
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from the two views the run
|
|
762
791
|
// produces of the SAME slicing. The parent's subagent dispatches + their terminal tool_results
|
|
@@ -917,8 +946,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
917
946
|
summary,
|
|
918
947
|
stats,
|
|
919
948
|
stderrTail,
|
|
920
|
-
|
|
921
|
-
publisher,
|
|
949
|
+
capture,
|
|
922
950
|
usage,
|
|
923
951
|
subagents,
|
|
924
952
|
expectSubagentCalls: telemetry.expectsWatcherCalls(),
|
|
@@ -926,11 +954,11 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
926
954
|
})
|
|
927
955
|
} catch (err) {
|
|
928
956
|
// The stream ended abnormally (guard trip, watchdog kill, CLI crash). Complete the call in
|
|
929
|
-
// flight anyway
|
|
930
|
-
//
|
|
931
|
-
//
|
|
957
|
+
// flight anyway: a killed run never returns an outcome, so the live channel is the ONLY record
|
|
958
|
+
// of what it spent, and dropping its last turn is what the streaming exists to avoid. No
|
|
959
|
+
// terminal `result` event arrived, so there is no cumulative total to reconcile against and no
|
|
960
|
+
// remainder row to file — every captured turn already streamed as it was completed.
|
|
932
961
|
telemetry.flush()
|
|
933
|
-
publisher.flush()
|
|
934
962
|
// A tripped no-progress guard aborted the CLI; streamCli rejects with its generic abort
|
|
935
963
|
// message, so replace it with the guard's actionable diagnostic — carrying the stderr tail it
|
|
936
964
|
// attached, since that is usually the only evidence of what the CLI was doing when it was
|
|
@@ -1088,9 +1116,8 @@ async function assembleClaudeOutcome(args: {
|
|
|
1088
1116
|
summary: string
|
|
1089
1117
|
stats: PiRunStats
|
|
1090
1118
|
stderrTail: string
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
publisher: CallMetricPublisher
|
|
1119
|
+
/** This run's per-call telemetry, settled here with the terminal usage. */
|
|
1120
|
+
capture: ClaudeCallCapture
|
|
1094
1121
|
usage: { inputTokens: number; outputTokens: number } | undefined
|
|
1095
1122
|
subagents: ReturnType<typeof startSubagentWatcher> | undefined
|
|
1096
1123
|
/**
|
|
@@ -1102,14 +1129,11 @@ async function assembleClaudeOutcome(args: {
|
|
|
1102
1129
|
expectSubagentCalls: boolean
|
|
1103
1130
|
log?: Logger
|
|
1104
1131
|
}): Promise<PiRunOutcome> {
|
|
1105
|
-
const { summary, stats, stderrTail,
|
|
1106
|
-
|
|
1107
|
-
//
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
// alongside the result, and the backend records the attributed numbers rather than the zeros
|
|
1111
|
-
// they carried while the run was in flight.
|
|
1112
|
-
publisher.flush()
|
|
1132
|
+
const { summary, stats, stderrTail, capture, usage, subagents } = args
|
|
1133
|
+
const calls = capture.calls
|
|
1134
|
+
// What the parent's narrated turns did not account for, as its OWN row (never tokens grafted onto
|
|
1135
|
+
// a real turn).
|
|
1136
|
+
capture.settleUsage(usage)
|
|
1113
1137
|
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
1114
1138
|
// fold the subagents' usage + per-call telemetry into the run's outcome.
|
|
1115
1139
|
await subagents?.stop()
|
|
@@ -1138,21 +1162,6 @@ async function assembleClaudeOutcome(args: {
|
|
|
1138
1162
|
}
|
|
1139
1163
|
}
|
|
1140
1164
|
|
|
1141
|
-
function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number } | undefined {
|
|
1142
|
-
if (!isObject(raw)) return undefined
|
|
1143
|
-
// Count every input bucket Anthropic bills: fresh input plus BOTH cache reads and
|
|
1144
|
-
// cache writes (cache_creation_input_tokens), which are real consumed tokens — and
|
|
1145
|
-
// are the dominant share on a long agent run. Omitting them under-weights a token's
|
|
1146
|
-
// true load in the usage-aware rotation window.
|
|
1147
|
-
const input =
|
|
1148
|
-
numberOf(raw.input_tokens) +
|
|
1149
|
-
numberOf(raw.cache_read_input_tokens) +
|
|
1150
|
-
numberOf(raw.cache_creation_input_tokens)
|
|
1151
|
-
const output = numberOf(raw.output_tokens)
|
|
1152
|
-
if (input === 0 && output === 0) return undefined
|
|
1153
|
-
return { inputTokens: input, outputTokens: output }
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
1165
|
// ---------------------------------------------------------------------------
|
|
1157
1166
|
// Codex
|
|
1158
1167
|
// ---------------------------------------------------------------------------
|
|
@@ -452,12 +452,25 @@ export interface ClaudeRunTelemetry {
|
|
|
452
452
|
* `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
|
|
453
453
|
* instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
|
|
454
454
|
* billed by neither channel, and an under-count reads as a cheap run rather than as an error.
|
|
455
|
+
*
|
|
456
|
+
* Which is also why the two are published through SEPARATE callbacks. A caller reconciling the
|
|
457
|
+
* parent's terminal cumulative usage needs the parent's calls alone, and with one shared callback
|
|
458
|
+
* the fallback channel silently mixed subagent turns into that list — where they both understated
|
|
459
|
+
* the shortfall and, being last, attracted it (`unaccountedUsageCall`). `publishSubagent` is
|
|
460
|
+
* optional so a caller that draws no distinction (a test, the settled-transcript path where nothing
|
|
461
|
+
* arrives here anyway) keeps one sink.
|
|
455
462
|
*/
|
|
456
463
|
export function createClaudeRunTelemetry(
|
|
457
|
-
opts: ClaudeStreamTelemetryOptions & {
|
|
464
|
+
opts: ClaudeStreamTelemetryOptions & {
|
|
465
|
+
watcherOwnsSubagents: boolean
|
|
466
|
+
/** Where a SUBAGENT conversation's call goes. Absent ⇒ `publish`, the parent's sink. */
|
|
467
|
+
publishSubagent?: (metric: HarnessCallMetric) => void
|
|
468
|
+
},
|
|
458
469
|
): ClaudeRunTelemetry {
|
|
459
470
|
const parent = createClaudeStreamTelemetry(opts)
|
|
460
|
-
const subagents = opts.watcherOwnsSubagents
|
|
471
|
+
const subagents = opts.watcherOwnsSubagents
|
|
472
|
+
? undefined
|
|
473
|
+
: createSubagentStreamTelemetry({ ...opts, publish: opts.publishSubagent ?? opts.publish })
|
|
461
474
|
let sawSubagentTurn = false
|
|
462
475
|
|
|
463
476
|
return {
|
package/src/inline.ts
CHANGED
|
@@ -18,14 +18,26 @@ import type { RunOptions } from './runner.js'
|
|
|
18
18
|
// is injected.
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
21
|
+
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the inline
|
|
22
|
+
* `finishReason` the reviewer keys off, or `undefined` when the CLI reported NONE.
|
|
23
|
+
*
|
|
24
|
+
* Undefined rather than `stop`, which is what this returned for years on the strength of a
|
|
25
|
+
* comment claiming Claude Code reports the reason. It does not: its `stream-json` `assistant`
|
|
26
|
+
* envelopes carry the message-START snapshot, whose `stop_reason` is null, so every call metric
|
|
27
|
+
* a claude-code or codex run produces has a null reason and this answered `stop` for all of them.
|
|
28
|
+
* `stop` is a positive claim that the model finished of its own accord, and it is the exact claim
|
|
29
|
+
* a truncation check is trying to disprove — so the one caller keyed off it
|
|
30
|
+
* (`finishReason === 'length'`) could never fire, and every store that kept the row recorded a
|
|
31
|
+
* clean stop nobody observed.
|
|
32
|
+
*
|
|
33
|
+
* Kept as a MAPPING rather than deleted because the field remains reachable: a subagent turn is
|
|
34
|
+
* read from a completed JSONL transcript, which does carry `stop_reason`, and a future CLI build
|
|
35
|
+
* (or `--include-partial-messages`) would restore it on the parent stream too.
|
|
25
36
|
*/
|
|
26
|
-
function deriveFinishReason(calls: HarnessCallMetric[] | undefined): 'stop' | 'length' {
|
|
37
|
+
function deriveFinishReason(calls: HarnessCallMetric[] | undefined): 'stop' | 'length' | undefined {
|
|
27
38
|
const last = calls?.[calls.length - 1]
|
|
28
|
-
const reason = last?.finishReason?.toLowerCase()
|
|
39
|
+
const reason = last?.finishReason?.toLowerCase()
|
|
40
|
+
if (!reason) return undefined
|
|
29
41
|
return reason === 'max_tokens' || reason === 'length' ? 'length' : 'stop'
|
|
30
42
|
}
|
|
31
43
|
|
|
@@ -57,9 +69,10 @@ export async function handleInline(job: InlineJob, opts: RunOptions): Promise<In
|
|
|
57
69
|
...(opts.signal ? { signal: opts.signal } : {}),
|
|
58
70
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
59
71
|
})
|
|
72
|
+
const finishReason = deriveFinishReason(outcome.callMetrics)
|
|
60
73
|
return {
|
|
61
74
|
text: outcome.summary,
|
|
62
|
-
finishReason:
|
|
75
|
+
...(finishReason ? { finishReason } : {}),
|
|
63
76
|
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
64
77
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
65
78
|
}
|
package/src/job.ts
CHANGED
|
@@ -1164,7 +1164,14 @@ export interface InlineJob extends HarnessAuthFields {
|
|
|
1164
1164
|
/** The inline completion result: the reply text plus lifted token usage / per-call telemetry. */
|
|
1165
1165
|
export interface InlineResult {
|
|
1166
1166
|
text: string
|
|
1167
|
-
/**
|
|
1167
|
+
/**
|
|
1168
|
+
* `length` when the model hit its output cap (the reviewer rejects a truncated doc), `stop`
|
|
1169
|
+
* when it finished of its own accord, ABSENT when the CLI reported no stop reason at all.
|
|
1170
|
+
*
|
|
1171
|
+
* Absent is the normal case today: neither subscription CLI exposes a per-call stop reason on
|
|
1172
|
+
* its parent stream, and the three states must stay distinct because a reader that takes
|
|
1173
|
+
* absent for `stop` is asserting the one thing a truncation check exists to disprove.
|
|
1174
|
+
*/
|
|
1168
1175
|
finishReason?: 'stop' | 'length'
|
|
1169
1176
|
/**
|
|
1170
1177
|
* The job's token usage with the input side split into its three ORTHOGONAL classes:
|
package/src/pi.ts
CHANGED
|
@@ -642,6 +642,16 @@ export interface HarnessCallMetric {
|
|
|
642
642
|
* never disagree about which phase billed a call.
|
|
643
643
|
*/
|
|
644
644
|
phase?: string
|
|
645
|
+
/**
|
|
646
|
+
* This row is not a TURN: it stands for the job as a whole, carrying the spend the CLI reported
|
|
647
|
+
* in its terminal cumulative total and did not attribute to any turn it narrated (see
|
|
648
|
+
* {@link unaccountedUsageCall}). It has no bodies, because there was no request to capture.
|
|
649
|
+
*
|
|
650
|
+
* The backend files it with a NULL turn index for that reason, while still deriving its row id
|
|
651
|
+
* from {@link seq} so a replayed poll re-records instead of duplicating. Absent on every real
|
|
652
|
+
* turn. `CliInlineLanguageModel`'s step-level row is the same idea on the inline path.
|
|
653
|
+
*/
|
|
654
|
+
standsForJob?: boolean
|
|
645
655
|
}
|
|
646
656
|
|
|
647
657
|
/**
|
|
@@ -657,9 +667,13 @@ export interface HarnessCallMetric {
|
|
|
657
667
|
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
658
668
|
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
659
669
|
* the chain tip it was written against), which means a field mutated after publishing never
|
|
660
|
-
* reaches the store.
|
|
661
|
-
*
|
|
662
|
-
*
|
|
670
|
+
* reaches the store.
|
|
671
|
+
*
|
|
672
|
+
* That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
|
|
673
|
+
* shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
|
|
674
|
+
* last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
|
|
675
|
+
* could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
|
|
676
|
+
* the wrapper had no reason left to exist.
|
|
663
677
|
*/
|
|
664
678
|
export function publishCallMetric(
|
|
665
679
|
calls: HarnessCallMetric[],
|
|
@@ -670,54 +684,6 @@ export function publishCallMetric(
|
|
|
670
684
|
onCallMetric?.(call)
|
|
671
685
|
}
|
|
672
686
|
|
|
673
|
-
/** Appends captured calls to a run's list, streaming each one as soon as it is final. */
|
|
674
|
-
export interface CallMetricPublisher {
|
|
675
|
-
/** Append a captured call, streaming it now unless its tokens can still be rewritten. */
|
|
676
|
-
publish(call: HarnessCallMetric): void
|
|
677
|
-
/** Stream whatever is still withheld. Call once the run's totals are attributed. */
|
|
678
|
-
flush(): void
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
/**
|
|
682
|
-
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
683
|
-
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
684
|
-
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
685
|
-
* arrives.
|
|
686
|
-
*
|
|
687
|
-
* Since a published call must be final (the backend stores it on the drain and ignores the
|
|
688
|
-
* terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
|
|
689
|
-
* live stream — otherwise it records as a zero-token row and the attributed numbers never land.
|
|
690
|
-
* The withholding window closes the moment any call IS costed: attribution can no longer fire, so
|
|
691
|
-
* everything held is final and released at once, in capture order, and every later call streams
|
|
692
|
-
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
693
|
-
*/
|
|
694
|
-
export function createCallMetricPublisher(
|
|
695
|
-
calls: HarnessCallMetric[],
|
|
696
|
-
onCallMetric?: (call: HarnessCallMetric) => void,
|
|
697
|
-
): CallMetricPublisher {
|
|
698
|
-
const withheld: HarnessCallMetric[] = []
|
|
699
|
-
let anyCosted = false
|
|
700
|
-
const flush = (): void => {
|
|
701
|
-
for (const call of withheld) onCallMetric?.(call)
|
|
702
|
-
withheld.length = 0
|
|
703
|
-
}
|
|
704
|
-
return {
|
|
705
|
-
publish(call) {
|
|
706
|
-
const costed = call.inputTokens > 0 || call.outputTokens > 0
|
|
707
|
-
if (!costed && !anyCosted) {
|
|
708
|
-
publishCallMetric(calls, call)
|
|
709
|
-
withheld.push(call)
|
|
710
|
-
return
|
|
711
|
-
}
|
|
712
|
-
if (costed) anyCosted = true
|
|
713
|
-
// Released BEFORE this call so the live sequence stays in capture order.
|
|
714
|
-
flush()
|
|
715
|
-
publishCallMetric(calls, call, onCallMetric)
|
|
716
|
-
},
|
|
717
|
-
flush,
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
|
|
721
687
|
/** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
|
|
722
688
|
export interface PiRunOutcome {
|
|
723
689
|
summary: string
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { isObject, numberOf } from './claude-stream.js'
|
|
2
|
+
import type { HarnessCallMetric } from './pi.js'
|
|
3
|
+
|
|
4
|
+
// How a subscription CLI's TWO token channels are reconciled into the per-call rows the backend
|
|
5
|
+
// stores: the per-turn usage the stream narrates, and the cumulative total the terminal `result`
|
|
6
|
+
// event reports. They disagree routinely and in a specific direction, so the reconciliation is a
|
|
7
|
+
// concern of its own rather than a helper beside the stream reader that happens to need it.
|
|
8
|
+
//
|
|
9
|
+
// Split out of `agent-runner.ts` when it hit its size budget.
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Read Claude Code's terminal cumulative usage.
|
|
13
|
+
*
|
|
14
|
+
* Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
|
|
15
|
+
* (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
|
|
16
|
+
* long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
|
|
17
|
+
* window. `undefined` when the event carried no usage at all, so a caller can tell that from a
|
|
18
|
+
* genuine zero.
|
|
19
|
+
*/
|
|
20
|
+
export function claudeUsage(
|
|
21
|
+
raw: unknown,
|
|
22
|
+
): { inputTokens: number; outputTokens: number } | undefined {
|
|
23
|
+
if (!isObject(raw)) return undefined
|
|
24
|
+
const input =
|
|
25
|
+
numberOf(raw.input_tokens) +
|
|
26
|
+
numberOf(raw.cache_read_input_tokens) +
|
|
27
|
+
numberOf(raw.cache_creation_input_tokens)
|
|
28
|
+
const output = numberOf(raw.output_tokens)
|
|
29
|
+
if (input === 0 && output === 0) return undefined
|
|
30
|
+
return { inputTokens: input, outputTokens: output }
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
|
|
35
|
+
* usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
|
|
36
|
+
* add up, so nothing is double counted.
|
|
37
|
+
*
|
|
38
|
+
* The per-side part is why this exists at all, and it replaced an all-or-nothing guard
|
|
39
|
+
* (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
|
|
40
|
+
* CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
|
|
41
|
+
* carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
|
|
42
|
+
* `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
|
|
43
|
+
* turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
|
|
44
|
+
* board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
|
|
45
|
+
* `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
|
|
46
|
+
* figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
|
|
47
|
+
* tokens had been reported.
|
|
48
|
+
*
|
|
49
|
+
* **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
|
|
50
|
+
* thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
|
|
51
|
+
* measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
|
|
52
|
+
* step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
|
|
53
|
+
* inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
|
|
54
|
+
* step-level row; this is the same answer for the channel that has a call list. {@link
|
|
55
|
+
* HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
|
|
56
|
+
*
|
|
57
|
+
* **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
|
|
58
|
+
* parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
|
|
59
|
+
* subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
|
|
60
|
+
* would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
|
|
61
|
+
* CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
|
|
62
|
+
* turns are captured through the same publisher as the parent's.
|
|
63
|
+
*
|
|
64
|
+
* {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
|
|
65
|
+
* all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
|
|
66
|
+
* lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
|
|
67
|
+
*
|
|
68
|
+
* Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
|
|
69
|
+
* reported the two inconsistently, and negative spend is not a thing to record.
|
|
70
|
+
*/
|
|
71
|
+
export function unaccountedUsageCall(
|
|
72
|
+
parentCalls: readonly HarnessCallMetric[],
|
|
73
|
+
usage: { inputTokens: number; outputTokens: number } | undefined,
|
|
74
|
+
): HarnessCallMetric | undefined {
|
|
75
|
+
if (!usage) return undefined
|
|
76
|
+
let accountedInput = 0
|
|
77
|
+
let accountedOutput = 0
|
|
78
|
+
for (const call of parentCalls) {
|
|
79
|
+
accountedInput += call.inputTokens + call.cacheReadTokens + call.cacheWriteTokens
|
|
80
|
+
accountedOutput += call.outputTokens
|
|
81
|
+
}
|
|
82
|
+
const inputTokens = Math.max(0, usage.inputTokens - accountedInput)
|
|
83
|
+
const outputTokens = Math.max(0, usage.outputTokens - accountedOutput)
|
|
84
|
+
if (!inputTokens && !outputTokens) return undefined
|
|
85
|
+
return {
|
|
86
|
+
// No `model`: the terminal event names none, and the recorder then files the row under the
|
|
87
|
+
// model the step DISPATCHED, which is the same answer without this claiming to have observed
|
|
88
|
+
// it. (Claude Code serves some turns with a different model, so a guess here misprices.)
|
|
89
|
+
promptText: '',
|
|
90
|
+
messageCount: 0,
|
|
91
|
+
responseText: '',
|
|
92
|
+
reasoningText: '',
|
|
93
|
+
inputTokens,
|
|
94
|
+
// Both 0 rather than a split of `inputTokens`: the terminal figure is one number and says
|
|
95
|
+
// nothing about which input class the remainder belonged to.
|
|
96
|
+
cacheReadTokens: 0,
|
|
97
|
+
cacheWriteTokens: 0,
|
|
98
|
+
outputTokens,
|
|
99
|
+
finishReason: null,
|
|
100
|
+
standsForJob: true,
|
|
101
|
+
}
|
|
102
|
+
}
|