@cat-factory/executor-harness 1.64.2 → 1.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runner.js +94 -50
- package/dist/claude-call-aggregator.js +232 -0
- package/dist/claude-stream.js +12 -6
- package/dist/inline.js +29 -1
- package/dist/subagents.js +14 -3
- package/package.json +4 -4
- package/src/agent-runner.ts +123 -60
- package/src/claude-call-aggregator.ts +331 -0
- package/src/claude-stream.ts +15 -7
- package/src/inline.ts +34 -1
- package/src/job.ts +14 -1
- package/src/pi.ts +14 -1
- package/src/subagents.ts +26 -6
package/dist/agent-runner.js
CHANGED
|
@@ -2,7 +2,8 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
|
-
import { claudeAssistantContent,
|
|
5
|
+
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js';
|
|
6
|
+
import { createClaudeRunTelemetry, subagentDispatchId } from './claude-call-aggregator.js';
|
|
6
7
|
import { createCallMetricPublisher, publishCallMetric, } from './pi.js';
|
|
7
8
|
import { ProgressGuard } from './progress-guard.js';
|
|
8
9
|
import { killChildProcess, spawnDetached } from './process.js';
|
|
@@ -214,30 +215,39 @@ export async function runClaudeCode(opts) {
|
|
|
214
215
|
});
|
|
215
216
|
}
|
|
216
217
|
// Reconstruct the full per-call request/response bodies for telemetry from the
|
|
217
|
-
// stream. `--output-format stream-json --verbose` emits
|
|
218
|
-
//
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
// credential-scrubbed (they can echo the leased token).
|
|
218
|
+
// stream. `--output-format stream-json --verbose` emits a near-verbatim Anthropic
|
|
219
|
+
// Messages envelope per response CONTENT BLOCK (not per call), so the aggregator below
|
|
220
|
+
// folds the envelopes sharing a `message.id` back into one call and buffers that call's
|
|
221
|
+
// `user` tool_result turns — together the growing prompt transcript, in the shape the
|
|
222
|
+
// model was actually sent. We seed it with the inputs the harness supplies (they never
|
|
223
|
+
// appear in the stream): the system + first user message when the prompt rides argv, or
|
|
224
|
+
// a single folded user turn when it doesn't — so the reconstruction never shows a system
|
|
225
|
+
// turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
225
226
|
const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [];
|
|
226
|
-
const messages = folded
|
|
227
|
-
? [{ role: 'user', content: prompt }]
|
|
228
|
-
: [
|
|
229
|
-
{ role: 'system', content: opts.systemPrompt },
|
|
230
|
-
{ role: 'user', content: opts.userPrompt },
|
|
231
|
-
];
|
|
232
227
|
const calls = [];
|
|
233
228
|
// Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
|
|
234
229
|
// may still rewrite below (a published call must be final — see the publisher).
|
|
235
230
|
const publisher = createCallMetricPublisher(calls, opts.onCallMetric);
|
|
231
|
+
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring below: it is started only when
|
|
232
|
+
// the CLI has an isolated config home to watch, which an `ambientAuth` run does not have. The
|
|
233
|
+
// telemetry routes the CLI's tagged subagent turns accordingly — see `createClaudeRunTelemetry`.
|
|
234
|
+
const telemetry = createClaudeRunTelemetry({
|
|
235
|
+
seed: folded
|
|
236
|
+
? [{ role: 'user', content: prompt }]
|
|
237
|
+
: [
|
|
238
|
+
{ role: 'system', content: opts.systemPrompt },
|
|
239
|
+
{ role: 'user', content: opts.userPrompt },
|
|
240
|
+
],
|
|
241
|
+
secrets,
|
|
242
|
+
watcherOwnsSubagents: !opts.ambientAuth,
|
|
243
|
+
publish: (metric) => publisher.publish(metric),
|
|
244
|
+
});
|
|
236
245
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from the two views the run
|
|
237
246
|
// produces of the SAME slicing. The parent's subagent dispatches + their terminal tool_results
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
// `planTracker` + `lastTodo`) is the
|
|
247
|
+
// appear on this stream (as do the subagents' own intermediate turns, tagged with the dispatch
|
|
248
|
+
// that spawned them — see `isSubagentEvent`), so `sliceTracker` knows which slices are in flight
|
|
249
|
+
// and which have returned; the parent's own plan (tracked by `planTracker` + `lastTodo`) is the
|
|
250
|
+
// only place a not-yet-dispatched slice is named at all.
|
|
241
251
|
//
|
|
242
252
|
// The plan arrives in one of two tool vocabularies depending on the bundled CLI build:
|
|
243
253
|
// `TodoWrite` (whole-list snapshots, tracked in `lastTodo`) or the incremental
|
|
@@ -295,12 +305,19 @@ export async function runClaudeCode(opts) {
|
|
|
295
305
|
};
|
|
296
306
|
const onEvent = (event, meta) => {
|
|
297
307
|
const type = event.type;
|
|
308
|
+
// A subagent's turns ride the parent's stdout tagged with the dispatch that spawned them;
|
|
309
|
+
// `telemetry` routes them off the parent's chain (and decides who bills them). Progress, slice
|
|
310
|
+
// tracking, the guard and `stats` below deliberately see EVERY event: a subagent grinding on
|
|
311
|
+
// errors should trip the guard exactly as the parent would, and whether the agent acted at all
|
|
312
|
+
// does not depend on which channel billed it.
|
|
313
|
+
const dispatchId = subagentDispatchId(event);
|
|
298
314
|
if (type === 'assistant' && isObject(event.message)) {
|
|
299
315
|
const message = event.message;
|
|
300
316
|
const content = Array.isArray(message.content) ? message.content : [];
|
|
301
|
-
const { text,
|
|
317
|
+
const { text, toolUses } = claudeAssistantContent(content);
|
|
302
318
|
stats.assistantChars += text.length;
|
|
303
319
|
stats.toolCalls += toolUses;
|
|
320
|
+
telemetry.onAssistant(dispatchId, message);
|
|
304
321
|
for (const block of content) {
|
|
305
322
|
if (!isObject(block) || block.type !== 'tool_use')
|
|
306
323
|
continue;
|
|
@@ -318,22 +335,6 @@ export async function runClaudeCode(opts) {
|
|
|
318
335
|
sliceTracker.onAssistant(content);
|
|
319
336
|
planTracker.onAssistant(content);
|
|
320
337
|
emitProgress();
|
|
321
|
-
// Record this call BEFORE appending its turn: the prompt is the history that
|
|
322
|
-
// produced this response. The append-only array keeps each call's prompt a strict
|
|
323
|
-
// prefix of the next, so the backend's telemetry chain delta-compresses cleanly.
|
|
324
|
-
const u = claudeCallUsage(message.usage);
|
|
325
|
-
publisher.publish({
|
|
326
|
-
...(typeof message.model === 'string' ? { model: message.model } : {}),
|
|
327
|
-
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
328
|
-
messageCount: messages.length,
|
|
329
|
-
responseText: redactBody(text, secrets),
|
|
330
|
-
reasoningText: redactBody(reasoning, secrets),
|
|
331
|
-
inputTokens: u.inputTokens,
|
|
332
|
-
cachedInputTokens: u.cachedInputTokens,
|
|
333
|
-
outputTokens: u.outputTokens,
|
|
334
|
-
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
335
|
-
});
|
|
336
|
-
messages.push({ role: 'assistant', content });
|
|
337
338
|
}
|
|
338
339
|
else if (type === 'user' && isObject(event.message)) {
|
|
339
340
|
// tool_result blocks the harness fed back to the model — part of the next prompt.
|
|
@@ -346,7 +347,7 @@ export async function runClaudeCode(opts) {
|
|
|
346
347
|
// would kill nothing and only convert a clean exit into a spurious failure.
|
|
347
348
|
if (!meta?.final)
|
|
348
349
|
feedGuard(content);
|
|
349
|
-
|
|
350
|
+
telemetry.onToolResult(dispatchId, content);
|
|
350
351
|
}
|
|
351
352
|
}
|
|
352
353
|
else if (type === 'result') {
|
|
@@ -432,6 +433,8 @@ export async function runClaudeCode(opts) {
|
|
|
432
433
|
...appendArgs,
|
|
433
434
|
],
|
|
434
435
|
}, prompt, { ...opts, signal: runSignal }, env, opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [], onEvent);
|
|
436
|
+
// The stream has ended, so the last call has no successor envelope to complete it.
|
|
437
|
+
telemetry.flush();
|
|
435
438
|
return await assembleClaudeOutcome({
|
|
436
439
|
summary,
|
|
437
440
|
stats,
|
|
@@ -440,9 +443,17 @@ export async function runClaudeCode(opts) {
|
|
|
440
443
|
publisher,
|
|
441
444
|
usage,
|
|
442
445
|
subagents,
|
|
446
|
+
expectSubagentCalls: telemetry.expectsWatcherCalls(),
|
|
447
|
+
...(opts.log ? { log: opts.log } : {}),
|
|
443
448
|
});
|
|
444
449
|
}
|
|
445
450
|
catch (err) {
|
|
451
|
+
// The stream ended abnormally (guard trip, watchdog kill, CLI crash). Complete the call in
|
|
452
|
+
// flight anyway, and release whatever the publisher was withholding: a killed run never
|
|
453
|
+
// returns an outcome, so the live channel is the ONLY record of what it spent, and dropping
|
|
454
|
+
// its last turn is what the streaming exists to avoid.
|
|
455
|
+
telemetry.flush();
|
|
456
|
+
publisher.flush();
|
|
446
457
|
// A tripped no-progress guard aborted the CLI; streamCli rejects with its generic abort
|
|
447
458
|
// message, so replace it with the guard's actionable diagnostic — carrying the stderr tail
|
|
448
459
|
// it attached, since that is usually the only evidence of what the CLI was doing when it was
|
|
@@ -498,6 +509,10 @@ function buildClaudeEnv(opts, configHome) {
|
|
|
498
509
|
* terminal `result` event's cumulative) covers ONLY the parent loop, and the subagent tokens live
|
|
499
510
|
* exclusively in the per-session `subagents/*.jsonl` transcripts the watcher reads. Extracted from
|
|
500
511
|
* {@link runClaudeCode} verbatim to keep its cyclomatic complexity down.
|
|
512
|
+
*
|
|
513
|
+
* The invariant is about the aggregate `usage` only. `calls` was NEVER disjoint from the watcher's
|
|
514
|
+
* on its own: the CLI streams a subagent's turns onto the parent's stdout as well, so the parent
|
|
515
|
+
* loop's telemetry must filter them (`subagentDispatchId`) for this concatenation to hold.
|
|
501
516
|
*/
|
|
502
517
|
async function assembleClaudeOutcome(args) {
|
|
503
518
|
const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args;
|
|
@@ -513,6 +528,10 @@ async function assembleClaudeOutcome(args) {
|
|
|
513
528
|
await subagents?.stop();
|
|
514
529
|
const subUsage = subagents?.usage() ?? { inputTokens: 0, outputTokens: 0 };
|
|
515
530
|
const subCalls = subagents?.calls() ?? [];
|
|
531
|
+
if (args.expectSubagentCalls && !subCalls.length) {
|
|
532
|
+
args.log?.warn('subagent turns were streamed but the transcript watcher captured no calls; their token ' +
|
|
533
|
+
'spend is missing from this run’s telemetry (check the CLI’s subagents/*.jsonl layout)');
|
|
534
|
+
}
|
|
516
535
|
const mergedCalls = [...calls, ...subCalls];
|
|
517
536
|
const mergedUsage = usage || subUsage.inputTokens || subUsage.outputTokens
|
|
518
537
|
? {
|
|
@@ -555,7 +574,11 @@ function claudeUsage(raw) {
|
|
|
555
574
|
export async function runCodex(opts) {
|
|
556
575
|
const stats = { toolCalls: 0, assistantChars: 0 };
|
|
557
576
|
let summary = '';
|
|
558
|
-
|
|
577
|
+
// The running CUMULATIVE total, kept in its reported (inclusive) form plus the cached share
|
|
578
|
+
// it contains. `PiRunOutcome.usage` needs the inclusive figure — it is the key-rotation
|
|
579
|
+
// weight — while the fallback call metric below needs the split, so both are derived from
|
|
580
|
+
// this one value rather than one being reconstructed from the other.
|
|
581
|
+
let cumulative;
|
|
559
582
|
// Codex reads its credentials from $CODEX_HOME/auth.json with file-backed
|
|
560
583
|
// storage. CRITICAL: this home must live OUTSIDE the cloned checkout (`opts.cwd`)
|
|
561
584
|
// — the blueprint/requirements/conflict-resolver handlers finish with
|
|
@@ -616,7 +639,7 @@ export async function runCodex(opts) {
|
|
|
616
639
|
opts.onProgress(progress);
|
|
617
640
|
const turnUsage = codexUsage(event);
|
|
618
641
|
if (turnUsage)
|
|
619
|
-
|
|
642
|
+
cumulative = turnUsage;
|
|
620
643
|
// A `token_count` event closes a model turn: pair its per-turn usage with the
|
|
621
644
|
// assistant text seen since the previous turn as one telemetry call.
|
|
622
645
|
const perTurn = codexLastTurnUsage(event);
|
|
@@ -628,7 +651,8 @@ export async function runCodex(opts) {
|
|
|
628
651
|
responseText: redactBody(pendingText, secrets),
|
|
629
652
|
reasoningText: '',
|
|
630
653
|
inputTokens: perTurn.inputTokens,
|
|
631
|
-
|
|
654
|
+
cacheReadTokens: perTurn.cacheReadTokens,
|
|
655
|
+
cacheWriteTokens: perTurn.cacheWriteTokens,
|
|
632
656
|
outputTokens: perTurn.outputTokens,
|
|
633
657
|
finishReason: null,
|
|
634
658
|
}, opts.onCallMetric);
|
|
@@ -654,19 +678,29 @@ export async function runCodex(opts) {
|
|
|
654
678
|
}, prompt, opts, { ...opts.extraEnv, ...(codexHome ? { CODEX_HOME: codexHome } : {}) }, opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [], onEvent);
|
|
655
679
|
// Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
|
|
656
680
|
// single call from the cumulative total + final text so the run is still observable.
|
|
657
|
-
|
|
681
|
+
// The cumulative total is inclusive of its cached share exactly as a per-turn one is, so
|
|
682
|
+
// it is split the same way rather than being filed wholesale as fresh — which would report
|
|
683
|
+
// a cache-heavy run as if nothing had been cached, the one reading this telemetry exists
|
|
684
|
+
// to rule out.
|
|
685
|
+
if (calls.length === 0 && (cumulative || summary)) {
|
|
658
686
|
publishCallMetric(calls, {
|
|
659
687
|
model: opts.model,
|
|
660
688
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
661
689
|
messageCount: messages.length,
|
|
662
690
|
responseText: redactBody(summary, secrets),
|
|
663
691
|
reasoningText: '',
|
|
664
|
-
inputTokens:
|
|
665
|
-
|
|
666
|
-
|
|
692
|
+
inputTokens: Math.max(0, (cumulative?.inputTokens ?? 0) - (cumulative?.cachedInputTokens ?? 0)),
|
|
693
|
+
cacheReadTokens: cumulative?.cachedInputTokens ?? 0,
|
|
694
|
+
// Codex reports no separate cache-WRITE class; 0 rather than guessed.
|
|
695
|
+
cacheWriteTokens: 0,
|
|
696
|
+
outputTokens: cumulative?.outputTokens ?? 0,
|
|
667
697
|
finishReason: null,
|
|
668
698
|
}, opts.onCallMetric);
|
|
669
699
|
}
|
|
700
|
+
// The outcome's usage is the key-rotation WEIGHT, so it keeps the inclusive input count.
|
|
701
|
+
const usage = cumulative
|
|
702
|
+
? { inputTokens: cumulative.inputTokens, outputTokens: cumulative.outputTokens }
|
|
703
|
+
: undefined;
|
|
670
704
|
return {
|
|
671
705
|
summary,
|
|
672
706
|
stats,
|
|
@@ -746,8 +780,6 @@ function codexPlanProgress(event) {
|
|
|
746
780
|
* other shapes put it on `usage` / `info.usage` directly. We read the cumulative
|
|
747
781
|
* total when present so the caller can simply overwrite (not sum) — summing
|
|
748
782
|
* cumulative totals across events would multiply-count. Checked most-likely first.
|
|
749
|
-
* `input_tokens` is the TOTAL prompt count (OpenAI semantics: `cached_input_tokens`
|
|
750
|
-
* is a subset already inside it), so it is NOT summed with the cached share.
|
|
751
783
|
*/
|
|
752
784
|
function codexUsage(event) {
|
|
753
785
|
const info = isObject(event.info) ? event.info : undefined;
|
|
@@ -761,14 +793,21 @@ function codexUsage(event) {
|
|
|
761
793
|
const output = numberOf(raw.output_tokens);
|
|
762
794
|
if (input === 0 && output === 0)
|
|
763
795
|
return undefined;
|
|
764
|
-
return {
|
|
796
|
+
return {
|
|
797
|
+
inputTokens: input,
|
|
798
|
+
cachedInputTokens: numberOf(raw.cached_input_tokens),
|
|
799
|
+
outputTokens: output,
|
|
800
|
+
};
|
|
765
801
|
}
|
|
766
802
|
/**
|
|
767
803
|
* Per-TURN Codex token usage off a `token_count` event's `info.last_token_usage` (the
|
|
768
804
|
* delta for the turn just completed, as opposed to `codexUsage`'s cumulative total).
|
|
769
|
-
*
|
|
770
|
-
*
|
|
771
|
-
*
|
|
805
|
+
*
|
|
806
|
+
* OpenAI semantics: `input_tokens` is the turn's WHOLE prompt count and already INCLUDES
|
|
807
|
+
* the cached share, so the fresh figure is the difference. Clamped at 0 because the two
|
|
808
|
+
* counts come off the same event and a vendor inconsistency must not mint a negative token
|
|
809
|
+
* count. Codex reports no separate cache-WRITE class, so that class is 0 here rather than
|
|
810
|
+
* guessed.
|
|
772
811
|
*/
|
|
773
812
|
function codexLastTurnUsage(event) {
|
|
774
813
|
const info = isObject(event.info) ? event.info : undefined;
|
|
@@ -780,7 +819,12 @@ function codexLastTurnUsage(event) {
|
|
|
780
819
|
const output = numberOf(raw.output_tokens);
|
|
781
820
|
if (input === 0 && output === 0)
|
|
782
821
|
return undefined;
|
|
783
|
-
return {
|
|
822
|
+
return {
|
|
823
|
+
inputTokens: Math.max(0, input - cached),
|
|
824
|
+
cacheReadTokens: cached,
|
|
825
|
+
cacheWriteTokens: 0,
|
|
826
|
+
outputTokens: output,
|
|
827
|
+
};
|
|
784
828
|
}
|
|
785
829
|
/** Dispatch to the configured subscription harness runner. */
|
|
786
830
|
export function runSubscriptionHarness(harness, opts) {
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { claudeAssistantContent, claudeCallUsage, isObject, redactBody } from './claude-stream.js';
|
|
2
|
+
/**
|
|
3
|
+
* Assemble per-call telemetry out of Claude Code's per-block stream envelopes.
|
|
4
|
+
*
|
|
5
|
+
* `onCallStart` fires when a call's FIRST envelope arrives, which is the moment the caller must
|
|
6
|
+
* snapshot the prompt: the history at that point is what produced the response. `onCall` fires
|
|
7
|
+
* once the call is complete (a different `message.id` began, or the stream ended).
|
|
8
|
+
*
|
|
9
|
+
* Usage is merged as the MAXIMUM of each bucket across the call's envelopes rather than the last
|
|
10
|
+
* one seen. The envelopes carry a snapshot of the same call's usage, and which of them holds the
|
|
11
|
+
* final output count is a CLI detail we should not depend on; a max is right whether the value is
|
|
12
|
+
* repeated verbatim or grows.
|
|
13
|
+
*
|
|
14
|
+
* An envelope with no `message.id` cannot be attributed, so it is treated as a call of its own —
|
|
15
|
+
* the pre-aggregation behaviour, kept so a CLI build (or a transcript) that omits the id degrades
|
|
16
|
+
* to over-counting rather than to silently merging unrelated calls.
|
|
17
|
+
*/
|
|
18
|
+
export function createClaudeCallAggregator(handlers) {
|
|
19
|
+
let pending;
|
|
20
|
+
let anonymous = 0;
|
|
21
|
+
const complete = () => {
|
|
22
|
+
if (!pending)
|
|
23
|
+
return;
|
|
24
|
+
const { id: _id, ...call } = pending;
|
|
25
|
+
pending = undefined;
|
|
26
|
+
handlers.onCall(call);
|
|
27
|
+
};
|
|
28
|
+
return {
|
|
29
|
+
onAssistant(message) {
|
|
30
|
+
// `#anon-<n>` cannot collide with a real id (the API mints `msg_…`), so an envelope
|
|
31
|
+
// with no id keeps its own call rather than merging into whatever came before it.
|
|
32
|
+
const id = typeof message.id === 'string' && message.id ? message.id : `#anon-${anonymous++}`;
|
|
33
|
+
if (pending && pending.id !== id)
|
|
34
|
+
complete();
|
|
35
|
+
const content = Array.isArray(message.content) ? message.content : [];
|
|
36
|
+
const { text, reasoning, toolUses } = claudeAssistantContent(content);
|
|
37
|
+
const usage = claudeCallUsage(message.usage);
|
|
38
|
+
const stopReason = typeof message.stop_reason === 'string' ? message.stop_reason : null;
|
|
39
|
+
const model = typeof message.model === 'string' ? message.model : undefined;
|
|
40
|
+
if (!pending) {
|
|
41
|
+
pending = {
|
|
42
|
+
id,
|
|
43
|
+
content: [],
|
|
44
|
+
text: '',
|
|
45
|
+
reasoning: '',
|
|
46
|
+
stopReason: null,
|
|
47
|
+
inputTokens: 0,
|
|
48
|
+
cacheReadTokens: 0,
|
|
49
|
+
cacheWriteTokens: 0,
|
|
50
|
+
outputTokens: 0,
|
|
51
|
+
toolResults: [],
|
|
52
|
+
toolUses: 0,
|
|
53
|
+
};
|
|
54
|
+
handlers.onCallStart?.();
|
|
55
|
+
}
|
|
56
|
+
pending.content.push(...content);
|
|
57
|
+
pending.text += text;
|
|
58
|
+
pending.reasoning += reasoning;
|
|
59
|
+
pending.toolUses += toolUses;
|
|
60
|
+
pending.inputTokens = Math.max(pending.inputTokens, usage.inputTokens);
|
|
61
|
+
pending.cacheReadTokens = Math.max(pending.cacheReadTokens, usage.cacheReadTokens);
|
|
62
|
+
pending.cacheWriteTokens = Math.max(pending.cacheWriteTokens, usage.cacheWriteTokens);
|
|
63
|
+
pending.outputTokens = Math.max(pending.outputTokens, usage.outputTokens);
|
|
64
|
+
// A block-split response reports its stop reason on the envelope that carries the end of the
|
|
65
|
+
// message; earlier ones report none. Keep the first non-null rather than the last seen.
|
|
66
|
+
if (stopReason && !pending.stopReason)
|
|
67
|
+
pending.stopReason = stopReason;
|
|
68
|
+
if (model && !pending.model)
|
|
69
|
+
pending.model = model;
|
|
70
|
+
},
|
|
71
|
+
onToolResult(content) {
|
|
72
|
+
// Results can only belong to the tool_use blocks of the call in flight. Before the first
|
|
73
|
+
// assistant envelope there is nothing they could attach to.
|
|
74
|
+
if (pending)
|
|
75
|
+
pending.toolResults.push(content);
|
|
76
|
+
},
|
|
77
|
+
flush: complete,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Assemble ONE conversation's per-call telemetry from the CLI stream: the growing request
|
|
82
|
+
* transcript and the per-call token/body metrics.
|
|
83
|
+
*
|
|
84
|
+
* Owns the transcript because the two are one concern — a call's `promptText` is the transcript as
|
|
85
|
+
* of that call, and its turns may only be appended once the call that produced them is complete.
|
|
86
|
+
* `seed` is what the harness supplied and the stream therefore never shows (the system + first user
|
|
87
|
+
* message, or the single folded user turn), so the reconstruction never claims a system turn that
|
|
88
|
+
* was not sent. A subagent's conversation seeds EMPTY — its prompt was minted by the CLI and never
|
|
89
|
+
* crosses this stream — which is also why its first call carries `messageCount: 0` (the backend's
|
|
90
|
+
* `latestChainTip` skips those on purpose: there is no re-sendable chain to delta against).
|
|
91
|
+
* Bodies are credential-scrubbed; they can echo the leased token.
|
|
92
|
+
*
|
|
93
|
+
* Deliberately does NOT touch {@link PiRunStats}: the run's tool/output counters describe whether
|
|
94
|
+
* the agent ACTED at all (`agentNeverActed`), which is true of a subagent's turns whichever channel
|
|
95
|
+
* ends up owning their telemetry rows. The caller accumulates them off the raw stream instead.
|
|
96
|
+
*/
|
|
97
|
+
export function createClaudeStreamTelemetry(opts) {
|
|
98
|
+
const messages = [...opts.seed];
|
|
99
|
+
let callPrompt = '';
|
|
100
|
+
let callMessageCount = 0;
|
|
101
|
+
// The aggregator IS the surface: the transcript and metric work happens in its callbacks, so
|
|
102
|
+
// there is nothing to wrap it in.
|
|
103
|
+
return createClaudeCallAggregator({
|
|
104
|
+
// Snapshotted when a call's FIRST envelope arrives: the history at that moment is what
|
|
105
|
+
// produced the response, and later envelopes of the same call must not see the turns it
|
|
106
|
+
// went on to add.
|
|
107
|
+
onCallStart: () => {
|
|
108
|
+
callPrompt = redactBody(JSON.stringify(messages), opts.secrets);
|
|
109
|
+
callMessageCount = messages.length;
|
|
110
|
+
},
|
|
111
|
+
onCall: (call) => {
|
|
112
|
+
opts.publish({
|
|
113
|
+
...(call.model ? { model: call.model } : {}),
|
|
114
|
+
promptText: callPrompt,
|
|
115
|
+
messageCount: callMessageCount,
|
|
116
|
+
responseText: redactBody(call.text, opts.secrets),
|
|
117
|
+
reasoningText: redactBody(call.reasoning, opts.secrets),
|
|
118
|
+
inputTokens: call.inputTokens,
|
|
119
|
+
cacheReadTokens: call.cacheReadTokens,
|
|
120
|
+
cacheWriteTokens: call.cacheWriteTokens,
|
|
121
|
+
outputTokens: call.outputTokens,
|
|
122
|
+
finishReason: call.stopReason,
|
|
123
|
+
});
|
|
124
|
+
// Appended only now, so each call's prompt stays a strict prefix of the next and the
|
|
125
|
+
// backend's telemetry chain delta-compresses cleanly.
|
|
126
|
+
messages.push({ role: 'assistant', content: call.content });
|
|
127
|
+
for (const result of call.toolResults)
|
|
128
|
+
messages.push({ role: 'tool', content: result });
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* The dispatch (`Agent`/`Task` tool_use) id a stream envelope is tagged with, or `undefined` for a
|
|
134
|
+
* parent-loop turn.
|
|
135
|
+
*
|
|
136
|
+
* Claude Code streams the turns of the subagents it dispatches onto the parent's stdout, tagged
|
|
137
|
+
* with the tool_use id that spawned them. Those same turns are also written to the per-session
|
|
138
|
+
* `subagents/*.jsonl` transcripts the watcher reads, so recording both channels counted every
|
|
139
|
+
* subagent call twice — and splicing them into the parent's message reconstruction produced a
|
|
140
|
+
* `promptText` chain that interleaves several conversations and therefore matches no real request.
|
|
141
|
+
*
|
|
142
|
+
* The id is what makes the fallback below possible: concurrent subagents interleave on one stdout,
|
|
143
|
+
* so it is the ONLY thing separating their conversations.
|
|
144
|
+
*/
|
|
145
|
+
export function subagentDispatchId(event) {
|
|
146
|
+
if (!isObject(event))
|
|
147
|
+
return undefined;
|
|
148
|
+
const id = event.parent_tool_use_id;
|
|
149
|
+
return typeof id === 'string' && id ? id : undefined;
|
|
150
|
+
}
|
|
151
|
+
/** Whether a stream envelope describes a SUBAGENT's turn rather than the parent loop's. */
|
|
152
|
+
export function isSubagentEvent(event) {
|
|
153
|
+
return subagentDispatchId(event) !== undefined;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Per-call telemetry for the subagents whose turns ride the parent's stdout — the FALLBACK channel,
|
|
157
|
+
* used only when no `subagents/*.jsonl` watcher will run (see `startSubagentWatcher`, which is
|
|
158
|
+
* wired only when the CLI has an isolated config home; an `ambientAuth` run has none).
|
|
159
|
+
*
|
|
160
|
+
* Without this, filtering tagged events out of the parent's telemetry leaves a subagent-heavy run
|
|
161
|
+
* with its spend recorded by NEITHER channel — an under-count, which reads as a cheap run and is
|
|
162
|
+
* the worse failure direction than the double-count the filter exists to fix.
|
|
163
|
+
*
|
|
164
|
+
* Each dispatch id gets its OWN transcript, because concurrent subagents interleave arbitrarily on
|
|
165
|
+
* one stream: folding them into a single chain is exactly the defect this whole module removes,
|
|
166
|
+
* one level down.
|
|
167
|
+
*/
|
|
168
|
+
function createSubagentStreamTelemetry(opts) {
|
|
169
|
+
const perDispatch = new Map();
|
|
170
|
+
const forDispatch = (dispatchId) => {
|
|
171
|
+
let telemetry = perDispatch.get(dispatchId);
|
|
172
|
+
if (!telemetry) {
|
|
173
|
+
// Seeded EMPTY: the CLI minted this subagent's prompt and it never crossed the stream.
|
|
174
|
+
telemetry = createClaudeStreamTelemetry({
|
|
175
|
+
seed: [],
|
|
176
|
+
secrets: opts.secrets,
|
|
177
|
+
publish: opts.publish,
|
|
178
|
+
});
|
|
179
|
+
perDispatch.set(dispatchId, telemetry);
|
|
180
|
+
}
|
|
181
|
+
return telemetry;
|
|
182
|
+
};
|
|
183
|
+
return {
|
|
184
|
+
onAssistant: (dispatchId, message) => forDispatch(dispatchId).onAssistant(message),
|
|
185
|
+
// Only against a dispatch already seen: a result for a subagent whose assistant turns never
|
|
186
|
+
// reached us has no conversation to attach to, and minting one would publish a call that is
|
|
187
|
+
// all tool output and no request.
|
|
188
|
+
onToolResult: (dispatchId, content) => perDispatch.get(dispatchId)?.onToolResult(content),
|
|
189
|
+
flush: () => {
|
|
190
|
+
for (const telemetry of perDispatch.values())
|
|
191
|
+
telemetry.flush();
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Assemble a run's per-call telemetry, routing each envelope to the conversation it belongs to.
|
|
197
|
+
*
|
|
198
|
+
* The routing is the whole point. A subagent's turns ride the parent's stdout tagged with the
|
|
199
|
+
* dispatch that spawned them, and they must never join the PARENT's chain — that splice produced a
|
|
200
|
+
* `promptText` interleaving several conversations, matching no request that was ever sent.
|
|
201
|
+
*
|
|
202
|
+
* Who RECORDS them is a separate question, decided once per run rather than per event:
|
|
203
|
+
* `watcherOwnsSubagents` says a `subagents/*.jsonl` watcher will run, and it is the better source
|
|
204
|
+
* (it reads the settled transcript, so its usage and stop reason are final). With no watcher — an
|
|
205
|
+
* `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
|
|
206
|
+
* instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
|
|
207
|
+
* billed by neither channel, and an under-count reads as a cheap run rather than as an error.
|
|
208
|
+
*/
|
|
209
|
+
export function createClaudeRunTelemetry(opts) {
|
|
210
|
+
const parent = createClaudeStreamTelemetry(opts);
|
|
211
|
+
const subagents = opts.watcherOwnsSubagents ? undefined : createSubagentStreamTelemetry(opts);
|
|
212
|
+
let sawSubagentTurn = false;
|
|
213
|
+
return {
|
|
214
|
+
onAssistant(dispatchId, message) {
|
|
215
|
+
if (!dispatchId)
|
|
216
|
+
return parent.onAssistant(message);
|
|
217
|
+
sawSubagentTurn = true;
|
|
218
|
+
subagents?.onAssistant(dispatchId, message);
|
|
219
|
+
},
|
|
220
|
+
onToolResult(dispatchId, content) {
|
|
221
|
+
if (!dispatchId)
|
|
222
|
+
return parent.onToolResult(content);
|
|
223
|
+
sawSubagentTurn = true;
|
|
224
|
+
subagents?.onToolResult(dispatchId, content);
|
|
225
|
+
},
|
|
226
|
+
flush() {
|
|
227
|
+
parent.flush();
|
|
228
|
+
subagents?.flush();
|
|
229
|
+
},
|
|
230
|
+
expectsWatcherCalls: () => opts.watcherOwnsSubagents && sawSubagentTurn,
|
|
231
|
+
};
|
|
232
|
+
}
|
package/dist/claude-stream.js
CHANGED
|
@@ -51,16 +51,22 @@ export function claudeAssistantContent(content) {
|
|
|
51
51
|
}
|
|
52
52
|
/**
|
|
53
53
|
* Per-CALL token usage off a Claude `assistant` message's `usage` (this turn only, not
|
|
54
|
-
* the cumulative `result` total).
|
|
55
|
-
*
|
|
54
|
+
* the cumulative `result` total).
|
|
55
|
+
*
|
|
56
|
+
* Anthropic reports all three input classes SEPARATELY and `input_tokens` is already
|
|
57
|
+
* exclusive of both caches, so the three fields here are orthogonal and additive:
|
|
58
|
+
* total input = `inputTokens + cacheReadTokens + cacheWriteTokens`. Do NOT re-lump the
|
|
59
|
+
* reads and the writes — a cache write costs 1.25–2× base input while a read costs ~0.1×,
|
|
60
|
+
* so a turn that keeps invalidating the prefix and one that rides a warm cache are
|
|
61
|
+
* indistinguishable once they are summed.
|
|
56
62
|
*/
|
|
57
63
|
export function claudeCallUsage(raw) {
|
|
58
64
|
if (!isObject(raw))
|
|
59
|
-
return { inputTokens: 0,
|
|
60
|
-
const cached = numberOf(raw.cache_read_input_tokens) + numberOf(raw.cache_creation_input_tokens);
|
|
65
|
+
return { inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, outputTokens: 0 };
|
|
61
66
|
return {
|
|
62
|
-
inputTokens: numberOf(raw.input_tokens)
|
|
63
|
-
|
|
67
|
+
inputTokens: numberOf(raw.input_tokens),
|
|
68
|
+
cacheReadTokens: numberOf(raw.cache_read_input_tokens),
|
|
69
|
+
cacheWriteTokens: numberOf(raw.cache_creation_input_tokens),
|
|
64
70
|
outputTokens: numberOf(raw.output_tokens),
|
|
65
71
|
};
|
|
66
72
|
}
|
package/dist/inline.js
CHANGED
|
@@ -48,7 +48,7 @@ export async function handleInline(job, opts) {
|
|
|
48
48
|
return {
|
|
49
49
|
text: outcome.summary,
|
|
50
50
|
finishReason: deriveFinishReason(outcome.callMetrics),
|
|
51
|
-
...(outcome.usage ? { usage: outcome.usage } : {}),
|
|
51
|
+
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
52
52
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
53
53
|
};
|
|
54
54
|
}
|
|
@@ -56,3 +56,31 @@ export async function handleInline(job, opts) {
|
|
|
56
56
|
await rm(cwd, { recursive: true, force: true }).catch(() => { });
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Split the run's coarse usage into the three orthogonal input classes an {@link InlineResult}
|
|
61
|
+
* carries. `outcome.usage` is the ROTATION-window weight — every billed input bucket summed —
|
|
62
|
+
* so the split has to come from the per-call metrics, the only channel that kept the classes
|
|
63
|
+
* apart. Fresh input is likewise taken from the calls rather than derived by subtraction, so a
|
|
64
|
+
* CLI whose per-call and cumulative counts disagree can never produce a negative class.
|
|
65
|
+
*
|
|
66
|
+
* With no per-call telemetry (an older CLI build that streams nothing) the coarse total is
|
|
67
|
+
* reported as fresh with both cache classes 0. That is the honest reading: nothing is KNOWN to
|
|
68
|
+
* have been cached, and inventing a split would be worse than admitting the channel is silent.
|
|
69
|
+
*/
|
|
70
|
+
function inlineUsage(usage, calls) {
|
|
71
|
+
if (!calls?.length) {
|
|
72
|
+
return {
|
|
73
|
+
inputTokens: usage.inputTokens,
|
|
74
|
+
cacheReadTokens: 0,
|
|
75
|
+
cacheWriteTokens: 0,
|
|
76
|
+
outputTokens: usage.outputTokens,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
const sum = (pick) => calls.reduce((total, call) => total + pick(call), 0);
|
|
80
|
+
return {
|
|
81
|
+
inputTokens: sum((call) => call.inputTokens),
|
|
82
|
+
cacheReadTokens: sum((call) => call.cacheReadTokens),
|
|
83
|
+
cacheWriteTokens: sum((call) => call.cacheWriteTokens),
|
|
84
|
+
outputTokens: usage.outputTokens,
|
|
85
|
+
};
|
|
86
|
+
}
|
package/dist/subagents.js
CHANGED
|
@@ -137,7 +137,13 @@ export function startSubagentWatcher(root, opts) {
|
|
|
137
137
|
return;
|
|
138
138
|
const message = event.message;
|
|
139
139
|
const u = claudeCallUsage(message.usage);
|
|
140
|
-
|
|
140
|
+
// Every input class counts towards "did this turn report usage at all": a turn riding a
|
|
141
|
+
// warm cache legitimately reports 0 fresh input, and skipping it would drop precisely the
|
|
142
|
+
// cache-heavy calls this telemetry exists to weigh.
|
|
143
|
+
if (u.inputTokens === 0 &&
|
|
144
|
+
u.cacheReadTokens === 0 &&
|
|
145
|
+
u.cacheWriteTokens === 0 &&
|
|
146
|
+
u.outputTokens === 0)
|
|
141
147
|
return;
|
|
142
148
|
const content = Array.isArray(message.content) ? message.content : [];
|
|
143
149
|
const { text, reasoning } = claudeAssistantContent(content);
|
|
@@ -154,11 +160,16 @@ export function startSubagentWatcher(root, opts) {
|
|
|
154
160
|
responseText: redactBody(text, secrets),
|
|
155
161
|
reasoningText: redactBody(reasoning, secrets),
|
|
156
162
|
inputTokens: u.inputTokens,
|
|
157
|
-
|
|
163
|
+
cacheReadTokens: u.cacheReadTokens,
|
|
164
|
+
cacheWriteTokens: u.cacheWriteTokens,
|
|
158
165
|
outputTokens: u.outputTokens,
|
|
159
166
|
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
160
167
|
}, opts.onCallMetric);
|
|
161
|
-
usage
|
|
168
|
+
// The run-level `usage` is the COARSE rotation-window weight, which counts every billed
|
|
169
|
+
// input bucket — unlike the per-call metric above, whose `inputTokens` is fresh-only. Sum
|
|
170
|
+
// all three classes back together here or a cache-heavy subagent looks nearly free to the
|
|
171
|
+
// rotation.
|
|
172
|
+
usage.inputTokens += u.inputTokens + u.cacheReadTokens + u.cacheWriteTokens;
|
|
162
173
|
usage.outputTokens += u.outputTokens;
|
|
163
174
|
};
|
|
164
175
|
const NEWLINE = 0x0a;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.66.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
"hono": "^4.12.32",
|
|
27
27
|
"typescript": "7.0.2",
|
|
28
28
|
"vitest": "^4.1.10",
|
|
29
|
-
"@cat-factory/kernel": "0.
|
|
30
|
-
"@cat-factory/server": "0.
|
|
31
|
-
"@cat-factory/spend": "0.12.
|
|
29
|
+
"@cat-factory/kernel": "0.175.0",
|
|
30
|
+
"@cat-factory/server": "0.165.0",
|
|
31
|
+
"@cat-factory/spend": "0.12.104"
|
|
32
32
|
},
|
|
33
33
|
"scripts": {
|
|
34
34
|
"build": "tsc -p tsconfig.json",
|