@cat-factory/executor-harness 1.64.4 → 1.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runner.js +39 -14
- package/dist/claude-call-aggregator.js +6 -3
- package/dist/claude-stream.js +12 -6
- package/dist/inline.js +29 -1
- package/dist/subagents.js +14 -3
- package/package.json +4 -4
- package/src/agent-runner.ts +58 -18
- package/src/claude-call-aggregator.ts +8 -4
- package/src/claude-stream.ts +15 -7
- package/src/inline.ts +34 -1
- package/src/job.ts +14 -1
- package/src/pi.ts +14 -1
- package/src/subagents.ts +17 -3
package/dist/agent-runner.js
CHANGED
|
@@ -574,7 +574,11 @@ function claudeUsage(raw) {
|
|
|
574
574
|
export async function runCodex(opts) {
|
|
575
575
|
const stats = { toolCalls: 0, assistantChars: 0 };
|
|
576
576
|
let summary = '';
|
|
577
|
-
|
|
577
|
+
// The running CUMULATIVE total, kept in its reported (inclusive) form plus the cached share
|
|
578
|
+
// it contains. `PiRunOutcome.usage` needs the inclusive figure — it is the key-rotation
|
|
579
|
+
// weight — while the fallback call metric below needs the split, so both are derived from
|
|
580
|
+
// this one value rather than one being reconstructed from the other.
|
|
581
|
+
let cumulative;
|
|
578
582
|
// Codex reads its credentials from $CODEX_HOME/auth.json with file-backed
|
|
579
583
|
// storage. CRITICAL: this home must live OUTSIDE the cloned checkout (`opts.cwd`)
|
|
580
584
|
// — the blueprint/requirements/conflict-resolver handlers finish with
|
|
@@ -635,7 +639,7 @@ export async function runCodex(opts) {
|
|
|
635
639
|
opts.onProgress(progress);
|
|
636
640
|
const turnUsage = codexUsage(event);
|
|
637
641
|
if (turnUsage)
|
|
638
|
-
|
|
642
|
+
cumulative = turnUsage;
|
|
639
643
|
// A `token_count` event closes a model turn: pair its per-turn usage with the
|
|
640
644
|
// assistant text seen since the previous turn as one telemetry call.
|
|
641
645
|
const perTurn = codexLastTurnUsage(event);
|
|
@@ -647,7 +651,8 @@ export async function runCodex(opts) {
|
|
|
647
651
|
responseText: redactBody(pendingText, secrets),
|
|
648
652
|
reasoningText: '',
|
|
649
653
|
inputTokens: perTurn.inputTokens,
|
|
650
|
-
|
|
654
|
+
cacheReadTokens: perTurn.cacheReadTokens,
|
|
655
|
+
cacheWriteTokens: perTurn.cacheWriteTokens,
|
|
651
656
|
outputTokens: perTurn.outputTokens,
|
|
652
657
|
finishReason: null,
|
|
653
658
|
}, opts.onCallMetric);
|
|
@@ -673,19 +678,29 @@ export async function runCodex(opts) {
|
|
|
673
678
|
}, prompt, opts, { ...opts.extraEnv, ...(codexHome ? { CODEX_HOME: codexHome } : {}) }, opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : [], onEvent);
|
|
674
679
|
// Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
|
|
675
680
|
// single call from the cumulative total + final text so the run is still observable.
|
|
676
|
-
|
|
681
|
+
// The cumulative total is inclusive of its cached share exactly as a per-turn one is, so
|
|
682
|
+
// it is split the same way rather than being filed wholesale as fresh — which would report
|
|
683
|
+
// a cache-heavy run as if nothing had been cached, the one reading this telemetry exists
|
|
684
|
+
// to rule out.
|
|
685
|
+
if (calls.length === 0 && (cumulative || summary)) {
|
|
677
686
|
publishCallMetric(calls, {
|
|
678
687
|
model: opts.model,
|
|
679
688
|
promptText: redactBody(JSON.stringify(messages), secrets),
|
|
680
689
|
messageCount: messages.length,
|
|
681
690
|
responseText: redactBody(summary, secrets),
|
|
682
691
|
reasoningText: '',
|
|
683
|
-
inputTokens:
|
|
684
|
-
|
|
685
|
-
|
|
692
|
+
inputTokens: Math.max(0, (cumulative?.inputTokens ?? 0) - (cumulative?.cachedInputTokens ?? 0)),
|
|
693
|
+
cacheReadTokens: cumulative?.cachedInputTokens ?? 0,
|
|
694
|
+
// Codex reports no separate cache-WRITE class; 0 rather than guessed.
|
|
695
|
+
cacheWriteTokens: 0,
|
|
696
|
+
outputTokens: cumulative?.outputTokens ?? 0,
|
|
686
697
|
finishReason: null,
|
|
687
698
|
}, opts.onCallMetric);
|
|
688
699
|
}
|
|
700
|
+
// The outcome's usage is the key-rotation WEIGHT, so it keeps the inclusive input count.
|
|
701
|
+
const usage = cumulative
|
|
702
|
+
? { inputTokens: cumulative.inputTokens, outputTokens: cumulative.outputTokens }
|
|
703
|
+
: undefined;
|
|
689
704
|
return {
|
|
690
705
|
summary,
|
|
691
706
|
stats,
|
|
@@ -765,8 +780,6 @@ function codexPlanProgress(event) {
|
|
|
765
780
|
* other shapes put it on `usage` / `info.usage` directly. We read the cumulative
|
|
766
781
|
* total when present so the caller can simply overwrite (not sum) — summing
|
|
767
782
|
* cumulative totals across events would multiply-count. Checked most-likely first.
|
|
768
|
-
* `input_tokens` is the TOTAL prompt count (OpenAI semantics: `cached_input_tokens`
|
|
769
|
-
* is a subset already inside it), so it is NOT summed with the cached share.
|
|
770
783
|
*/
|
|
771
784
|
function codexUsage(event) {
|
|
772
785
|
const info = isObject(event.info) ? event.info : undefined;
|
|
@@ -780,14 +793,21 @@ function codexUsage(event) {
|
|
|
780
793
|
const output = numberOf(raw.output_tokens);
|
|
781
794
|
if (input === 0 && output === 0)
|
|
782
795
|
return undefined;
|
|
783
|
-
return {
|
|
796
|
+
return {
|
|
797
|
+
inputTokens: input,
|
|
798
|
+
cachedInputTokens: numberOf(raw.cached_input_tokens),
|
|
799
|
+
outputTokens: output,
|
|
800
|
+
};
|
|
784
801
|
}
|
|
785
802
|
/**
|
|
786
803
|
* Per-TURN Codex token usage off a `token_count` event's `info.last_token_usage` (the
|
|
787
804
|
* delta for the turn just completed, as opposed to `codexUsage`'s cumulative total).
|
|
788
|
-
*
|
|
789
|
-
*
|
|
790
|
-
*
|
|
805
|
+
*
|
|
806
|
+
* OpenAI semantics: `input_tokens` is the turn's WHOLE prompt count and already INCLUDES
|
|
807
|
+
* the cached share, so the fresh figure is the difference. Clamped at 0 because the two
|
|
808
|
+
* counts come off the same event and a vendor inconsistency must not mint a negative token
|
|
809
|
+
* count. Codex reports no separate cache-WRITE class, so that class is 0 here rather than
|
|
810
|
+
* guessed.
|
|
791
811
|
*/
|
|
792
812
|
function codexLastTurnUsage(event) {
|
|
793
813
|
const info = isObject(event.info) ? event.info : undefined;
|
|
@@ -799,7 +819,12 @@ function codexLastTurnUsage(event) {
|
|
|
799
819
|
const output = numberOf(raw.output_tokens);
|
|
800
820
|
if (input === 0 && output === 0)
|
|
801
821
|
return undefined;
|
|
802
|
-
return {
|
|
822
|
+
return {
|
|
823
|
+
inputTokens: Math.max(0, input - cached),
|
|
824
|
+
cacheReadTokens: cached,
|
|
825
|
+
cacheWriteTokens: 0,
|
|
826
|
+
outputTokens: output,
|
|
827
|
+
};
|
|
803
828
|
}
|
|
804
829
|
/** Dispatch to the configured subscription harness runner. */
|
|
805
830
|
export function runSubscriptionHarness(harness, opts) {
|
|
@@ -45,7 +45,8 @@ export function createClaudeCallAggregator(handlers) {
|
|
|
45
45
|
reasoning: '',
|
|
46
46
|
stopReason: null,
|
|
47
47
|
inputTokens: 0,
|
|
48
|
-
|
|
48
|
+
cacheReadTokens: 0,
|
|
49
|
+
cacheWriteTokens: 0,
|
|
49
50
|
outputTokens: 0,
|
|
50
51
|
toolResults: [],
|
|
51
52
|
toolUses: 0,
|
|
@@ -57,7 +58,8 @@ export function createClaudeCallAggregator(handlers) {
|
|
|
57
58
|
pending.reasoning += reasoning;
|
|
58
59
|
pending.toolUses += toolUses;
|
|
59
60
|
pending.inputTokens = Math.max(pending.inputTokens, usage.inputTokens);
|
|
60
|
-
pending.
|
|
61
|
+
pending.cacheReadTokens = Math.max(pending.cacheReadTokens, usage.cacheReadTokens);
|
|
62
|
+
pending.cacheWriteTokens = Math.max(pending.cacheWriteTokens, usage.cacheWriteTokens);
|
|
61
63
|
pending.outputTokens = Math.max(pending.outputTokens, usage.outputTokens);
|
|
62
64
|
// A block-split response reports its stop reason on the envelope that carries the end of the
|
|
63
65
|
// message; earlier ones report none. Keep the first non-null rather than the last seen.
|
|
@@ -114,7 +116,8 @@ export function createClaudeStreamTelemetry(opts) {
|
|
|
114
116
|
responseText: redactBody(call.text, opts.secrets),
|
|
115
117
|
reasoningText: redactBody(call.reasoning, opts.secrets),
|
|
116
118
|
inputTokens: call.inputTokens,
|
|
117
|
-
|
|
119
|
+
cacheReadTokens: call.cacheReadTokens,
|
|
120
|
+
cacheWriteTokens: call.cacheWriteTokens,
|
|
118
121
|
outputTokens: call.outputTokens,
|
|
119
122
|
finishReason: call.stopReason,
|
|
120
123
|
});
|
package/dist/claude-stream.js
CHANGED
|
@@ -51,16 +51,22 @@ export function claudeAssistantContent(content) {
|
|
|
51
51
|
}
|
|
52
52
|
/**
|
|
53
53
|
* Per-CALL token usage off a Claude `assistant` message's `usage` (this turn only, not
|
|
54
|
-
* the cumulative `result` total).
|
|
55
|
-
*
|
|
54
|
+
* the cumulative `result` total).
|
|
55
|
+
*
|
|
56
|
+
* Anthropic reports all three input classes SEPARATELY and `input_tokens` is already
|
|
57
|
+
* exclusive of both caches, so the three fields here are orthogonal and additive:
|
|
58
|
+
* total input = `inputTokens + cacheReadTokens + cacheWriteTokens`. Do NOT re-lump the
|
|
59
|
+
* reads and the writes — a cache write costs 1.25–2× base input while a read costs ~0.1×,
|
|
60
|
+
* so a turn that keeps invalidating the prefix and one that rides a warm cache are
|
|
61
|
+
* indistinguishable once they are summed.
|
|
56
62
|
*/
|
|
57
63
|
export function claudeCallUsage(raw) {
|
|
58
64
|
if (!isObject(raw))
|
|
59
|
-
return { inputTokens: 0,
|
|
60
|
-
const cached = numberOf(raw.cache_read_input_tokens) + numberOf(raw.cache_creation_input_tokens);
|
|
65
|
+
return { inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, outputTokens: 0 };
|
|
61
66
|
return {
|
|
62
|
-
inputTokens: numberOf(raw.input_tokens)
|
|
63
|
-
|
|
67
|
+
inputTokens: numberOf(raw.input_tokens),
|
|
68
|
+
cacheReadTokens: numberOf(raw.cache_read_input_tokens),
|
|
69
|
+
cacheWriteTokens: numberOf(raw.cache_creation_input_tokens),
|
|
64
70
|
outputTokens: numberOf(raw.output_tokens),
|
|
65
71
|
};
|
|
66
72
|
}
|
package/dist/inline.js
CHANGED
|
@@ -48,7 +48,7 @@ export async function handleInline(job, opts) {
|
|
|
48
48
|
return {
|
|
49
49
|
text: outcome.summary,
|
|
50
50
|
finishReason: deriveFinishReason(outcome.callMetrics),
|
|
51
|
-
...(outcome.usage ? { usage: outcome.usage } : {}),
|
|
51
|
+
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
52
52
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
53
53
|
};
|
|
54
54
|
}
|
|
@@ -56,3 +56,31 @@ export async function handleInline(job, opts) {
|
|
|
56
56
|
await rm(cwd, { recursive: true, force: true }).catch(() => { });
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Split the run's coarse usage into the three orthogonal input classes an {@link InlineResult}
|
|
61
|
+
* carries. `outcome.usage` is the ROTATION-window weight — every billed input bucket summed —
|
|
62
|
+
* so the split has to come from the per-call metrics, the only channel that kept the classes
|
|
63
|
+
* apart. Fresh input is likewise taken from the calls rather than derived by subtraction, so a
|
|
64
|
+
* CLI whose per-call and cumulative counts disagree can never produce a negative class.
|
|
65
|
+
*
|
|
66
|
+
* With no per-call telemetry (an older CLI build that streams nothing) the coarse total is
|
|
67
|
+
* reported as fresh with both cache classes 0. That is the honest reading: nothing is KNOWN to
|
|
68
|
+
* have been cached, and inventing a split would be worse than admitting the channel is silent.
|
|
69
|
+
*/
|
|
70
|
+
function inlineUsage(usage, calls) {
|
|
71
|
+
if (!calls?.length) {
|
|
72
|
+
return {
|
|
73
|
+
inputTokens: usage.inputTokens,
|
|
74
|
+
cacheReadTokens: 0,
|
|
75
|
+
cacheWriteTokens: 0,
|
|
76
|
+
outputTokens: usage.outputTokens,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
const sum = (pick) => calls.reduce((total, call) => total + pick(call), 0);
|
|
80
|
+
return {
|
|
81
|
+
inputTokens: sum((call) => call.inputTokens),
|
|
82
|
+
cacheReadTokens: sum((call) => call.cacheReadTokens),
|
|
83
|
+
cacheWriteTokens: sum((call) => call.cacheWriteTokens),
|
|
84
|
+
outputTokens: usage.outputTokens,
|
|
85
|
+
};
|
|
86
|
+
}
|
package/dist/subagents.js
CHANGED
|
@@ -137,7 +137,13 @@ export function startSubagentWatcher(root, opts) {
|
|
|
137
137
|
return;
|
|
138
138
|
const message = event.message;
|
|
139
139
|
const u = claudeCallUsage(message.usage);
|
|
140
|
-
|
|
140
|
+
// Every input class counts towards "did this turn report usage at all": a turn riding a
|
|
141
|
+
// warm cache legitimately reports 0 fresh input, and skipping it would drop precisely the
|
|
142
|
+
// cache-heavy calls this telemetry exists to weigh.
|
|
143
|
+
if (u.inputTokens === 0 &&
|
|
144
|
+
u.cacheReadTokens === 0 &&
|
|
145
|
+
u.cacheWriteTokens === 0 &&
|
|
146
|
+
u.outputTokens === 0)
|
|
141
147
|
return;
|
|
142
148
|
const content = Array.isArray(message.content) ? message.content : [];
|
|
143
149
|
const { text, reasoning } = claudeAssistantContent(content);
|
|
@@ -154,11 +160,16 @@ export function startSubagentWatcher(root, opts) {
|
|
|
154
160
|
responseText: redactBody(text, secrets),
|
|
155
161
|
reasoningText: redactBody(reasoning, secrets),
|
|
156
162
|
inputTokens: u.inputTokens,
|
|
157
|
-
|
|
163
|
+
cacheReadTokens: u.cacheReadTokens,
|
|
164
|
+
cacheWriteTokens: u.cacheWriteTokens,
|
|
158
165
|
outputTokens: u.outputTokens,
|
|
159
166
|
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
160
167
|
}, opts.onCallMetric);
|
|
161
|
-
usage
|
|
168
|
+
// The run-level `usage` is the COARSE rotation-window weight, which counts every billed
|
|
169
|
+
// input bucket — unlike the per-call metric above, whose `inputTokens` is fresh-only. Sum
|
|
170
|
+
// all three classes back together here or a cache-heavy subagent looks nearly free to the
|
|
171
|
+
// rotation.
|
|
172
|
+
usage.inputTokens += u.inputTokens + u.cacheReadTokens + u.cacheWriteTokens;
|
|
162
173
|
usage.outputTokens += u.outputTokens;
|
|
163
174
|
};
|
|
164
175
|
const NEWLINE = 0x0a;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.66.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
"hono": "^4.12.32",
|
|
27
27
|
"typescript": "7.0.2",
|
|
28
28
|
"vitest": "^4.1.10",
|
|
29
|
-
"@cat-factory/kernel": "0.
|
|
30
|
-
"@cat-factory/server": "0.
|
|
31
|
-
"@cat-factory/spend": "0.12.
|
|
29
|
+
"@cat-factory/kernel": "0.175.0",
|
|
30
|
+
"@cat-factory/server": "0.165.0",
|
|
31
|
+
"@cat-factory/spend": "0.12.104"
|
|
32
32
|
},
|
|
33
33
|
"scripts": {
|
|
34
34
|
"build": "tsc -p tsconfig.json",
|
package/src/agent-runner.ts
CHANGED
|
@@ -747,7 +747,11 @@ function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number
|
|
|
747
747
|
export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
748
748
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
749
749
|
let summary = ''
|
|
750
|
-
|
|
750
|
+
// The running CUMULATIVE total, kept in its reported (inclusive) form plus the cached share
|
|
751
|
+
// it contains. `PiRunOutcome.usage` needs the inclusive figure — it is the key-rotation
|
|
752
|
+
// weight — while the fallback call metric below needs the split, so both are derived from
|
|
753
|
+
// this one value rather than one being reconstructed from the other.
|
|
754
|
+
let cumulative: CodexCumulativeUsage | undefined
|
|
751
755
|
|
|
752
756
|
// Codex reads its credentials from $CODEX_HOME/auth.json with file-backed
|
|
753
757
|
// storage. CRITICAL: this home must live OUTSIDE the cloned checkout (`opts.cwd`)
|
|
@@ -812,7 +816,7 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
812
816
|
const progress = codexPlanProgress(event)
|
|
813
817
|
if (progress && opts.onProgress) opts.onProgress(progress)
|
|
814
818
|
const turnUsage = codexUsage(event)
|
|
815
|
-
if (turnUsage)
|
|
819
|
+
if (turnUsage) cumulative = turnUsage
|
|
816
820
|
// A `token_count` event closes a model turn: pair its per-turn usage with the
|
|
817
821
|
// assistant text seen since the previous turn as one telemetry call.
|
|
818
822
|
const perTurn = codexLastTurnUsage(event)
|
|
@@ -826,7 +830,8 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
826
830
|
responseText: redactBody(pendingText, secrets),
|
|
827
831
|
reasoningText: '',
|
|
828
832
|
inputTokens: perTurn.inputTokens,
|
|
829
|
-
|
|
833
|
+
cacheReadTokens: perTurn.cacheReadTokens,
|
|
834
|
+
cacheWriteTokens: perTurn.cacheWriteTokens,
|
|
830
835
|
outputTokens: perTurn.outputTokens,
|
|
831
836
|
finishReason: null,
|
|
832
837
|
},
|
|
@@ -862,7 +867,11 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
862
867
|
|
|
863
868
|
// Fallback for a CLI/version that never emits per-turn `last_token_usage`: record a
|
|
864
869
|
// single call from the cumulative total + final text so the run is still observable.
|
|
865
|
-
|
|
870
|
+
// The cumulative total is inclusive of its cached share exactly as a per-turn one is, so
|
|
871
|
+
// it is split the same way rather than being filed wholesale as fresh — which would report
|
|
872
|
+
// a cache-heavy run as if nothing had been cached, the one reading this telemetry exists
|
|
873
|
+
// to rule out.
|
|
874
|
+
if (calls.length === 0 && (cumulative || summary)) {
|
|
866
875
|
publishCallMetric(
|
|
867
876
|
calls,
|
|
868
877
|
{
|
|
@@ -871,14 +880,23 @@ export async function runCodex(opts: SubscriptionRunOptions): Promise<PiRunOutco
|
|
|
871
880
|
messageCount: messages.length,
|
|
872
881
|
responseText: redactBody(summary, secrets),
|
|
873
882
|
reasoningText: '',
|
|
874
|
-
inputTokens:
|
|
875
|
-
|
|
876
|
-
|
|
883
|
+
inputTokens: Math.max(
|
|
884
|
+
0,
|
|
885
|
+
(cumulative?.inputTokens ?? 0) - (cumulative?.cachedInputTokens ?? 0),
|
|
886
|
+
),
|
|
887
|
+
cacheReadTokens: cumulative?.cachedInputTokens ?? 0,
|
|
888
|
+
// Codex reports no separate cache-WRITE class; 0 rather than guessed.
|
|
889
|
+
cacheWriteTokens: 0,
|
|
890
|
+
outputTokens: cumulative?.outputTokens ?? 0,
|
|
877
891
|
finishReason: null,
|
|
878
892
|
},
|
|
879
893
|
opts.onCallMetric,
|
|
880
894
|
)
|
|
881
895
|
}
|
|
896
|
+
// The outcome's usage is the key-rotation WEIGHT, so it keeps the inclusive input count.
|
|
897
|
+
const usage = cumulative
|
|
898
|
+
? { inputTokens: cumulative.inputTokens, outputTokens: cumulative.outputTokens }
|
|
899
|
+
: undefined
|
|
882
900
|
return {
|
|
883
901
|
summary,
|
|
884
902
|
stats,
|
|
@@ -949,6 +967,19 @@ function codexPlanProgress(event: Record<string, unknown>): TodoProgress | undef
|
|
|
949
967
|
return toProgress(items)
|
|
950
968
|
}
|
|
951
969
|
|
|
970
|
+
/**
|
|
971
|
+
* Codex's running cumulative usage, kept in the form the CLI reports it: `inputTokens` is the
|
|
972
|
+
* TOTAL prompt count (OpenAI semantics) with `cachedInputTokens` a SUBSET already inside it,
|
|
973
|
+
* never a bucket to add on top. The cached share is carried rather than discarded so a
|
|
974
|
+
* consumer that needs the fresh figure can subtract it at the point of use, instead of the
|
|
975
|
+
* only two readings of this number being "inclusive" and "lost".
|
|
976
|
+
*/
|
|
977
|
+
interface CodexCumulativeUsage {
|
|
978
|
+
inputTokens: number
|
|
979
|
+
cachedInputTokens: number
|
|
980
|
+
outputTokens: number
|
|
981
|
+
}
|
|
982
|
+
|
|
952
983
|
/**
|
|
953
984
|
* Best-effort: pull token usage out of a Codex usage event. Codex `exec --json`
|
|
954
985
|
* reports a running CUMULATIVE total on `token_count` events under
|
|
@@ -956,12 +987,8 @@ function codexPlanProgress(event: Record<string, unknown>): TodoProgress | undef
|
|
|
956
987
|
* other shapes put it on `usage` / `info.usage` directly. We read the cumulative
|
|
957
988
|
* total when present so the caller can simply overwrite (not sum) — summing
|
|
958
989
|
* cumulative totals across events would multiply-count. Checked most-likely first.
|
|
959
|
-
* `input_tokens` is the TOTAL prompt count (OpenAI semantics: `cached_input_tokens`
|
|
960
|
-
* is a subset already inside it), so it is NOT summed with the cached share.
|
|
961
990
|
*/
|
|
962
|
-
function codexUsage(
|
|
963
|
-
event: Record<string, unknown>,
|
|
964
|
-
): { inputTokens: number; outputTokens: number } | undefined {
|
|
991
|
+
function codexUsage(event: Record<string, unknown>): CodexCumulativeUsage | undefined {
|
|
965
992
|
const info = isObject(event.info) ? (event.info as Record<string, unknown>) : undefined
|
|
966
993
|
const raw =
|
|
967
994
|
(info && isObject(info.total_token_usage) ? info.total_token_usage : undefined) ??
|
|
@@ -972,20 +999,28 @@ function codexUsage(
|
|
|
972
999
|
const input = numberOf(raw.input_tokens)
|
|
973
1000
|
const output = numberOf(raw.output_tokens)
|
|
974
1001
|
if (input === 0 && output === 0) return undefined
|
|
975
|
-
return {
|
|
1002
|
+
return {
|
|
1003
|
+
inputTokens: input,
|
|
1004
|
+
cachedInputTokens: numberOf(raw.cached_input_tokens),
|
|
1005
|
+
outputTokens: output,
|
|
1006
|
+
}
|
|
976
1007
|
}
|
|
977
1008
|
|
|
978
1009
|
/**
|
|
979
1010
|
* Per-TURN Codex token usage off a `token_count` event's `info.last_token_usage` (the
|
|
980
1011
|
* delta for the turn just completed, as opposed to `codexUsage`'s cumulative total).
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
*
|
|
1012
|
+
*
|
|
1013
|
+
* OpenAI semantics: `input_tokens` is the turn's WHOLE prompt count and already INCLUDES
|
|
1014
|
+
* the cached share, so the fresh figure is the difference. Clamped at 0 because the two
|
|
1015
|
+
* counts come off the same event and a vendor inconsistency must not mint a negative token
|
|
1016
|
+
* count. Codex reports no separate cache-WRITE class, so that class is 0 here rather than
|
|
1017
|
+
* guessed.
|
|
984
1018
|
*/
|
|
985
1019
|
function codexLastTurnUsage(event: Record<string, unknown>):
|
|
986
1020
|
| {
|
|
987
1021
|
inputTokens: number
|
|
988
|
-
|
|
1022
|
+
cacheReadTokens: number
|
|
1023
|
+
cacheWriteTokens: number
|
|
989
1024
|
outputTokens: number
|
|
990
1025
|
}
|
|
991
1026
|
| undefined {
|
|
@@ -996,7 +1031,12 @@ function codexLastTurnUsage(event: Record<string, unknown>):
|
|
|
996
1031
|
const cached = numberOf(raw.cached_input_tokens)
|
|
997
1032
|
const output = numberOf(raw.output_tokens)
|
|
998
1033
|
if (input === 0 && output === 0) return undefined
|
|
999
|
-
return {
|
|
1034
|
+
return {
|
|
1035
|
+
inputTokens: Math.max(0, input - cached),
|
|
1036
|
+
cacheReadTokens: cached,
|
|
1037
|
+
cacheWriteTokens: 0,
|
|
1038
|
+
outputTokens: output,
|
|
1039
|
+
}
|
|
1000
1040
|
}
|
|
1001
1041
|
|
|
1002
1042
|
/** Dispatch to the configured subscription harness runner. */
|
|
@@ -22,7 +22,8 @@ export interface AggregatedClaudeCall {
|
|
|
22
22
|
reasoning: string
|
|
23
23
|
stopReason: string | null
|
|
24
24
|
inputTokens: number
|
|
25
|
-
|
|
25
|
+
cacheReadTokens: number
|
|
26
|
+
cacheWriteTokens: number
|
|
26
27
|
outputTokens: number
|
|
27
28
|
/** The `user` turns carrying this call's tool_result blocks, in arrival order. */
|
|
28
29
|
toolResults: unknown[][]
|
|
@@ -95,7 +96,8 @@ export function createClaudeCallAggregator(handlers: {
|
|
|
95
96
|
reasoning: '',
|
|
96
97
|
stopReason: null,
|
|
97
98
|
inputTokens: 0,
|
|
98
|
-
|
|
99
|
+
cacheReadTokens: 0,
|
|
100
|
+
cacheWriteTokens: 0,
|
|
99
101
|
outputTokens: 0,
|
|
100
102
|
toolResults: [],
|
|
101
103
|
toolUses: 0,
|
|
@@ -107,7 +109,8 @@ export function createClaudeCallAggregator(handlers: {
|
|
|
107
109
|
pending.reasoning += reasoning
|
|
108
110
|
pending.toolUses += toolUses
|
|
109
111
|
pending.inputTokens = Math.max(pending.inputTokens, usage.inputTokens)
|
|
110
|
-
pending.
|
|
112
|
+
pending.cacheReadTokens = Math.max(pending.cacheReadTokens, usage.cacheReadTokens)
|
|
113
|
+
pending.cacheWriteTokens = Math.max(pending.cacheWriteTokens, usage.cacheWriteTokens)
|
|
111
114
|
pending.outputTokens = Math.max(pending.outputTokens, usage.outputTokens)
|
|
112
115
|
// A block-split response reports its stop reason on the envelope that carries the end of the
|
|
113
116
|
// message; earlier ones report none. Keep the first non-null rather than the last seen.
|
|
@@ -185,7 +188,8 @@ export function createClaudeStreamTelemetry(opts: {
|
|
|
185
188
|
responseText: redactBody(call.text, opts.secrets),
|
|
186
189
|
reasoningText: redactBody(call.reasoning, opts.secrets),
|
|
187
190
|
inputTokens: call.inputTokens,
|
|
188
|
-
|
|
191
|
+
cacheReadTokens: call.cacheReadTokens,
|
|
192
|
+
cacheWriteTokens: call.cacheWriteTokens,
|
|
189
193
|
outputTokens: call.outputTokens,
|
|
190
194
|
finishReason: call.stopReason,
|
|
191
195
|
})
|
package/src/claude-stream.ts
CHANGED
|
@@ -59,19 +59,27 @@ export function claudeAssistantContent(content: unknown[]): {
|
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
61
|
* Per-CALL token usage off a Claude `assistant` message's `usage` (this turn only, not
|
|
62
|
-
* the cumulative `result` total).
|
|
63
|
-
*
|
|
62
|
+
* the cumulative `result` total).
|
|
63
|
+
*
|
|
64
|
+
* Anthropic reports all three input classes SEPARATELY and `input_tokens` is already
|
|
65
|
+
* exclusive of both caches, so the three fields here are orthogonal and additive:
|
|
66
|
+
* total input = `inputTokens + cacheReadTokens + cacheWriteTokens`. Do NOT re-lump the
|
|
67
|
+
* reads and the writes — a cache write costs 1.25–2× base input while a read costs ~0.1×,
|
|
68
|
+
* so a turn that keeps invalidating the prefix and one that rides a warm cache are
|
|
69
|
+
* indistinguishable once they are summed.
|
|
64
70
|
*/
|
|
65
71
|
export function claudeCallUsage(raw: unknown): {
|
|
66
72
|
inputTokens: number
|
|
67
|
-
|
|
73
|
+
cacheReadTokens: number
|
|
74
|
+
cacheWriteTokens: number
|
|
68
75
|
outputTokens: number
|
|
69
76
|
} {
|
|
70
|
-
if (!isObject(raw))
|
|
71
|
-
|
|
77
|
+
if (!isObject(raw))
|
|
78
|
+
return { inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, outputTokens: 0 }
|
|
72
79
|
return {
|
|
73
|
-
inputTokens: numberOf(raw.input_tokens)
|
|
74
|
-
|
|
80
|
+
inputTokens: numberOf(raw.input_tokens),
|
|
81
|
+
cacheReadTokens: numberOf(raw.cache_read_input_tokens),
|
|
82
|
+
cacheWriteTokens: numberOf(raw.cache_creation_input_tokens),
|
|
75
83
|
outputTokens: numberOf(raw.output_tokens),
|
|
76
84
|
}
|
|
77
85
|
}
|
package/src/inline.ts
CHANGED
|
@@ -54,10 +54,43 @@ export async function handleInline(job: InlineJob, opts: RunOptions): Promise<In
|
|
|
54
54
|
return {
|
|
55
55
|
text: outcome.summary,
|
|
56
56
|
finishReason: deriveFinishReason(outcome.callMetrics),
|
|
57
|
-
...(outcome.usage ? { usage: outcome.usage } : {}),
|
|
57
|
+
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
58
58
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
59
59
|
}
|
|
60
60
|
} finally {
|
|
61
61
|
await rm(cwd, { recursive: true, force: true }).catch(() => {})
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Split the run's coarse usage into the three orthogonal input classes an {@link InlineResult}
|
|
67
|
+
* carries. `outcome.usage` is the ROTATION-window weight — every billed input bucket summed —
|
|
68
|
+
* so the split has to come from the per-call metrics, the only channel that kept the classes
|
|
69
|
+
* apart. Fresh input is likewise taken from the calls rather than derived by subtraction, so a
|
|
70
|
+
* CLI whose per-call and cumulative counts disagree can never produce a negative class.
|
|
71
|
+
*
|
|
72
|
+
* With no per-call telemetry (an older CLI build that streams nothing) the coarse total is
|
|
73
|
+
* reported as fresh with both cache classes 0. That is the honest reading: nothing is KNOWN to
|
|
74
|
+
* have been cached, and inventing a split would be worse than admitting the channel is silent.
|
|
75
|
+
*/
|
|
76
|
+
function inlineUsage(
|
|
77
|
+
usage: { inputTokens: number; outputTokens: number },
|
|
78
|
+
calls: HarnessCallMetric[] | undefined,
|
|
79
|
+
): NonNullable<InlineResult['usage']> {
|
|
80
|
+
if (!calls?.length) {
|
|
81
|
+
return {
|
|
82
|
+
inputTokens: usage.inputTokens,
|
|
83
|
+
cacheReadTokens: 0,
|
|
84
|
+
cacheWriteTokens: 0,
|
|
85
|
+
outputTokens: usage.outputTokens,
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const sum = (pick: (call: HarnessCallMetric) => number): number =>
|
|
89
|
+
calls.reduce((total, call) => total + pick(call), 0)
|
|
90
|
+
return {
|
|
91
|
+
inputTokens: sum((call) => call.inputTokens),
|
|
92
|
+
cacheReadTokens: sum((call) => call.cacheReadTokens),
|
|
93
|
+
cacheWriteTokens: sum((call) => call.cacheWriteTokens),
|
|
94
|
+
outputTokens: usage.outputTokens,
|
|
95
|
+
}
|
|
96
|
+
}
|
package/src/job.ts
CHANGED
|
@@ -1251,7 +1251,20 @@ export interface InlineResult {
|
|
|
1251
1251
|
text: string
|
|
1252
1252
|
/** `length` when the model hit its output cap (the reviewer rejects a truncated doc). */
|
|
1253
1253
|
finishReason?: 'stop' | 'length'
|
|
1254
|
-
|
|
1254
|
+
/**
|
|
1255
|
+
* The job's token usage with the input side split into its three ORTHOGONAL classes:
|
|
1256
|
+
* `inputTokens` is FRESH input only, so the total input is
|
|
1257
|
+
* `inputTokens + cacheReadTokens + cacheWriteTokens`. Folded from the per-call metrics below,
|
|
1258
|
+
* which is the only channel that knows the split; a CLI that streamed none falls back to the
|
|
1259
|
+
* coarse total with both cache classes 0 — honest, since on that shape nothing is known to
|
|
1260
|
+
* have been cached.
|
|
1261
|
+
*/
|
|
1262
|
+
usage?: {
|
|
1263
|
+
inputTokens: number
|
|
1264
|
+
cacheReadTokens: number
|
|
1265
|
+
cacheWriteTokens: number
|
|
1266
|
+
outputTokens: number
|
|
1267
|
+
}
|
|
1255
1268
|
/** Per-model-call telemetry lifted from the CLI stream (recorded into `llm_call_metrics`). */
|
|
1256
1269
|
callMetrics?: HarnessCallMetric[]
|
|
1257
1270
|
/** A structured failure marks a job-level failure even on a clean HTTP exit (see JobResultBase). */
|
package/src/pi.ts
CHANGED
|
@@ -509,8 +509,21 @@ export interface HarnessCallMetric {
|
|
|
509
509
|
responseText: string
|
|
510
510
|
/** The reasoning/thinking trace, as a plain string (`''` when none). */
|
|
511
511
|
reasoningText: string
|
|
512
|
+
/**
|
|
513
|
+
* FRESH (uncached) input tokens: exclusive of BOTH cache classes below, so the three
|
|
514
|
+
* are orthogonal and additive. Every producer normalises to this — reading the already
|
|
515
|
+
* exclusive field where the vendor reports the classes apart (Anthropic), subtracting
|
|
516
|
+
* the cached share where the vendor reports an inclusive prompt count (Codex/OpenAI).
|
|
517
|
+
*/
|
|
512
518
|
inputTokens: number
|
|
513
|
-
|
|
519
|
+
/** Input tokens served from the vendor's prompt cache (~0.1× base input). */
|
|
520
|
+
cacheReadTokens: number
|
|
521
|
+
/**
|
|
522
|
+
* Input tokens written INTO the vendor's cache (1.25–2× base input — dearer than fresh),
|
|
523
|
+
* kept apart from the reads so a loop that keeps re-writing the prefix is distinguishable
|
|
524
|
+
* from one riding a warm cache. 0 where the CLI reports no separate write class.
|
|
525
|
+
*/
|
|
526
|
+
cacheWriteTokens: number
|
|
514
527
|
outputTokens: number
|
|
515
528
|
/** The provider finish/stop reason when the CLI reports one (else null). */
|
|
516
529
|
finishReason: string | null
|
package/src/subagents.ts
CHANGED
|
@@ -236,7 +236,16 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
|
|
|
236
236
|
if (event.type !== 'assistant' || !isObject(event.message)) return
|
|
237
237
|
const message = event.message as Record<string, unknown>
|
|
238
238
|
const u = claudeCallUsage(message.usage)
|
|
239
|
-
|
|
239
|
+
// Every input class counts towards "did this turn report usage at all": a turn riding a
|
|
240
|
+
// warm cache legitimately reports 0 fresh input, and skipping it would drop precisely the
|
|
241
|
+
// cache-heavy calls this telemetry exists to weigh.
|
|
242
|
+
if (
|
|
243
|
+
u.inputTokens === 0 &&
|
|
244
|
+
u.cacheReadTokens === 0 &&
|
|
245
|
+
u.cacheWriteTokens === 0 &&
|
|
246
|
+
u.outputTokens === 0
|
|
247
|
+
)
|
|
248
|
+
return
|
|
240
249
|
const content = Array.isArray(message.content) ? message.content : []
|
|
241
250
|
const { text, reasoning } = claudeAssistantContent(content)
|
|
242
251
|
publishCallMetric(
|
|
@@ -254,13 +263,18 @@ export function startSubagentWatcher(root: string, opts: SubagentWatcherOptions)
|
|
|
254
263
|
responseText: redactBody(text, secrets),
|
|
255
264
|
reasoningText: redactBody(reasoning, secrets),
|
|
256
265
|
inputTokens: u.inputTokens,
|
|
257
|
-
|
|
266
|
+
cacheReadTokens: u.cacheReadTokens,
|
|
267
|
+
cacheWriteTokens: u.cacheWriteTokens,
|
|
258
268
|
outputTokens: u.outputTokens,
|
|
259
269
|
finishReason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
|
260
270
|
},
|
|
261
271
|
opts.onCallMetric,
|
|
262
272
|
)
|
|
263
|
-
usage
|
|
273
|
+
// The run-level `usage` is the COARSE rotation-window weight, which counts every billed
|
|
274
|
+
// input bucket — unlike the per-call metric above, whose `inputTokens` is fresh-only. Sum
|
|
275
|
+
// all three classes back together here or a cache-heavy subagent looks nearly free to the
|
|
276
|
+
// rotation.
|
|
277
|
+
usage.inputTokens += u.inputTokens + u.cacheReadTokens + u.cacheWriteTokens
|
|
264
278
|
usage.outputTokens += u.outputTokens
|
|
265
279
|
}
|
|
266
280
|
|