@cat-factory/executor-harness 1.114.0 → 1.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -1
- package/dist/agent-runner.js +68 -74
- package/dist/claude-call-aggregator.d.ts +9 -0
- package/dist/claude-call-aggregator.js +10 -1
- package/dist/coding-agent.js +50 -9
- package/dist/failure.d.ts +7 -1
- package/dist/failure.js +7 -0
- package/dist/git.d.ts +118 -3
- package/dist/git.js +162 -9
- package/dist/inline.js +20 -6
- package/dist/job.d.ts +8 -1
- package/dist/pi.d.ts +17 -24
- package/dist/pi.js +7 -41
- package/dist/usage-attribution.d.ts +56 -0
- package/dist/usage-attribution.js +96 -0
- package/package.json +4 -4
- package/src/agent-runner.ts +89 -80
- package/src/claude-call-aggregator.ts +15 -2
- package/src/coding-agent.ts +52 -8
- package/src/failure.ts +7 -0
- package/src/git.ts +212 -9
- package/src/inline.ts +20 -7
- package/src/job.ts +8 -1
- package/src/pi.ts +17 -51
- package/src/usage-attribution.ts +102 -0
package/dist/git.js
CHANGED
|
@@ -91,6 +91,48 @@ export function isGitTimeoutKill(err, aborted) {
|
|
|
91
91
|
function gitSubcommand(args) {
|
|
92
92
|
return args.find((a) => a !== '' && !a.startsWith('-')) ?? 'command';
|
|
93
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* Whether `stderr` is a REFUSED push, and which shape. Ordered: the lease/fetch-first shapes are
|
|
96
|
+
* checked first, because a `(stale info)` refusal also prints the generic "failed to push some
|
|
97
|
+
* refs" line the non-fast-forward shape shares. Pure, so both branches are unit-tested against
|
|
98
|
+
* real git output rather than inferred.
|
|
99
|
+
*/
|
|
100
|
+
export function classifyPushRejection(stderr) {
|
|
101
|
+
// A HOST-side refusal is not contention, and re-dispatching cannot help: branch protection, a
|
|
102
|
+
// pre-receive hook or a token policy is declining the write itself, and GitHub's protected-branch
|
|
103
|
+
// message says "refusing to allow a non-fast-forward push", which would otherwise read as a
|
|
104
|
+
// rewrite. Git's own labels separate the two cleanly (`! [remote rejected]` is the server
|
|
105
|
+
// declining, `! [rejected]` is git's own fast-forward/lease check), so such a failure stays a
|
|
106
|
+
// plain `git` fault with the write-access remedy below.
|
|
107
|
+
if (/remote rejected|protected branch|hook declined|refusing to allow/i.test(stderr)) {
|
|
108
|
+
return undefined;
|
|
109
|
+
}
|
|
110
|
+
if (/\(stale info\)|\(fetch first\)|remote contains work that you do not/i.test(stderr)) {
|
|
111
|
+
return 'remote-writer';
|
|
112
|
+
}
|
|
113
|
+
if (/\(non-fast-forward\)|tip of your current branch is behind|branch tip is behind/i.test(stderr)) {
|
|
114
|
+
return 'local-rewrite';
|
|
115
|
+
}
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The remedy each {@link PushRejection} earns. A `Record`, so a new rejection shape cannot be
|
|
120
|
+
* classified without saying what a human should do about it. Neither is "run `git pull`", which is
|
|
121
|
+
* what git's own hint advises and is advice for a person at a terminal, not for an autonomous run.
|
|
122
|
+
*/
|
|
123
|
+
const PUSH_REJECTION_REMEDIES = {
|
|
124
|
+
'local-rewrite': 'The push was refused because the commit it publishes is not descended from the one the work ' +
|
|
125
|
+
'branch already holds: this checkout rewrote history that had already been pushed (an amend, ' +
|
|
126
|
+
"reset or rebase of an existing commit). The platform checkpoint-pushes the agent's commits " +
|
|
127
|
+
'while it works and lets a run force over its OWN published checkpoint, so what stays refused ' +
|
|
128
|
+
'is a rewrite it cannot attribute to this pass: commits an earlier run published, or a rewrite ' +
|
|
129
|
+
'that dropped the branch tip this pass started from. The engine re-dispatches the step to ' +
|
|
130
|
+
'resume from the branch as it stands; work already on the branch is never dropped.',
|
|
131
|
+
'remote-writer': 'The push was refused because another writer advanced this work branch while the run was ' +
|
|
132
|
+
'working (a second dispatch for the same block, or a person pushing to it). Nothing is lost: ' +
|
|
133
|
+
"the other writer's commits stay on the branch and the engine re-dispatches the step so the " +
|
|
134
|
+
'agent resumes on top of them. If it recurs, check whether two runs are active for the same block.',
|
|
135
|
+
};
|
|
94
136
|
/**
|
|
95
137
|
* Classify the common shapes of git's own stderr into an actionable remedy, else undefined
|
|
96
138
|
* (an unrecognized failure keeps just its raw stderr). This is the FIRST-WRAP-POINT for
|
|
@@ -102,6 +144,11 @@ function gitSubcommand(args) {
|
|
|
102
144
|
*/
|
|
103
145
|
export function describeGitFailure(stderr) {
|
|
104
146
|
const s = stderr.toLowerCase();
|
|
147
|
+
// A refused push first: its stderr carries neither an auth nor an access shape, so a miss here
|
|
148
|
+
// would leave the operator git's own "use 'git pull' before pushing again" hint and nothing else.
|
|
149
|
+
const rejection = classifyPushRejection(stderr);
|
|
150
|
+
if (rejection)
|
|
151
|
+
return PUSH_REJECTION_REMEDIES[rejection];
|
|
105
152
|
// Rate-limit / abuse-detection first: the host returns these as a 403, which would
|
|
106
153
|
// otherwise fall into the write-access shape below and be mislabeled as a permission
|
|
107
154
|
// problem — but the fix is to wait, not to grant access.
|
|
@@ -154,13 +201,21 @@ function gitFailure(err, args, aborted) {
|
|
|
154
201
|
}
|
|
155
202
|
const stderr = typeof e?.stderr === 'string' ? e.stderr : (e?.stderr?.toString() ?? '');
|
|
156
203
|
const base = e instanceof Error ? e.message : String(err);
|
|
157
|
-
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
//
|
|
204
|
+
// `execFile` builds its rejection message as `Command failed: <cmd>\n<stderr>`, so for the
|
|
205
|
+
// ordinary non-zero exit the stderr is ALREADY in `base`, and appending it again printed every
|
|
206
|
+
// git failure's output twice, which reads as two attempts. Append only what `base` lacks
|
|
207
|
+
// (a killed/other rejection whose message carries no output).
|
|
208
|
+
const tail = stderr.trim();
|
|
209
|
+
const combined = tail && !base.includes(tail) ? `${base}\n${tail}` : base;
|
|
210
|
+
// Append a cause + fix for the recognized auth/access/push-rejection shapes, keeping the raw
|
|
211
|
+
// (scrubbed) stderr above it as the detail. The remedy is static text with no secrets, so it is
|
|
212
|
+
// added after redaction.
|
|
161
213
|
const remedy = describeGitFailure(combined);
|
|
162
214
|
const message = remedy ? `${redactSecrets(combined)}\n${remedy}` : redactSecrets(combined);
|
|
163
|
-
|
|
215
|
+
// A REFUSED push is not a generic `git` fault: the branch moved under this run, which the engine
|
|
216
|
+
// recovers from by re-dispatching the step onto the branch as it now stands. It gets its own
|
|
217
|
+
// structured cause so that recovery keys off a classification rather than this message.
|
|
218
|
+
const failure = new HarnessFailure(classifyPushRejection(combined) ? 'branch-contended' : 'git', message);
|
|
164
219
|
if (e?.stack)
|
|
165
220
|
failure.stack = redactSecrets(e.stack);
|
|
166
221
|
return failure;
|
|
@@ -851,15 +906,113 @@ export async function fetchPullRequestHead(opts) {
|
|
|
851
906
|
}
|
|
852
907
|
}
|
|
853
908
|
/**
|
|
854
|
-
* Push the work branch to origin. The remote URL carries only the
|
|
855
|
-
* the token is supplied here via the askpass env (never in argv).
|
|
909
|
+
* Push the work branch to origin and return the sha it PUBLISHED. The remote URL carries only the
|
|
910
|
+
* username, so the token is supplied here via the askpass env (never in argv).
|
|
911
|
+
*
|
|
912
|
+
* The push names an explicit SOURCE COMMIT (`<sha>:refs/heads/<branch>`) rather than the branch,
|
|
913
|
+
* which is what makes the return value exact rather than a guess. The agent commits while this
|
|
914
|
+
* runs, so `git push origin <branch>` publishes whatever the branch ref holds at the moment git
|
|
915
|
+
* reads it, and a caller that leases against a sha it read either side of that has leased against
|
|
916
|
+
* the wrong commit. Reading it back from `refs/remotes/origin/<branch>` afterwards is worse than
|
|
917
|
+
* inexact, it is EMPTY on the production checkout: a fresh coding run clones one branch
|
|
918
|
+
* (`cloneRepo`), so the remote's fetch refspec covers the base alone and `git push` creates no
|
|
919
|
+
* tracking ref for the work branch at all. Naming the sha needs no ref and no round trip.
|
|
920
|
+
*
|
|
921
|
+
* `-u` goes with it: with a non-branch source git sets no upstream config (verified), nothing in
|
|
922
|
+
* the harness reads that config, and the agent is told never to push or pull.
|
|
923
|
+
*
|
|
924
|
+
* `expectRemoteSha` turns the push into a LEASED force (`--force-with-lease=<branch>:<sha>`), which
|
|
925
|
+
* is how a run whose own checkpoint push it has since rewritten still lands. It is deliberately NOT
|
|
926
|
+
* a plain `--force`: the lease succeeds only while the remote still holds the sha THIS run
|
|
927
|
+
* published, so a second writer's commits refuse the push (`(stale info)`) instead of being
|
|
928
|
+
* clobbered. Callers therefore pass only a sha this same pass published; leasing against a tip we
|
|
929
|
+
* merely CLONED would force over an earlier run's work.
|
|
856
930
|
*/
|
|
857
|
-
export async function pushBranch(dir, branch, ghToken, signal) {
|
|
858
|
-
await git(['
|
|
931
|
+
export async function pushBranch(dir, branch, ghToken, signal, opts = {}) {
|
|
932
|
+
const sha = (await git(['rev-parse', '--verify', `refs/heads/${branch}`], { cwd: dir, signal })).trim();
|
|
933
|
+
const lease = opts.expectRemoteSha ? [`--force-with-lease=${branch}:${opts.expectRemoteSha}`] : [];
|
|
934
|
+
await git(['push', ...lease, 'origin', `${sha}:refs/heads/${branch}`], {
|
|
859
935
|
cwd: dir,
|
|
860
936
|
signal,
|
|
861
937
|
env: await authEnv(ghToken),
|
|
862
938
|
});
|
|
939
|
+
return sha;
|
|
940
|
+
}
|
|
941
|
+
/**
|
|
942
|
+
* Whether `sha` is still reachable from `branch`'s tip, i.e. the branch CONTAINS it:
|
|
943
|
+
* `git rev-list --count --max-count=1 <sha> --not refs/heads/<branch>` is 0 when everything
|
|
944
|
+
* reachable from `sha` is reachable from the branch too (the tip itself counts as contained).
|
|
945
|
+
*
|
|
946
|
+
* Phrased as a rev-list rather than `merge-base --is-ancestor` on purpose: the latter answers "no"
|
|
947
|
+
* by EXITING 1, which is indistinguishable here from a broken checkout, and this probe's whole job
|
|
948
|
+
* is to be trusted only when it is a definite answer. Tri-state for the same reason (as
|
|
949
|
+
* {@link branchAheadOfBase} is):
|
|
950
|
+
*
|
|
951
|
+
* - `true`: confirmed contained.
|
|
952
|
+
* - `false`: confirmed dropped, so the branch was rewritten below `sha`.
|
|
953
|
+
* - `undefined`: could not determine (an unknown object, a rev-list error). A caller must not read
|
|
954
|
+
* a failed probe as either answer.
|
|
955
|
+
*
|
|
956
|
+
* The work-branch lease is gated on this: see {@link workBranchLease}.
|
|
957
|
+
*/
|
|
958
|
+
export async function branchContainsCommit(dir, branch, sha, signal) {
|
|
959
|
+
try {
|
|
960
|
+
const out = await git(['rev-list', '--count', '--max-count=1', sha, '--not', `refs/heads/${branch}`], { cwd: dir, signal });
|
|
961
|
+
const count = Number(out.trim());
|
|
962
|
+
return Number.isNaN(count) ? undefined : count === 0;
|
|
963
|
+
}
|
|
964
|
+
catch {
|
|
965
|
+
return undefined;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* The work branch's tip when it holds something UNPUBLISHED, else undefined: the answer to whether a
|
|
970
|
+
* checkpoint tick has anything to do. Two ways of having nothing:
|
|
971
|
+
*
|
|
972
|
+
* - the tip is still `baseSha`, so this pass has committed nothing. Pushing here would create the
|
|
973
|
+
* work branch at the base commit, and a later retry would see that zero-diff branch via
|
|
974
|
+
* `remoteBranchExists`, resume it as work, and fail to open a PR ("no commits between base and
|
|
975
|
+
* head"). A pass that never commits must leave NO branch behind.
|
|
976
|
+
* - the tip is `publishedSha`, so the last push already published it. Without this the checkpoint
|
|
977
|
+
* re-pushed an unchanged branch on every tick: an hour-long run committing eight times issued
|
|
978
|
+
* ~60 pushes, ~52 of them a full authenticated round trip answering "Everything up-to-date",
|
|
979
|
+
* each one counting against the host's push rate limits.
|
|
980
|
+
*
|
|
981
|
+
* That second condition is also what keeps the INTERVAL the right knob. It expresses the acceptable
|
|
982
|
+
* loss window when a container dies (a property of the deployment's infra churn), not a rate: gated
|
|
983
|
+
* this way, the tick publishes at most one push per commit the agent makes, whatever the model or
|
|
984
|
+
* the run's length, so nothing here needs to be tuned per model.
|
|
985
|
+
*/
|
|
986
|
+
export async function unpublishedWorkBranchTip(args) {
|
|
987
|
+
const head = await headCommit(args.dir, args.signal);
|
|
988
|
+
if (head === args.baseSha || head === args.publishedSha)
|
|
989
|
+
return undefined;
|
|
990
|
+
return head;
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* The lease a work-branch push is entitled to (the `opts` {@link pushBranch} takes): the sha this
|
|
994
|
+
* pass last published, and nothing at all before it has published one.
|
|
995
|
+
*
|
|
996
|
+
* The extra condition is what bounds the force to THIS pass's own commits, which the lease alone
|
|
997
|
+
* does not do and the design promises. Once one checkpoint has landed, a rewrite that drops
|
|
998
|
+
* `baseSha` (the tip the pass started from, which on a RESUMED branch is an earlier run's published
|
|
999
|
+
* work) would still lease successfully against our own checkpoint and carry those earlier commits
|
|
1000
|
+
* away with it. So the lease is withheld unless the branch still CONTAINS `baseSha`: the push then
|
|
1001
|
+
* goes out plain, git refuses it as a non-fast-forward, and the engine re-dispatches onto the
|
|
1002
|
+
* branch as it stands.
|
|
1003
|
+
*
|
|
1004
|
+
* A probe that could not answer withholds it too (`onWithheld('unreadable')`), because the two
|
|
1005
|
+
* mistakes are not symmetric: withholding costs a refused rewrite and one re-dispatch, trusting an
|
|
1006
|
+
* unreadable probe costs commits.
|
|
1007
|
+
*/
|
|
1008
|
+
export async function workBranchLease(args) {
|
|
1009
|
+
if (!args.publishedSha)
|
|
1010
|
+
return {};
|
|
1011
|
+
const contains = await branchContainsCommit(args.dir, args.branch, args.baseSha, args.signal);
|
|
1012
|
+
if (contains === true)
|
|
1013
|
+
return { expectRemoteSha: args.publishedSha };
|
|
1014
|
+
args.onWithheld?.(contains === undefined ? 'unreadable' : 'dropped');
|
|
1015
|
+
return {};
|
|
863
1016
|
}
|
|
864
1017
|
/**
|
|
865
1018
|
* Reset the working tree's git history to a single bootstrap commit and push it
|
package/dist/inline.js
CHANGED
|
@@ -13,14 +13,27 @@ import { runSubscriptionHarness } from './agent-runner.js';
|
|
|
13
13
|
// `auth.json`), so the container and coding paths can never disagree on how a credential
|
|
14
14
|
// is injected.
|
|
15
15
|
/**
|
|
16
|
-
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the inline
|
|
17
|
+
* `finishReason` the reviewer keys off, or `undefined` when the CLI reported NONE.
|
|
18
|
+
*
|
|
19
|
+
* Undefined rather than `stop`, which is what this returned for years on the strength of a
|
|
20
|
+
* comment claiming Claude Code reports the reason. It does not: its `stream-json` `assistant`
|
|
21
|
+
* envelopes carry the message-START snapshot, whose `stop_reason` is null, so every call metric
|
|
22
|
+
* a claude-code or codex run produces has a null reason and this answered `stop` for all of them.
|
|
23
|
+
* `stop` is a positive claim that the model finished of its own accord, and it is the exact claim
|
|
24
|
+
* a truncation check is trying to disprove — so the one caller keyed off it
|
|
25
|
+
* (`finishReason === 'length'`) could never fire, and every store that kept the row recorded a
|
|
26
|
+
* clean stop nobody observed.
|
|
27
|
+
*
|
|
28
|
+
* Kept as a MAPPING rather than deleted because the field remains reachable: a subagent turn is
|
|
29
|
+
* read from a completed JSONL transcript, which does carry `stop_reason`, and a future CLI build
|
|
30
|
+
* (or `--include-partial-messages`) would restore it on the parent stream too.
|
|
20
31
|
*/
|
|
21
32
|
function deriveFinishReason(calls) {
|
|
22
33
|
const last = calls?.[calls.length - 1];
|
|
23
|
-
const reason = last?.finishReason?.toLowerCase()
|
|
34
|
+
const reason = last?.finishReason?.toLowerCase();
|
|
35
|
+
if (!reason)
|
|
36
|
+
return undefined;
|
|
24
37
|
return reason === 'max_tokens' || reason === 'length' ? 'length' : 'stop';
|
|
25
38
|
}
|
|
26
39
|
/**
|
|
@@ -51,9 +64,10 @@ export async function handleInline(job, opts) {
|
|
|
51
64
|
...(opts.signal ? { signal: opts.signal } : {}),
|
|
52
65
|
...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
|
|
53
66
|
});
|
|
67
|
+
const finishReason = deriveFinishReason(outcome.callMetrics);
|
|
54
68
|
return {
|
|
55
69
|
text: outcome.summary,
|
|
56
|
-
finishReason:
|
|
70
|
+
...(finishReason ? { finishReason } : {}),
|
|
57
71
|
...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
|
|
58
72
|
...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
|
|
59
73
|
};
|
package/dist/job.d.ts
CHANGED
|
@@ -673,7 +673,14 @@ export interface InlineJob extends HarnessAuthFields {
|
|
|
673
673
|
/** The inline completion result: the reply text plus lifted token usage / per-call telemetry. */
|
|
674
674
|
export interface InlineResult {
|
|
675
675
|
text: string;
|
|
676
|
-
/**
|
|
676
|
+
/**
|
|
677
|
+
* `length` when the model hit its output cap (the reviewer rejects a truncated doc), `stop`
|
|
678
|
+
* when it finished of its own accord, ABSENT when the CLI reported no stop reason at all.
|
|
679
|
+
*
|
|
680
|
+
* Absent is the normal case today: neither subscription CLI exposes a per-call stop reason on
|
|
681
|
+
* its parent stream, and the three states must stay distinct because a reader that takes
|
|
682
|
+
* absent for `stop` is asserting the one thing a truncation check exists to disprove.
|
|
683
|
+
*/
|
|
677
684
|
finishReason?: 'stop' | 'length';
|
|
678
685
|
/**
|
|
679
686
|
* The job's token usage with the input side split into its three ORTHOGONAL classes:
|
package/dist/pi.d.ts
CHANGED
|
@@ -304,6 +304,16 @@ export interface HarnessCallMetric {
|
|
|
304
304
|
* never disagree about which phase billed a call.
|
|
305
305
|
*/
|
|
306
306
|
phase?: string;
|
|
307
|
+
/**
|
|
308
|
+
* This row is not a TURN: it stands for the job as a whole, carrying the spend the CLI reported
|
|
309
|
+
* in its terminal cumulative total and did not attribute to any turn it narrated (see
|
|
310
|
+
* {@link unaccountedUsageCall}). It has no bodies, because there was no request to capture.
|
|
311
|
+
*
|
|
312
|
+
* The backend files it with a NULL turn index for that reason, while still deriving its row id
|
|
313
|
+
* from {@link seq} so a replayed poll re-records instead of duplicating. Absent on every real
|
|
314
|
+
* turn. `CliInlineLanguageModel`'s step-level row is the same idea on the inline path.
|
|
315
|
+
*/
|
|
316
|
+
standsForJob?: boolean;
|
|
307
317
|
}
|
|
308
318
|
/**
|
|
309
319
|
* Publish one captured model call: append it to the run's list (which becomes the terminal
|
|
@@ -318,32 +328,15 @@ export interface HarnessCallMetric {
|
|
|
318
328
|
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
319
329
|
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
320
330
|
* the chain tip it was written against), which means a field mutated after publishing never
|
|
321
|
-
* reaches the store.
|
|
322
|
-
* whose totals arrive with the CLI's terminal `result` event) publishes through
|
|
323
|
-
* {@link createCallMetricPublisher} instead, which withholds exactly those.
|
|
324
|
-
*/
|
|
325
|
-
export declare function publishCallMetric(calls: HarnessCallMetric[], call: HarnessCallMetric, onCallMetric?: (call: HarnessCallMetric) => void): void;
|
|
326
|
-
/** Appends captured calls to a run's list, streaming each one as soon as it is final. */
|
|
327
|
-
export interface CallMetricPublisher {
|
|
328
|
-
/** Append a captured call, streaming it now unless its tokens can still be rewritten. */
|
|
329
|
-
publish(call: HarnessCallMetric): void;
|
|
330
|
-
/** Stream whatever is still withheld. Call once the run's totals are attributed. */
|
|
331
|
-
flush(): void;
|
|
332
|
-
}
|
|
333
|
-
/**
|
|
334
|
-
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
335
|
-
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
336
|
-
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
337
|
-
* arrives.
|
|
331
|
+
* reaches the store.
|
|
338
332
|
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
333
|
+
* That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
|
|
334
|
+
* shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
|
|
335
|
+
* last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
|
|
336
|
+
* could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
|
|
337
|
+
* the wrapper had no reason left to exist.
|
|
345
338
|
*/
|
|
346
|
-
export declare function
|
|
339
|
+
export declare function publishCallMetric(calls: HarnessCallMetric[], call: HarnessCallMetric, onCallMetric?: (call: HarnessCallMetric) => void): void;
|
|
347
340
|
/** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
|
|
348
341
|
export interface PiRunOutcome {
|
|
349
342
|
summary: string;
|
package/dist/pi.js
CHANGED
|
@@ -436,52 +436,18 @@ export async function writeWebToolsConfig(config) {
|
|
|
436
436
|
* A published call must be FINAL. The backend records it the moment the drain reaches it and
|
|
437
437
|
* IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
|
|
438
438
|
* the chain tip it was written against), which means a field mutated after publishing never
|
|
439
|
-
* reaches the store.
|
|
440
|
-
*
|
|
441
|
-
*
|
|
439
|
+
* reaches the store.
|
|
440
|
+
*
|
|
441
|
+
* That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
|
|
442
|
+
* shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
|
|
443
|
+
* last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
|
|
444
|
+
* could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
|
|
445
|
+
* the wrapper had no reason left to exist.
|
|
442
446
|
*/
|
|
443
447
|
export function publishCallMetric(calls, call, onCallMetric) {
|
|
444
448
|
calls.push(call);
|
|
445
449
|
onCallMetric?.(call);
|
|
446
450
|
}
|
|
447
|
-
/**
|
|
448
|
-
* A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
|
|
449
|
-
* the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
|
|
450
|
-
* `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
|
|
451
|
-
* arrives.
|
|
452
|
-
*
|
|
453
|
-
* Since a published call must be final (the backend stores it on the drain and ignores the
|
|
454
|
-
* terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
|
|
455
|
-
* live stream — otherwise it records as a zero-token row and the attributed numbers never land.
|
|
456
|
-
* The withholding window closes the moment any call IS costed: attribution can no longer fire, so
|
|
457
|
-
* everything held is final and released at once, in capture order, and every later call streams
|
|
458
|
-
* immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
|
|
459
|
-
*/
|
|
460
|
-
export function createCallMetricPublisher(calls, onCallMetric) {
|
|
461
|
-
const withheld = [];
|
|
462
|
-
let anyCosted = false;
|
|
463
|
-
const flush = () => {
|
|
464
|
-
for (const call of withheld)
|
|
465
|
-
onCallMetric?.(call);
|
|
466
|
-
withheld.length = 0;
|
|
467
|
-
};
|
|
468
|
-
return {
|
|
469
|
-
publish(call) {
|
|
470
|
-
const costed = call.inputTokens > 0 || call.outputTokens > 0;
|
|
471
|
-
if (!costed && !anyCosted) {
|
|
472
|
-
publishCallMetric(calls, call);
|
|
473
|
-
withheld.push(call);
|
|
474
|
-
return;
|
|
475
|
-
}
|
|
476
|
-
if (costed)
|
|
477
|
-
anyCosted = true;
|
|
478
|
-
// Released BEFORE this call so the live sequence stays in capture order.
|
|
479
|
-
flush();
|
|
480
|
-
publishCallMetric(calls, call, onCallMetric);
|
|
481
|
-
},
|
|
482
|
-
flush,
|
|
483
|
-
};
|
|
484
|
-
}
|
|
485
451
|
/**
|
|
486
452
|
* Pull the `todo` tool's result `details` out of a Pi `--mode json` event, or
|
|
487
453
|
* undefined if the event isn't a successful `todo` tool result.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { HarnessCallMetric } from './pi.js';
|
|
2
|
+
/**
|
|
3
|
+
* Read Claude Code's terminal cumulative usage.
|
|
4
|
+
*
|
|
5
|
+
* Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
|
|
6
|
+
* (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
|
|
7
|
+
* long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
|
|
8
|
+
* window. `undefined` when the event carried no usage at all, so a caller can tell that from a
|
|
9
|
+
* genuine zero.
|
|
10
|
+
*/
|
|
11
|
+
export declare function claudeUsage(raw: unknown): {
|
|
12
|
+
inputTokens: number;
|
|
13
|
+
outputTokens: number;
|
|
14
|
+
} | undefined;
|
|
15
|
+
/**
|
|
16
|
+
* The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
|
|
17
|
+
* usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
|
|
18
|
+
* add up, so nothing is double counted.
|
|
19
|
+
*
|
|
20
|
+
* The per-side part is why this exists at all, and it replaced an all-or-nothing guard
|
|
21
|
+
* (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
|
|
22
|
+
* CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
|
|
23
|
+
* carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
|
|
24
|
+
* `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
|
|
25
|
+
* turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
|
|
26
|
+
* board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
|
|
27
|
+
* `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
|
|
28
|
+
* figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
|
|
29
|
+
* tokens had been reported.
|
|
30
|
+
*
|
|
31
|
+
* **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
|
|
32
|
+
* thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
|
|
33
|
+
* measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
|
|
34
|
+
* step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
|
|
35
|
+
* inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
|
|
36
|
+
* step-level row; this is the same answer for the channel that has a call list. {@link
|
|
37
|
+
* HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
|
|
38
|
+
*
|
|
39
|
+
* **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
|
|
40
|
+
* parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
|
|
41
|
+
* subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
|
|
42
|
+
* would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
|
|
43
|
+
* CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
|
|
44
|
+
* turns are captured through the same publisher as the parent's.
|
|
45
|
+
*
|
|
46
|
+
* {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
|
|
47
|
+
* all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
|
|
48
|
+
* lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
|
|
49
|
+
*
|
|
50
|
+
* Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
|
|
51
|
+
* reported the two inconsistently, and negative spend is not a thing to record.
|
|
52
|
+
*/
|
|
53
|
+
export declare function unaccountedUsageCall(parentCalls: readonly HarnessCallMetric[], usage: {
|
|
54
|
+
inputTokens: number;
|
|
55
|
+
outputTokens: number;
|
|
56
|
+
} | undefined): HarnessCallMetric | undefined;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { isObject, numberOf } from './claude-stream.js';
|
|
2
|
+
// How a subscription CLI's TWO token channels are reconciled into the per-call rows the backend
|
|
3
|
+
// stores: the per-turn usage the stream narrates, and the cumulative total the terminal `result`
|
|
4
|
+
// event reports. They disagree routinely and in a specific direction, so the reconciliation is a
|
|
5
|
+
// concern of its own rather than a helper beside the stream reader that happens to need it.
|
|
6
|
+
//
|
|
7
|
+
// Split out of `agent-runner.ts` when it hit its size budget.
|
|
8
|
+
/**
|
|
9
|
+
* Read Claude Code's terminal cumulative usage.
|
|
10
|
+
*
|
|
11
|
+
* Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
|
|
12
|
+
* (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
|
|
13
|
+
* long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
|
|
14
|
+
* window. `undefined` when the event carried no usage at all, so a caller can tell that from a
|
|
15
|
+
* genuine zero.
|
|
16
|
+
*/
|
|
17
|
+
export function claudeUsage(raw) {
|
|
18
|
+
if (!isObject(raw))
|
|
19
|
+
return undefined;
|
|
20
|
+
const input = numberOf(raw.input_tokens) +
|
|
21
|
+
numberOf(raw.cache_read_input_tokens) +
|
|
22
|
+
numberOf(raw.cache_creation_input_tokens);
|
|
23
|
+
const output = numberOf(raw.output_tokens);
|
|
24
|
+
if (input === 0 && output === 0)
|
|
25
|
+
return undefined;
|
|
26
|
+
return { inputTokens: input, outputTokens: output };
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
|
|
30
|
+
* usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
|
|
31
|
+
* add up, so nothing is double counted.
|
|
32
|
+
*
|
|
33
|
+
* The per-side part is why this exists at all, and it replaced an all-or-nothing guard
|
|
34
|
+
* (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
|
|
35
|
+
* CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
|
|
36
|
+
* carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
|
|
37
|
+
* `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
|
|
38
|
+
* turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
|
|
39
|
+
* board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
|
|
40
|
+
* `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
|
|
41
|
+
* figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
|
|
42
|
+
* tokens had been reported.
|
|
43
|
+
*
|
|
44
|
+
* **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
|
|
45
|
+
* thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
|
|
46
|
+
* measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
|
|
47
|
+
* step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
|
|
48
|
+
* inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
|
|
49
|
+
* step-level row; this is the same answer for the channel that has a call list. {@link
|
|
50
|
+
* HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
|
|
51
|
+
*
|
|
52
|
+
* **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
|
|
53
|
+
* parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
|
|
54
|
+
* subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
|
|
55
|
+
* would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
|
|
56
|
+
* CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
|
|
57
|
+
* turns are captured through the same publisher as the parent's.
|
|
58
|
+
*
|
|
59
|
+
* {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
|
|
60
|
+
* all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
|
|
61
|
+
* lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
|
|
62
|
+
*
|
|
63
|
+
* Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
|
|
64
|
+
* reported the two inconsistently, and negative spend is not a thing to record.
|
|
65
|
+
*/
|
|
66
|
+
export function unaccountedUsageCall(parentCalls, usage) {
|
|
67
|
+
if (!usage)
|
|
68
|
+
return undefined;
|
|
69
|
+
let accountedInput = 0;
|
|
70
|
+
let accountedOutput = 0;
|
|
71
|
+
for (const call of parentCalls) {
|
|
72
|
+
accountedInput += call.inputTokens + call.cacheReadTokens + call.cacheWriteTokens;
|
|
73
|
+
accountedOutput += call.outputTokens;
|
|
74
|
+
}
|
|
75
|
+
const inputTokens = Math.max(0, usage.inputTokens - accountedInput);
|
|
76
|
+
const outputTokens = Math.max(0, usage.outputTokens - accountedOutput);
|
|
77
|
+
if (!inputTokens && !outputTokens)
|
|
78
|
+
return undefined;
|
|
79
|
+
return {
|
|
80
|
+
// No `model`: the terminal event names none, and the recorder then files the row under the
|
|
81
|
+
// model the step DISPATCHED, which is the same answer without this claiming to have observed
|
|
82
|
+
// it. (Claude Code serves some turns with a different model, so a guess here misprices.)
|
|
83
|
+
promptText: '',
|
|
84
|
+
messageCount: 0,
|
|
85
|
+
responseText: '',
|
|
86
|
+
reasoningText: '',
|
|
87
|
+
inputTokens,
|
|
88
|
+
// Both 0 rather than a split of `inputTokens`: the terminal figure is one number and says
|
|
89
|
+
// nothing about which input class the remainder belonged to.
|
|
90
|
+
cacheReadTokens: 0,
|
|
91
|
+
cacheWriteTokens: 0,
|
|
92
|
+
outputTokens,
|
|
93
|
+
finishReason: null,
|
|
94
|
+
standsForJob: true,
|
|
95
|
+
};
|
|
96
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cat-factory/executor-harness",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.118.0",
|
|
4
4
|
"description": "Container payload: a thin TypeScript wrapper that runs the Pi coding agent against a cloned repo and opens a PR. Runs in the Cloudflare Container (and, in local native mode, as a host process); carries no secrets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"hono": "^4.13.1",
|
|
31
31
|
"typescript": "7.0.2",
|
|
32
32
|
"vitest": "^4.1.10",
|
|
33
|
-
"@cat-factory/kernel": "0.
|
|
34
|
-
"@cat-factory/server": "0.
|
|
35
|
-
"@cat-factory/spend": "0.15.
|
|
33
|
+
"@cat-factory/kernel": "0.298.1",
|
|
34
|
+
"@cat-factory/server": "0.284.1",
|
|
35
|
+
"@cat-factory/spend": "0.15.90"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.json",
|