@cat-factory/executor-harness 1.114.0 → 1.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -1
- package/dist/agent-runner.js +68 -74
- package/dist/claude-call-aggregator.d.ts +9 -0
- package/dist/claude-call-aggregator.js +10 -1
- package/dist/coding-agent.js +50 -9
- package/dist/failure.d.ts +7 -1
- package/dist/failure.js +7 -0
- package/dist/git.d.ts +118 -3
- package/dist/git.js +162 -9
- package/dist/inline.js +20 -6
- package/dist/job.d.ts +8 -1
- package/dist/pi.d.ts +17 -24
- package/dist/pi.js +7 -41
- package/dist/usage-attribution.d.ts +56 -0
- package/dist/usage-attribution.js +96 -0
- package/package.json +4 -4
- package/src/agent-runner.ts +89 -80
- package/src/claude-call-aggregator.ts +15 -2
- package/src/coding-agent.ts +52 -8
- package/src/failure.ts +7 -0
- package/src/git.ts +212 -9
- package/src/inline.ts +20 -7
- package/src/job.ts +8 -1
- package/src/pi.ts +17 -51
- package/src/usage-attribution.ts +102 -0
package/src/agent-runner.ts
CHANGED
|
@@ -3,7 +3,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
|
3
3
|
import { tmpdir } from 'node:os'
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js'
|
|
6
|
-
import {
|
|
6
|
+
import { claudeUsage, unaccountedUsageCall } from './usage-attribution.js'
|
|
7
|
+
import {
|
|
8
|
+
createClaudeRunTelemetry,
|
|
9
|
+
subagentDispatchId,
|
|
10
|
+
type ClaudeRunTelemetry,
|
|
11
|
+
} from './claude-call-aggregator.js'
|
|
7
12
|
import {
|
|
8
13
|
ToolCallTracker,
|
|
9
14
|
type TrackedToolCall,
|
|
@@ -12,9 +17,7 @@ import {
|
|
|
12
17
|
import { log, type Logger } from './logger.js'
|
|
13
18
|
import { NO_TOOL_WINDOW, type ToolProgressWindow } from './tool-silence.js'
|
|
14
19
|
import {
|
|
15
|
-
createCallMetricPublisher,
|
|
16
20
|
publishCallMetric,
|
|
17
|
-
type CallMetricPublisher,
|
|
18
21
|
type HarnessCallMetric,
|
|
19
22
|
type PiRunOutcome,
|
|
20
23
|
type TodoProgress,
|
|
@@ -202,24 +205,6 @@ export interface SubscriptionRunOptions {
|
|
|
202
205
|
log?: Logger
|
|
203
206
|
}
|
|
204
207
|
|
|
205
|
-
/**
|
|
206
|
-
* Fallback token attribution: if a CLI reported a cumulative total but no per-turn
|
|
207
|
-
* usage (so every captured call has zero tokens), pin the whole total onto the LAST
|
|
208
|
-
* call rather than dropping it — the run's tokens are still accounted, just not split
|
|
209
|
-
* per turn. A no-op when the calls already carry per-turn tokens.
|
|
210
|
-
*/
|
|
211
|
-
function attributeCumulativeUsage(
|
|
212
|
-
calls: HarnessCallMetric[],
|
|
213
|
-
usage: { inputTokens: number; outputTokens: number } | undefined,
|
|
214
|
-
): void {
|
|
215
|
-
if (!usage || calls.length === 0) return
|
|
216
|
-
const anyTokens = calls.some((c) => c.inputTokens > 0 || c.outputTokens > 0)
|
|
217
|
-
if (anyTokens) return
|
|
218
|
-
const last = calls[calls.length - 1]!
|
|
219
|
-
last.inputTokens = usage.inputTokens
|
|
220
|
-
last.outputTokens = usage.outputTokens
|
|
221
|
-
}
|
|
222
|
-
|
|
223
208
|
/**
|
|
224
209
|
* Drive one CLI subprocess to completion, streaming LF-framed JSONL from stdout
|
|
225
210
|
* through `onEvent`. Mirrors `runPi`'s lifecycle: prompt over stdin (out-of-band,
|
|
@@ -711,6 +696,75 @@ function carriesToolResult(content: unknown[]): boolean {
|
|
|
711
696
|
return content.some((block) => isObject(block) && block.type === 'tool_result')
|
|
712
697
|
}
|
|
713
698
|
|
|
699
|
+
/** One claude-code run's per-call telemetry: what was captured, and how it is settled. */
|
|
700
|
+
interface ClaudeCallCapture {
|
|
701
|
+
/** Every captured call, terminal-result order — the parent's, the subagents', the remainder. */
|
|
702
|
+
calls: HarnessCallMetric[]
|
|
703
|
+
telemetry: ClaudeRunTelemetry
|
|
704
|
+
/**
|
|
705
|
+
* File whatever the parent's narrated turns did not account for, once its terminal cumulative
|
|
706
|
+
* usage is known. A no-op when they add up. See {@link unaccountedUsageCall}.
|
|
707
|
+
*/
|
|
708
|
+
settleUsage: (usage: { inputTokens: number; outputTokens: number } | undefined) => void
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/**
|
|
712
|
+
* Open the per-call telemetry capture for one claude-code run.
|
|
713
|
+
*
|
|
714
|
+
* It reconstructs the full per-call request/response bodies from the stream.
|
|
715
|
+
* `--output-format stream-json --verbose` emits a near-verbatim Anthropic Messages envelope per
|
|
716
|
+
* response CONTENT BLOCK (not per call), so the aggregator folds the envelopes sharing a
|
|
717
|
+
* `message.id` back into one call and buffers that call's `user` tool_result turns — together the
|
|
718
|
+
* growing prompt transcript, in the shape the model was actually sent. It is SEEDED with the inputs
|
|
719
|
+
* the harness supplies (they never appear in the stream): the system + first user message when the
|
|
720
|
+
* prompt rides argv, or a single folded user turn when it doesn't, so the reconstruction never shows
|
|
721
|
+
* a system turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
722
|
+
*
|
|
723
|
+
* The parent loop's calls are tracked SEPARATELY, by reference into the same list, because the
|
|
724
|
+
* terminal `result` event's cumulative usage covers only the parent conversation. In `ambientAuth`
|
|
725
|
+
* mode there is no transcript watcher, so the CLI's tagged subagent turns are captured here too and
|
|
726
|
+
* `calls` holds both; reconciling against that mixed list is what once billed a subagent for the
|
|
727
|
+
* parent's whole output shortfall.
|
|
728
|
+
*/
|
|
729
|
+
function openClaudeCallCapture(
|
|
730
|
+
opts: SubscriptionRunOptions,
|
|
731
|
+
stream: { prompt: string; folded: boolean; secrets: string[] },
|
|
732
|
+
): ClaudeCallCapture {
|
|
733
|
+
const calls: HarnessCallMetric[] = []
|
|
734
|
+
const parentCalls: HarnessCallMetric[] = []
|
|
735
|
+
const publish = (metric: HarnessCallMetric): void =>
|
|
736
|
+
publishCallMetric(calls, metric, opts.onCallMetric)
|
|
737
|
+
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring in the caller: it is started
|
|
738
|
+
// only when the CLI has an isolated config home to watch, which an `ambientAuth` run does not
|
|
739
|
+
// have. The telemetry routes the CLI's tagged subagent turns accordingly — see
|
|
740
|
+
// `createClaudeRunTelemetry`.
|
|
741
|
+
const telemetry = createClaudeRunTelemetry({
|
|
742
|
+
seed: stream.folded
|
|
743
|
+
? [{ role: 'user', content: stream.prompt }]
|
|
744
|
+
: [
|
|
745
|
+
{ role: 'system', content: opts.systemPrompt },
|
|
746
|
+
{ role: 'user', content: opts.userPrompt },
|
|
747
|
+
],
|
|
748
|
+
secrets: stream.secrets,
|
|
749
|
+
watcherOwnsSubagents: !opts.ambientAuth,
|
|
750
|
+
publish: (metric) => {
|
|
751
|
+
parentCalls.push(metric)
|
|
752
|
+
publish(metric)
|
|
753
|
+
},
|
|
754
|
+
publishSubagent: publish,
|
|
755
|
+
})
|
|
756
|
+
return {
|
|
757
|
+
calls,
|
|
758
|
+
telemetry,
|
|
759
|
+
settleUsage: (usage) => {
|
|
760
|
+
// Published like any other call so the live drain records it too, which is also what stamps
|
|
761
|
+
// its `seq` and therefore its stable row id.
|
|
762
|
+
const remainder = unaccountedUsageCall(parentCalls, usage)
|
|
763
|
+
if (remainder) publish(remainder)
|
|
764
|
+
},
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
714
768
|
export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
|
|
715
769
|
const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
|
|
716
770
|
let summary = ''
|
|
@@ -729,34 +783,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
729
783
|
})
|
|
730
784
|
}
|
|
731
785
|
|
|
732
|
-
// Reconstruct the full per-call request/response bodies for telemetry from the
|
|
733
|
-
// stream. `--output-format stream-json --verbose` emits a near-verbatim Anthropic
|
|
734
|
-
// Messages envelope per response CONTENT BLOCK (not per call), so the aggregator below
|
|
735
|
-
// folds the envelopes sharing a `message.id` back into one call and buffers that call's
|
|
736
|
-
// `user` tool_result turns — together the growing prompt transcript, in the shape the
|
|
737
|
-
// model was actually sent. We seed it with the inputs the harness supplies (they never
|
|
738
|
-
// appear in the stream): the system + first user message when the prompt rides argv, or
|
|
739
|
-
// a single folded user turn when it doesn't — so the reconstruction never shows a system
|
|
740
|
-
// turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
|
|
741
786
|
const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : []
|
|
742
|
-
const
|
|
743
|
-
|
|
744
|
-
// may still rewrite below (a published call must be final — see the publisher).
|
|
745
|
-
const publisher = createCallMetricPublisher(calls, opts.onCallMetric)
|
|
746
|
-
// `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring below: it is started only when
|
|
747
|
-
// the CLI has an isolated config home to watch, which an `ambientAuth` run does not have. The
|
|
748
|
-
// telemetry routes the CLI's tagged subagent turns accordingly — see `createClaudeRunTelemetry`.
|
|
749
|
-
const telemetry = createClaudeRunTelemetry({
|
|
750
|
-
seed: folded
|
|
751
|
-
? [{ role: 'user', content: prompt }]
|
|
752
|
-
: [
|
|
753
|
-
{ role: 'system', content: opts.systemPrompt },
|
|
754
|
-
{ role: 'user', content: opts.userPrompt },
|
|
755
|
-
],
|
|
756
|
-
secrets,
|
|
757
|
-
watcherOwnsSubagents: !opts.ambientAuth,
|
|
758
|
-
publish: (metric) => publisher.publish(metric),
|
|
759
|
-
})
|
|
787
|
+
const capture = openClaudeCallCapture(opts, { prompt, folded, secrets })
|
|
788
|
+
const telemetry = capture.telemetry
|
|
760
789
|
|
|
761
790
|
// ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from the two views the run
|
|
762
791
|
// produces of the SAME slicing. The parent's subagent dispatches + their terminal tool_results
|
|
@@ -917,8 +946,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
917
946
|
summary,
|
|
918
947
|
stats,
|
|
919
948
|
stderrTail,
|
|
920
|
-
|
|
921
|
-
publisher,
|
|
949
|
+
capture,
|
|
922
950
|
usage,
|
|
923
951
|
subagents,
|
|
924
952
|
expectSubagentCalls: telemetry.expectsWatcherCalls(),
|
|
@@ -926,11 +954,11 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
|
|
|
926
954
|
})
|
|
927
955
|
} catch (err) {
|
|
928
956
|
// The stream ended abnormally (guard trip, watchdog kill, CLI crash). Complete the call in
|
|
929
|
-
// flight anyway
|
|
930
|
-
//
|
|
931
|
-
//
|
|
957
|
+
// flight anyway: a killed run never returns an outcome, so the live channel is the ONLY record
|
|
958
|
+
// of what it spent, and dropping its last turn is what the streaming exists to avoid. No
|
|
959
|
+
// terminal `result` event arrived, so there is no cumulative total to reconcile against and no
|
|
960
|
+
// remainder row to file — every captured turn already streamed as it was completed.
|
|
932
961
|
telemetry.flush()
|
|
933
|
-
publisher.flush()
|
|
934
962
|
// A tripped no-progress guard aborted the CLI; streamCli rejects with its generic abort
|
|
935
963
|
// message, so replace it with the guard's actionable diagnostic — carrying the stderr tail it
|
|
936
964
|
// attached, since that is usually the only evidence of what the CLI was doing when it was
|
|
@@ -1088,9 +1116,8 @@ async function assembleClaudeOutcome(args: {
|
|
|
1088
1116
|
summary: string
|
|
1089
1117
|
stats: PiRunStats
|
|
1090
1118
|
stderrTail: string
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
publisher: CallMetricPublisher
|
|
1119
|
+
/** This run's per-call telemetry, settled here with the terminal usage. */
|
|
1120
|
+
capture: ClaudeCallCapture
|
|
1094
1121
|
usage: { inputTokens: number; outputTokens: number } | undefined
|
|
1095
1122
|
subagents: ReturnType<typeof startSubagentWatcher> | undefined
|
|
1096
1123
|
/**
|
|
@@ -1102,14 +1129,11 @@ async function assembleClaudeOutcome(args: {
|
|
|
1102
1129
|
expectSubagentCalls: boolean
|
|
1103
1130
|
log?: Logger
|
|
1104
1131
|
}): Promise<PiRunOutcome> {
|
|
1105
|
-
const { summary, stats, stderrTail,
|
|
1106
|
-
|
|
1107
|
-
//
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
// alongside the result, and the backend records the attributed numbers rather than the zeros
|
|
1111
|
-
// they carried while the run was in flight.
|
|
1112
|
-
publisher.flush()
|
|
1132
|
+
const { summary, stats, stderrTail, capture, usage, subagents } = args
|
|
1133
|
+
const calls = capture.calls
|
|
1134
|
+
// What the parent's narrated turns did not account for, as its OWN row (never tokens grafted onto
|
|
1135
|
+
// a real turn).
|
|
1136
|
+
capture.settleUsage(usage)
|
|
1113
1137
|
// Final drain of any subagent transcript writes that landed after the last poll, then
|
|
1114
1138
|
// fold the subagents' usage + per-call telemetry into the run's outcome.
|
|
1115
1139
|
await subagents?.stop()
|
|
@@ -1138,21 +1162,6 @@ async function assembleClaudeOutcome(args: {
|
|
|
1138
1162
|
}
|
|
1139
1163
|
}
|
|
1140
1164
|
|
|
1141
|
-
function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number } | undefined {
|
|
1142
|
-
if (!isObject(raw)) return undefined
|
|
1143
|
-
// Count every input bucket Anthropic bills: fresh input plus BOTH cache reads and
|
|
1144
|
-
// cache writes (cache_creation_input_tokens), which are real consumed tokens — and
|
|
1145
|
-
// are the dominant share on a long agent run. Omitting them under-weights a token's
|
|
1146
|
-
// true load in the usage-aware rotation window.
|
|
1147
|
-
const input =
|
|
1148
|
-
numberOf(raw.input_tokens) +
|
|
1149
|
-
numberOf(raw.cache_read_input_tokens) +
|
|
1150
|
-
numberOf(raw.cache_creation_input_tokens)
|
|
1151
|
-
const output = numberOf(raw.output_tokens)
|
|
1152
|
-
if (input === 0 && output === 0) return undefined
|
|
1153
|
-
return { inputTokens: input, outputTokens: output }
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
1165
|
// ---------------------------------------------------------------------------
|
|
1157
1166
|
// Codex
|
|
1158
1167
|
// ---------------------------------------------------------------------------
|
|
@@ -452,12 +452,25 @@ export interface ClaudeRunTelemetry {
|
|
|
452
452
|
* `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
|
|
453
453
|
* instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
|
|
454
454
|
* billed by neither channel, and an under-count reads as a cheap run rather than as an error.
|
|
455
|
+
*
|
|
456
|
+
* Which is also why the two are published through SEPARATE callbacks. A caller reconciling the
|
|
457
|
+
* parent's terminal cumulative usage needs the parent's calls alone, and with one shared callback
|
|
458
|
+
* the fallback channel silently mixed subagent turns into that list — where they both understated
|
|
459
|
+
* the shortfall and, being last, attracted it (`unaccountedUsageCall`). `publishSubagent` is
|
|
460
|
+
* optional so a caller that draws no distinction (a test, the settled-transcript path where nothing
|
|
461
|
+
* arrives here anyway) keeps one sink.
|
|
455
462
|
*/
|
|
456
463
|
export function createClaudeRunTelemetry(
|
|
457
|
-
opts: ClaudeStreamTelemetryOptions & {
|
|
464
|
+
opts: ClaudeStreamTelemetryOptions & {
|
|
465
|
+
watcherOwnsSubagents: boolean
|
|
466
|
+
/** Where a SUBAGENT conversation's call goes. Absent ⇒ `publish`, the parent's sink. */
|
|
467
|
+
publishSubagent?: (metric: HarnessCallMetric) => void
|
|
468
|
+
},
|
|
458
469
|
): ClaudeRunTelemetry {
|
|
459
470
|
const parent = createClaudeStreamTelemetry(opts)
|
|
460
|
-
const subagents = opts.watcherOwnsSubagents
|
|
471
|
+
const subagents = opts.watcherOwnsSubagents
|
|
472
|
+
? undefined
|
|
473
|
+
: createSubagentStreamTelemetry({ ...opts, publish: opts.publishSubagent ?? opts.publish })
|
|
461
474
|
let sawSubagentTurn = false
|
|
462
475
|
|
|
463
476
|
return {
|
package/src/coding-agent.ts
CHANGED
|
@@ -24,6 +24,8 @@ import {
|
|
|
24
24
|
pushBranch,
|
|
25
25
|
refreshFromBaseIfClean,
|
|
26
26
|
remoteBranchExists,
|
|
27
|
+
unpublishedWorkBranchTip,
|
|
28
|
+
workBranchLease,
|
|
27
29
|
} from './git.js'
|
|
28
30
|
import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js'
|
|
29
31
|
import type { HarnessCallMetric } from './pi.js'
|
|
@@ -280,13 +282,35 @@ function followUpPollIntervalMs(): number {
|
|
|
280
282
|
* concurrently: overlapping pushes race on the remote ref and can make a push fail with a
|
|
281
283
|
* ref-lock / non-fast-forward error — which, on the FINAL push, would fail the whole run even
|
|
282
284
|
* though the work is committed. `pushWorkOnce` coalesces concurrent callers onto one push and only
|
|
283
|
-
* pushes
|
|
285
|
+
* pushes what is UNPUBLISHED ({@link unpublishedWorkBranchTip}: past `baseSha`, and not already the
|
|
286
|
+
* tip the last push published).
|
|
284
287
|
*
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
288
|
+
* Every push after the first LEASES against the sha this pass published (see
|
|
289
|
+
* {@link pushBranch}), because the checkpoint makes the harness its own competing writer: it
|
|
290
|
+
* publishes a commit within a minute of the agent making it, and the agent is then free to amend,
|
|
291
|
+
* reset or rebase that commit, which is perfectly ordinary git hygiene, and the delivery contract
|
|
292
|
+
* asks it to validate AFTER committing, exactly the sequence that produces an amend. Without the
|
|
293
|
+
* lease the final push is refused as a non-fast-forward and the whole run fails with its work
|
|
294
|
+
* already on the branch. The lease is what keeps that recovery from becoming a blanket `--force`:
|
|
295
|
+
* a SECOND writer (a concurrent dispatch for the same block) still refuses the push, which is the
|
|
296
|
+
* "never clobber another run's commits" property the resume design leans on.
|
|
297
|
+
*
|
|
298
|
+
* The lease alone does not bound the force to THIS pass's own commits, and that is the property
|
|
299
|
+
* the design promises, so it is checked rather than assumed: once one checkpoint has landed, a
|
|
300
|
+
* rewrite that drops `baseSha` (the tip the pass started from, which on a RESUMED branch is an
|
|
301
|
+
* earlier run's published work) would lease successfully against our own checkpoint and take the
|
|
302
|
+
* earlier commits with it. So the lease is armed only while the branch still CONTAINS `baseSha`;
|
|
303
|
+
* withheld, the push goes out plain, git refuses it, and the engine re-dispatches onto the branch
|
|
304
|
+
* as it stands. A rewrite this pass cannot prove is its own is never forced away.
|
|
305
|
+
*
|
|
306
|
+
* What is pushable is {@link unpublishedWorkBranchTip}'s question, and both of its answers matter
|
|
307
|
+
* here. A branch still at `baseSha` must not be pushed at all, or a later retry resumes a zero-diff
|
|
308
|
+
* branch and cannot open a PR for it. A branch already at the published tip has nothing to add, and
|
|
309
|
+
* skipping it is what keeps the interval a LOSS WINDOW rather than a push rate: an hour-long run
|
|
310
|
+
* that commits eight times pushes eight times, not sixty. That skip is invisible to the outcome by
|
|
311
|
+
* construction: `finalizeCodingRun` decides `pushed` from the BRANCH (advanced this pass, or
|
|
312
|
+
* resumed), never from whether the final call issued a `git push`, because a tip the checkpoint
|
|
313
|
+
* already published is published.
|
|
290
314
|
*/
|
|
291
315
|
function createWorkBranchPusher(args: {
|
|
292
316
|
dir: string
|
|
@@ -301,11 +325,31 @@ function createWorkBranchPusher(args: {
|
|
|
301
325
|
} {
|
|
302
326
|
const { dir, spec, baseSha, logger, signal } = args
|
|
303
327
|
let pushInFlight: Promise<void> | null = null
|
|
328
|
+
// The sha THIS pass last published to the work branch, and the only value it will ever lease a
|
|
329
|
+
// force push against. Starts unset even on a RESUMED branch: the tip we merely cloned is an
|
|
330
|
+
// earlier run's work, so a rewrite of it is refused (and re-driven) rather than forced away.
|
|
331
|
+
let publishedSha: string | undefined
|
|
304
332
|
const pushWorkOnce = (): Promise<void> => {
|
|
305
333
|
if (pushInFlight) return pushInFlight
|
|
306
334
|
pushInFlight = (async () => {
|
|
307
|
-
if (!(await
|
|
308
|
-
|
|
335
|
+
if (!(await unpublishedWorkBranchTip({ dir, baseSha, publishedSha, signal }))) return
|
|
336
|
+
// The rule the lease is entitled to lives beside the push ({@link workBranchLease}); the
|
|
337
|
+
// warn is here, because a withheld lease is how a rewrite this pass cannot claim fails the
|
|
338
|
+
// push it is about to make, and the run's log is where that is read.
|
|
339
|
+
const lease = await workBranchLease({
|
|
340
|
+
dir,
|
|
341
|
+
branch: spec.pushBranch,
|
|
342
|
+
baseSha,
|
|
343
|
+
publishedSha,
|
|
344
|
+
signal,
|
|
345
|
+
onWithheld: (probe) =>
|
|
346
|
+
logger.warn('coding-agent: push lease withheld, the branch dropped its pre-run tip', {
|
|
347
|
+
baseSha,
|
|
348
|
+
publishedSha,
|
|
349
|
+
probe,
|
|
350
|
+
}),
|
|
351
|
+
})
|
|
352
|
+
publishedSha = await pushBranch(dir, spec.pushBranch, spec.ghToken, signal, lease)
|
|
309
353
|
})().finally(() => {
|
|
310
354
|
pushInFlight = null
|
|
311
355
|
})
|
package/src/failure.ts
CHANGED
|
@@ -26,6 +26,12 @@
|
|
|
26
26
|
* went quiet, this one says the model rabbit-holed while streaming.
|
|
27
27
|
* - `agent` — the agent ran but produced an unusable/failed result, or threw.
|
|
28
28
|
* - `git` — a git operation failed (clone/push/merge/PR).
|
|
29
|
+
* - `branch-contended`: a push to the work branch was REFUSED because the branch carries
|
|
30
|
+
* commits this push would drop (a second writer, or a rewrite of an
|
|
31
|
+
* earlier run's history). Split out of `git` because it is the one git
|
|
32
|
+
* fault the ENGINE can recover from on its own: re-dispatching the step
|
|
33
|
+
* resumes the branch as it now stands, where every other `git` failure
|
|
34
|
+
* would only fail again.
|
|
29
35
|
* - `api` — an upstream API call failed (e.g. the GitHub/GitLab PR/MR REST call).
|
|
30
36
|
* - `llm-upstream` — the model provider rejected every call (auth/quota/rate-limit) and Pi
|
|
31
37
|
* exhausted its retries, so the run never produced a result.
|
|
@@ -38,6 +44,7 @@ export const FAILURE_CAUSES = [
|
|
|
38
44
|
'no-tool-progress',
|
|
39
45
|
'agent',
|
|
40
46
|
'git',
|
|
47
|
+
'branch-contended',
|
|
41
48
|
'api',
|
|
42
49
|
'llm-upstream',
|
|
43
50
|
'no-usable-output',
|
package/src/git.ts
CHANGED
|
@@ -112,6 +112,69 @@ function gitSubcommand(args: string[]): string {
|
|
|
112
112
|
return args.find((a) => a !== '' && !a.startsWith('-')) ?? 'command'
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
+
/**
|
|
116
|
+
* Why a push to the work branch was REFUSED. Both mean the branch carries commits this push
|
|
117
|
+
* would drop, and git tells them apart by whether our object database HOLDS the tip the remote
|
|
118
|
+
* reports: it does for a tip our own checkout created, so the two are distinguishable and need
|
|
119
|
+
* different remedies (see {@link PUSH_REJECTION_REMEDIES}).
|
|
120
|
+
*
|
|
121
|
+
* - `local-rewrite`: we HAVE the remote's tip and are no longer descended from it, i.e. this
|
|
122
|
+
* checkout amended / reset / rebased a commit that had already been pushed. Git labels it
|
|
123
|
+
* `(non-fast-forward)`.
|
|
124
|
+
* - `remote-writer`: the remote's tip is a commit this checkout has never seen (`(fetch first)`),
|
|
125
|
+
* or our lease found the branch moved past what we published (`(stale info)`), so a SECOND
|
|
126
|
+
* writer owns the branch.
|
|
127
|
+
*/
|
|
128
|
+
export type PushRejection = 'local-rewrite' | 'remote-writer'
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Whether `stderr` is a REFUSED push, and which shape. Ordered: the lease/fetch-first shapes are
|
|
132
|
+
* checked first, because a `(stale info)` refusal also prints the generic "failed to push some
|
|
133
|
+
* refs" line the non-fast-forward shape shares. Pure, so both branches are unit-tested against
|
|
134
|
+
* real git output rather than inferred.
|
|
135
|
+
*/
|
|
136
|
+
export function classifyPushRejection(stderr: string): PushRejection | undefined {
|
|
137
|
+
// A HOST-side refusal is not contention, and re-dispatching cannot help: branch protection, a
|
|
138
|
+
// pre-receive hook or a token policy is declining the write itself, and GitHub's protected-branch
|
|
139
|
+
// message says "refusing to allow a non-fast-forward push", which would otherwise read as a
|
|
140
|
+
// rewrite. Git's own labels separate the two cleanly (`! [remote rejected]` is the server
|
|
141
|
+
// declining, `! [rejected]` is git's own fast-forward/lease check), so such a failure stays a
|
|
142
|
+
// plain `git` fault with the write-access remedy below.
|
|
143
|
+
if (/remote rejected|protected branch|hook declined|refusing to allow/i.test(stderr)) {
|
|
144
|
+
return undefined
|
|
145
|
+
}
|
|
146
|
+
if (/\(stale info\)|\(fetch first\)|remote contains work that you do not/i.test(stderr)) {
|
|
147
|
+
return 'remote-writer'
|
|
148
|
+
}
|
|
149
|
+
if (
|
|
150
|
+
/\(non-fast-forward\)|tip of your current branch is behind|branch tip is behind/i.test(stderr)
|
|
151
|
+
) {
|
|
152
|
+
return 'local-rewrite'
|
|
153
|
+
}
|
|
154
|
+
return undefined
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The remedy each {@link PushRejection} earns. A `Record`, so a new rejection shape cannot be
|
|
159
|
+
* classified without saying what a human should do about it. Neither is "run `git pull`", which is
|
|
160
|
+
* what git's own hint advises and is advice for a person at a terminal, not for an autonomous run.
|
|
161
|
+
*/
|
|
162
|
+
const PUSH_REJECTION_REMEDIES: Record<PushRejection, string> = {
|
|
163
|
+
'local-rewrite':
|
|
164
|
+
'The push was refused because the commit it publishes is not descended from the one the work ' +
|
|
165
|
+
'branch already holds: this checkout rewrote history that had already been pushed (an amend, ' +
|
|
166
|
+
"reset or rebase of an existing commit). The platform checkpoint-pushes the agent's commits " +
|
|
167
|
+
'while it works and lets a run force over its OWN published checkpoint, so what stays refused ' +
|
|
168
|
+
'is a rewrite it cannot attribute to this pass: commits an earlier run published, or a rewrite ' +
|
|
169
|
+
'that dropped the branch tip this pass started from. The engine re-dispatches the step to ' +
|
|
170
|
+
'resume from the branch as it stands; work already on the branch is never dropped.',
|
|
171
|
+
'remote-writer':
|
|
172
|
+
'The push was refused because another writer advanced this work branch while the run was ' +
|
|
173
|
+
'working (a second dispatch for the same block, or a person pushing to it). Nothing is lost: ' +
|
|
174
|
+
"the other writer's commits stay on the branch and the engine re-dispatches the step so the " +
|
|
175
|
+
'agent resumes on top of them. If it recurs, check whether two runs are active for the same block.',
|
|
176
|
+
}
|
|
177
|
+
|
|
115
178
|
/**
|
|
116
179
|
* Classify the common shapes of git's own stderr into an actionable remedy, else undefined
|
|
117
180
|
* (an unrecognized failure keeps just its raw stderr). This is the FIRST-WRAP-POINT for
|
|
@@ -123,6 +186,10 @@ function gitSubcommand(args: string[]): string {
|
|
|
123
186
|
*/
|
|
124
187
|
export function describeGitFailure(stderr: string): string | undefined {
|
|
125
188
|
const s = stderr.toLowerCase()
|
|
189
|
+
// A refused push first: its stderr carries neither an auth nor an access shape, so a miss here
|
|
190
|
+
// would leave the operator git's own "use 'git pull' before pushing again" hint and nothing else.
|
|
191
|
+
const rejection = classifyPushRejection(stderr)
|
|
192
|
+
if (rejection) return PUSH_REJECTION_REMEDIES[rejection]
|
|
126
193
|
// Rate-limit / abuse-detection first: the host returns these as a 403, which would
|
|
127
194
|
// otherwise fall into the write-access shape below and be mislabeled as a permission
|
|
128
195
|
// problem — but the fix is to wait, not to grant access.
|
|
@@ -200,13 +267,24 @@ function gitFailure(err: unknown, args: string[], aborted: boolean): HarnessFail
|
|
|
200
267
|
}
|
|
201
268
|
const stderr = typeof e?.stderr === 'string' ? e.stderr : (e?.stderr?.toString() ?? '')
|
|
202
269
|
const base = e instanceof Error ? e.message : String(err)
|
|
203
|
-
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
270
|
+
// `execFile` builds its rejection message as `Command failed: <cmd>\n<stderr>`, so for the
|
|
271
|
+
// ordinary non-zero exit the stderr is ALREADY in `base`, and appending it again printed every
|
|
272
|
+
// git failure's output twice, which reads as two attempts. Append only what `base` lacks
|
|
273
|
+
// (a killed/other rejection whose message carries no output).
|
|
274
|
+
const tail = stderr.trim()
|
|
275
|
+
const combined = tail && !base.includes(tail) ? `${base}\n${tail}` : base
|
|
276
|
+
// Append a cause + fix for the recognized auth/access/push-rejection shapes, keeping the raw
|
|
277
|
+
// (scrubbed) stderr above it as the detail. The remedy is static text with no secrets, so it is
|
|
278
|
+
// added after redaction.
|
|
207
279
|
const remedy = describeGitFailure(combined)
|
|
208
280
|
const message = remedy ? `${redactSecrets(combined)}\n${remedy}` : redactSecrets(combined)
|
|
209
|
-
|
|
281
|
+
// A REFUSED push is not a generic `git` fault: the branch moved under this run, which the engine
|
|
282
|
+
// recovers from by re-dispatching the step onto the branch as it now stands. It gets its own
|
|
283
|
+
// structured cause so that recovery keys off a classification rather than this message.
|
|
284
|
+
const failure = new HarnessFailure(
|
|
285
|
+
classifyPushRejection(combined) ? 'branch-contended' : 'git',
|
|
286
|
+
message,
|
|
287
|
+
)
|
|
210
288
|
if (e?.stack) failure.stack = redactSecrets(e.stack)
|
|
211
289
|
return failure
|
|
212
290
|
}
|
|
@@ -1054,20 +1132,145 @@ export async function fetchPullRequestHead(opts: {
|
|
|
1054
1132
|
}
|
|
1055
1133
|
|
|
1056
1134
|
/**
|
|
1057
|
-
* Push the work branch to origin. The remote URL carries only the
|
|
1058
|
-
* the token is supplied here via the askpass env (never in argv).
|
|
1135
|
+
* Push the work branch to origin and return the sha it PUBLISHED. The remote URL carries only the
|
|
1136
|
+
* username, so the token is supplied here via the askpass env (never in argv).
|
|
1137
|
+
*
|
|
1138
|
+
* The push names an explicit SOURCE COMMIT (`<sha>:refs/heads/<branch>`) rather than the branch,
|
|
1139
|
+
* which is what makes the return value exact rather than a guess. The agent commits while this
|
|
1140
|
+
* runs, so `git push origin <branch>` publishes whatever the branch ref holds at the moment git
|
|
1141
|
+
* reads it, and a caller that leases against a sha it read either side of that has leased against
|
|
1142
|
+
* the wrong commit. Reading it back from `refs/remotes/origin/<branch>` afterwards is worse than
|
|
1143
|
+
* inexact, it is EMPTY on the production checkout: a fresh coding run clones one branch
|
|
1144
|
+
* (`cloneRepo`), so the remote's fetch refspec covers the base alone and `git push` creates no
|
|
1145
|
+
* tracking ref for the work branch at all. Naming the sha needs no ref and no round trip.
|
|
1146
|
+
*
|
|
1147
|
+
* `-u` goes with it: with a non-branch source git sets no upstream config (verified), nothing in
|
|
1148
|
+
* the harness reads that config, and the agent is told never to push or pull.
|
|
1149
|
+
*
|
|
1150
|
+
* `expectRemoteSha` turns the push into a LEASED force (`--force-with-lease=<branch>:<sha>`), which
|
|
1151
|
+
* is how a run whose own checkpoint push it has since rewritten still lands. It is deliberately NOT
|
|
1152
|
+
* a plain `--force`: the lease succeeds only while the remote still holds the sha THIS run
|
|
1153
|
+
* published, so a second writer's commits refuse the push (`(stale info)`) instead of being
|
|
1154
|
+
* clobbered. Callers therefore pass only a sha this same pass published; leasing against a tip we
|
|
1155
|
+
* merely CLONED would force over an earlier run's work.
|
|
1059
1156
|
*/
|
|
1060
1157
|
export async function pushBranch(
|
|
1061
1158
|
dir: string,
|
|
1062
1159
|
branch: string,
|
|
1063
1160
|
ghToken: string,
|
|
1064
1161
|
signal?: AbortSignal,
|
|
1065
|
-
|
|
1066
|
-
|
|
1162
|
+
opts: { expectRemoteSha?: string } = {},
|
|
1163
|
+
): Promise<string> {
|
|
1164
|
+
const sha = (
|
|
1165
|
+
await git(['rev-parse', '--verify', `refs/heads/${branch}`], { cwd: dir, signal })
|
|
1166
|
+
).trim()
|
|
1167
|
+
const lease = opts.expectRemoteSha ? [`--force-with-lease=${branch}:${opts.expectRemoteSha}`] : []
|
|
1168
|
+
await git(['push', ...lease, 'origin', `${sha}:refs/heads/${branch}`], {
|
|
1067
1169
|
cwd: dir,
|
|
1068
1170
|
signal,
|
|
1069
1171
|
env: await authEnv(ghToken),
|
|
1070
1172
|
})
|
|
1173
|
+
return sha
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* Whether `sha` is still reachable from `branch`'s tip, i.e. the branch CONTAINS it:
|
|
1178
|
+
* `git rev-list --count --max-count=1 <sha> --not refs/heads/<branch>` is 0 when everything
|
|
1179
|
+
* reachable from `sha` is reachable from the branch too (the tip itself counts as contained).
|
|
1180
|
+
*
|
|
1181
|
+
* Phrased as a rev-list rather than `merge-base --is-ancestor` on purpose: the latter answers "no"
|
|
1182
|
+
* by EXITING 1, which is indistinguishable here from a broken checkout, and this probe's whole job
|
|
1183
|
+
* is to be trusted only when it is a definite answer. Tri-state for the same reason (as
|
|
1184
|
+
* {@link branchAheadOfBase} is):
|
|
1185
|
+
*
|
|
1186
|
+
* - `true`: confirmed contained.
|
|
1187
|
+
* - `false`: confirmed dropped, so the branch was rewritten below `sha`.
|
|
1188
|
+
* - `undefined`: could not determine (an unknown object, a rev-list error). A caller must not read
|
|
1189
|
+
* a failed probe as either answer.
|
|
1190
|
+
*
|
|
1191
|
+
* The work-branch lease is gated on this: see {@link workBranchLease}.
|
|
1192
|
+
*/
|
|
1193
|
+
export async function branchContainsCommit(
|
|
1194
|
+
dir: string,
|
|
1195
|
+
branch: string,
|
|
1196
|
+
sha: string,
|
|
1197
|
+
signal?: AbortSignal,
|
|
1198
|
+
): Promise<boolean | undefined> {
|
|
1199
|
+
try {
|
|
1200
|
+
const out = await git(
|
|
1201
|
+
['rev-list', '--count', '--max-count=1', sha, '--not', `refs/heads/${branch}`],
|
|
1202
|
+
{ cwd: dir, signal },
|
|
1203
|
+
)
|
|
1204
|
+
const count = Number(out.trim())
|
|
1205
|
+
return Number.isNaN(count) ? undefined : count === 0
|
|
1206
|
+
} catch {
|
|
1207
|
+
return undefined
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
/**
|
|
1212
|
+
* The work branch's tip when it holds something UNPUBLISHED, else undefined: the answer to whether a
|
|
1213
|
+
* checkpoint tick has anything to do. Two ways of having nothing:
|
|
1214
|
+
*
|
|
1215
|
+
* - the tip is still `baseSha`, so this pass has committed nothing. Pushing here would create the
|
|
1216
|
+
* work branch at the base commit, and a later retry would see that zero-diff branch via
|
|
1217
|
+
* `remoteBranchExists`, resume it as work, and fail to open a PR ("no commits between base and
|
|
1218
|
+
* head"). A pass that never commits must leave NO branch behind.
|
|
1219
|
+
* - the tip is `publishedSha`, so the last push already published it. Without this the checkpoint
|
|
1220
|
+
* re-pushed an unchanged branch on every tick: an hour-long run committing eight times issued
|
|
1221
|
+
* ~60 pushes, ~52 of them a full authenticated round trip answering "Everything up-to-date",
|
|
1222
|
+
* each one counting against the host's push rate limits.
|
|
1223
|
+
*
|
|
1224
|
+
* That second condition is also what keeps the INTERVAL the right knob. It expresses the acceptable
|
|
1225
|
+
* loss window when a container dies (a property of the deployment's infra churn), not a rate: gated
|
|
1226
|
+
* this way, the tick publishes at most one push per commit the agent makes, whatever the model or
|
|
1227
|
+
* the run's length, so nothing here needs to be tuned per model.
|
|
1228
|
+
*/
|
|
1229
|
+
export async function unpublishedWorkBranchTip(args: {
|
|
1230
|
+
dir: string
|
|
1231
|
+
/** The branch tip this pass started from. */
|
|
1232
|
+
baseSha: string
|
|
1233
|
+
/** The sha this pass published, if any ({@link pushBranch}'s return). */
|
|
1234
|
+
publishedSha: string | undefined
|
|
1235
|
+
signal?: AbortSignal
|
|
1236
|
+
}): Promise<string | undefined> {
|
|
1237
|
+
const head = await headCommit(args.dir, args.signal)
|
|
1238
|
+
if (head === args.baseSha || head === args.publishedSha) return undefined
|
|
1239
|
+
return head
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* The lease a work-branch push is entitled to (the `opts` {@link pushBranch} takes): the sha this
|
|
1244
|
+
* pass last published, and nothing at all before it has published one.
|
|
1245
|
+
*
|
|
1246
|
+
* The extra condition is what bounds the force to THIS pass's own commits, which the lease alone
|
|
1247
|
+
* does not do and the design promises. Once one checkpoint has landed, a rewrite that drops
|
|
1248
|
+
* `baseSha` (the tip the pass started from, which on a RESUMED branch is an earlier run's published
|
|
1249
|
+
* work) would still lease successfully against our own checkpoint and carry those earlier commits
|
|
1250
|
+
* away with it. So the lease is withheld unless the branch still CONTAINS `baseSha`: the push then
|
|
1251
|
+
* goes out plain, git refuses it as a non-fast-forward, and the engine re-dispatches onto the
|
|
1252
|
+
* branch as it stands.
|
|
1253
|
+
*
|
|
1254
|
+
* A probe that could not answer withholds it too (`onWithheld('unreadable')`), because the two
|
|
1255
|
+
* mistakes are not symmetric: withholding costs a refused rewrite and one re-dispatch, trusting an
|
|
1256
|
+
* unreadable probe costs commits.
|
|
1257
|
+
*/
|
|
1258
|
+
export async function workBranchLease(args: {
|
|
1259
|
+
dir: string
|
|
1260
|
+
branch: string
|
|
1261
|
+
/** The branch tip this pass started from. */
|
|
1262
|
+
baseSha: string
|
|
1263
|
+
/** The sha this pass published, if any (`pushBranch`'s return). */
|
|
1264
|
+
publishedSha: string | undefined
|
|
1265
|
+
signal?: AbortSignal
|
|
1266
|
+
/** Told why the lease was withheld, so the harness can log it with its own logger. */
|
|
1267
|
+
onWithheld?: (probe: 'unreadable' | 'dropped') => void
|
|
1268
|
+
}): Promise<{ expectRemoteSha?: string }> {
|
|
1269
|
+
if (!args.publishedSha) return {}
|
|
1270
|
+
const contains = await branchContainsCommit(args.dir, args.branch, args.baseSha, args.signal)
|
|
1271
|
+
if (contains === true) return { expectRemoteSha: args.publishedSha }
|
|
1272
|
+
args.onWithheld?.(contains === undefined ? 'unreadable' : 'dropped')
|
|
1273
|
+
return {}
|
|
1071
1274
|
}
|
|
1072
1275
|
|
|
1073
1276
|
/**
|