@cat-factory/executor-harness 1.114.0 → 1.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,12 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
3
3
  import { tmpdir } from 'node:os'
4
4
  import { dirname, join } from 'node:path'
5
5
  import { claudeAssistantContent, isObject, numberOf, redactBody } from './claude-stream.js'
6
- import { createClaudeRunTelemetry, subagentDispatchId } from './claude-call-aggregator.js'
6
+ import { claudeUsage, unaccountedUsageCall } from './usage-attribution.js'
7
+ import {
8
+ createClaudeRunTelemetry,
9
+ subagentDispatchId,
10
+ type ClaudeRunTelemetry,
11
+ } from './claude-call-aggregator.js'
7
12
  import {
8
13
  ToolCallTracker,
9
14
  type TrackedToolCall,
@@ -12,9 +17,7 @@ import {
12
17
  import { log, type Logger } from './logger.js'
13
18
  import { NO_TOOL_WINDOW, type ToolProgressWindow } from './tool-silence.js'
14
19
  import {
15
- createCallMetricPublisher,
16
20
  publishCallMetric,
17
- type CallMetricPublisher,
18
21
  type HarnessCallMetric,
19
22
  type PiRunOutcome,
20
23
  type TodoProgress,
@@ -202,24 +205,6 @@ export interface SubscriptionRunOptions {
202
205
  log?: Logger
203
206
  }
204
207
 
205
- /**
206
- * Fallback token attribution: if a CLI reported a cumulative total but no per-turn
207
- * usage (so every captured call has zero tokens), pin the whole total onto the LAST
208
- * call rather than dropping it — the run's tokens are still accounted, just not split
209
- * per turn. A no-op when the calls already carry per-turn tokens.
210
- */
211
- function attributeCumulativeUsage(
212
- calls: HarnessCallMetric[],
213
- usage: { inputTokens: number; outputTokens: number } | undefined,
214
- ): void {
215
- if (!usage || calls.length === 0) return
216
- const anyTokens = calls.some((c) => c.inputTokens > 0 || c.outputTokens > 0)
217
- if (anyTokens) return
218
- const last = calls[calls.length - 1]!
219
- last.inputTokens = usage.inputTokens
220
- last.outputTokens = usage.outputTokens
221
- }
222
-
223
208
  /**
224
209
  * Drive one CLI subprocess to completion, streaming LF-framed JSONL from stdout
225
210
  * through `onEvent`. Mirrors `runPi`'s lifecycle: prompt over stdin (out-of-band,
@@ -711,6 +696,75 @@ function carriesToolResult(content: unknown[]): boolean {
711
696
  return content.some((block) => isObject(block) && block.type === 'tool_result')
712
697
  }
713
698
 
699
+ /** One claude-code run's per-call telemetry: what was captured, and how it is settled. */
700
+ interface ClaudeCallCapture {
701
+ /** Every captured call, terminal-result order — the parent's, the subagents', the remainder. */
702
+ calls: HarnessCallMetric[]
703
+ telemetry: ClaudeRunTelemetry
704
+ /**
705
+ * File whatever the parent's narrated turns did not account for, once its terminal cumulative
706
+ * usage is known. A no-op when they add up. See {@link unaccountedUsageCall}.
707
+ */
708
+ settleUsage: (usage: { inputTokens: number; outputTokens: number } | undefined) => void
709
+ }
710
+
711
+ /**
712
+ * Open the per-call telemetry capture for one claude-code run.
713
+ *
714
+ * It reconstructs the full per-call request/response bodies from the stream.
715
+ * `--output-format stream-json --verbose` emits a near-verbatim Anthropic Messages envelope per
716
+ * response CONTENT BLOCK (not per call), so the aggregator folds the envelopes sharing a
717
+ * `message.id` back into one call and buffers that call's `user` tool_result turns — together the
718
+ * growing prompt transcript, in the shape the model was actually sent. It is SEEDED with the inputs
719
+ * the harness supplies (they never appear in the stream): the system + first user message when the
720
+ * prompt rides argv, or a single folded user turn when it doesn't, so the reconstruction never shows
721
+ * a system turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
722
+ *
723
+ * The parent loop's calls are tracked SEPARATELY, by reference into the same list, because the
724
+ * terminal `result` event's cumulative usage covers only the parent conversation. In `ambientAuth`
725
+ * mode there is no transcript watcher, so the CLI's tagged subagent turns are captured here too and
726
+ * `calls` holds both; reconciling against that mixed list is what once billed a subagent for the
727
+ * parent's whole output shortfall.
728
+ */
729
+ function openClaudeCallCapture(
730
+ opts: SubscriptionRunOptions,
731
+ stream: { prompt: string; folded: boolean; secrets: string[] },
732
+ ): ClaudeCallCapture {
733
+ const calls: HarnessCallMetric[] = []
734
+ const parentCalls: HarnessCallMetric[] = []
735
+ const publish = (metric: HarnessCallMetric): void =>
736
+ publishCallMetric(calls, metric, opts.onCallMetric)
737
+ // `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring in the caller: it is started
738
+ // only when the CLI has an isolated config home to watch, which an `ambientAuth` run does not
739
+ // have. The telemetry routes the CLI's tagged subagent turns accordingly — see
740
+ // `createClaudeRunTelemetry`.
741
+ const telemetry = createClaudeRunTelemetry({
742
+ seed: stream.folded
743
+ ? [{ role: 'user', content: stream.prompt }]
744
+ : [
745
+ { role: 'system', content: opts.systemPrompt },
746
+ { role: 'user', content: opts.userPrompt },
747
+ ],
748
+ secrets: stream.secrets,
749
+ watcherOwnsSubagents: !opts.ambientAuth,
750
+ publish: (metric) => {
751
+ parentCalls.push(metric)
752
+ publish(metric)
753
+ },
754
+ publishSubagent: publish,
755
+ })
756
+ return {
757
+ calls,
758
+ telemetry,
759
+ settleUsage: (usage) => {
760
+ // Published like any other call so the live drain records it too, which is also what stamps
761
+ // its `seq` and therefore its stable row id.
762
+ const remainder = unaccountedUsageCall(parentCalls, usage)
763
+ if (remainder) publish(remainder)
764
+ },
765
+ }
766
+ }
767
+
714
768
  export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRunOutcome> {
715
769
  const stats: PiRunStats = { toolCalls: 0, assistantChars: 0 }
716
770
  let summary = ''
@@ -729,34 +783,9 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
729
783
  })
730
784
  }
731
785
 
732
- // Reconstruct the full per-call request/response bodies for telemetry from the
733
- // stream. `--output-format stream-json --verbose` emits a near-verbatim Anthropic
734
- // Messages envelope per response CONTENT BLOCK (not per call), so the aggregator below
735
- // folds the envelopes sharing a `message.id` back into one call and buffers that call's
736
- // `user` tool_result turns — together the growing prompt transcript, in the shape the
737
- // model was actually sent. We seed it with the inputs the harness supplies (they never
738
- // appear in the stream): the system + first user message when the prompt rides argv, or
739
- // a single folded user turn when it doesn't — so the reconstruction never shows a system
740
- // turn that was never sent. Bodies are credential-scrubbed (they can echo the leased token).
741
786
  const secrets = opts.subscriptionToken ? secretsToRedact(opts.subscriptionToken) : []
742
- const calls: HarnessCallMetric[] = []
743
- // Streams each call as the CLI yields it, EXCEPT one whose tokens `attributeCumulativeUsage`
744
- // may still rewrite below (a published call must be final — see the publisher).
745
- const publisher = createCallMetricPublisher(calls, opts.onCallMetric)
746
- // `watcherOwnsSubagents` tracks the `startSubagentWatcher` wiring below: it is started only when
747
- // the CLI has an isolated config home to watch, which an `ambientAuth` run does not have. The
748
- // telemetry routes the CLI's tagged subagent turns accordingly — see `createClaudeRunTelemetry`.
749
- const telemetry = createClaudeRunTelemetry({
750
- seed: folded
751
- ? [{ role: 'user', content: prompt }]
752
- : [
753
- { role: 'system', content: opts.systemPrompt },
754
- { role: 'user', content: opts.userPrompt },
755
- ],
756
- secrets,
757
- watcherOwnsSubagents: !opts.ambientAuth,
758
- publish: (metric) => publisher.publish(metric),
759
- })
787
+ const capture = openClaudeCallCapture(opts, { prompt, folded, secrets })
788
+ const telemetry = capture.telemetry
760
789
 
761
790
  // ADR 0026 D2.1 + ADR 0027 Defect B: surface live slice progress from the two views the run
762
791
  // produces of the SAME slicing. The parent's subagent dispatches + their terminal tool_results
@@ -917,8 +946,7 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
917
946
  summary,
918
947
  stats,
919
948
  stderrTail,
920
- calls,
921
- publisher,
949
+ capture,
922
950
  usage,
923
951
  subagents,
924
952
  expectSubagentCalls: telemetry.expectsWatcherCalls(),
@@ -926,11 +954,11 @@ export async function runClaudeCode(opts: SubscriptionRunOptions): Promise<PiRun
926
954
  })
927
955
  } catch (err) {
928
956
  // The stream ended abnormally (guard trip, watchdog kill, CLI crash). Complete the call in
929
- // flight anyway, and release whatever the publisher was withholding: a killed run never
930
- // returns an outcome, so the live channel is the ONLY record of what it spent, and dropping
931
- // its last turn is what the streaming exists to avoid.
957
+ // flight anyway: a killed run never returns an outcome, so the live channel is the ONLY record
958
+ // of what it spent, and dropping its last turn is what the streaming exists to avoid. No
959
+ // terminal `result` event arrived, so there is no cumulative total to reconcile against and no
960
+ // remainder row to file — every captured turn already streamed as it was completed.
932
961
  telemetry.flush()
933
- publisher.flush()
934
962
  // A tripped no-progress guard aborted the CLI; streamCli rejects with its generic abort
935
963
  // message, so replace it with the guard's actionable diagnostic — carrying the stderr tail it
936
964
  // attached, since that is usually the only evidence of what the CLI was doing when it was
@@ -1088,9 +1116,8 @@ async function assembleClaudeOutcome(args: {
1088
1116
  summary: string
1089
1117
  stats: PiRunStats
1090
1118
  stderrTail: string
1091
- calls: HarnessCallMetric[]
1092
- /** The live-stream publisher, flushed once attribution has finalised the calls' tokens. */
1093
- publisher: CallMetricPublisher
1119
+ /** This run's per-call telemetry, settled here with the terminal usage. */
1120
+ capture: ClaudeCallCapture
1094
1121
  usage: { inputTokens: number; outputTokens: number } | undefined
1095
1122
  subagents: ReturnType<typeof startSubagentWatcher> | undefined
1096
1123
  /**
@@ -1102,14 +1129,11 @@ async function assembleClaudeOutcome(args: {
1102
1129
  expectSubagentCalls: boolean
1103
1130
  log?: Logger
1104
1131
  }): Promise<PiRunOutcome> {
1105
- const { summary, stats, stderrTail, calls, publisher, usage, subagents } = args
1106
- // The parent's cumulative-usage fallback applies to the PARENT calls only (before the
1107
- // subagent calls, which carry their own per-turn tokens, are concatenated).
1108
- attributeCumulativeUsage(calls, usage)
1109
- // The withheld calls are final only NOW, so stream them: the completion poll drains them
1110
- // alongside the result, and the backend records the attributed numbers rather than the zeros
1111
- // they carried while the run was in flight.
1112
- publisher.flush()
1132
+ const { summary, stats, stderrTail, capture, usage, subagents } = args
1133
+ const calls = capture.calls
1134
+ // What the parent's narrated turns did not account for, as its OWN row (never tokens grafted onto
1135
+ // a real turn).
1136
+ capture.settleUsage(usage)
1113
1137
  // Final drain of any subagent transcript writes that landed after the last poll, then
1114
1138
  // fold the subagents' usage + per-call telemetry into the run's outcome.
1115
1139
  await subagents?.stop()
@@ -1138,21 +1162,6 @@ async function assembleClaudeOutcome(args: {
1138
1162
  }
1139
1163
  }
1140
1164
 
1141
- function claudeUsage(raw: unknown): { inputTokens: number; outputTokens: number } | undefined {
1142
- if (!isObject(raw)) return undefined
1143
- // Count every input bucket Anthropic bills: fresh input plus BOTH cache reads and
1144
- // cache writes (cache_creation_input_tokens), which are real consumed tokens — and
1145
- // are the dominant share on a long agent run. Omitting them under-weights a token's
1146
- // true load in the usage-aware rotation window.
1147
- const input =
1148
- numberOf(raw.input_tokens) +
1149
- numberOf(raw.cache_read_input_tokens) +
1150
- numberOf(raw.cache_creation_input_tokens)
1151
- const output = numberOf(raw.output_tokens)
1152
- if (input === 0 && output === 0) return undefined
1153
- return { inputTokens: input, outputTokens: output }
1154
- }
1155
-
1156
1165
  // ---------------------------------------------------------------------------
1157
1166
  // Codex
1158
1167
  // ---------------------------------------------------------------------------
@@ -452,12 +452,25 @@ export interface ClaudeRunTelemetry {
452
452
  * `ambientAuth` run has no isolated config home to watch — the tagged turns are recorded here
453
453
  * instead, on per-dispatch transcripts of their own. Dropping them in that case would leave the run
454
454
  * billed by neither channel, and an under-count reads as a cheap run rather than as an error.
455
+ *
456
+ * Which is also why the two are published through SEPARATE callbacks. A caller reconciling the
457
+ * parent's terminal cumulative usage needs the parent's calls alone, and with one shared callback
458
+ * the fallback channel silently mixed subagent turns into that list — where they both understated
459
+ * the shortfall and, being last, attracted it (`unaccountedUsageCall`). `publishSubagent` is
460
+ * optional so a caller that draws no distinction (a test, the settled-transcript path where nothing
461
+ * arrives here anyway) keeps one sink.
455
462
  */
456
463
  export function createClaudeRunTelemetry(
457
- opts: ClaudeStreamTelemetryOptions & { watcherOwnsSubagents: boolean },
464
+ opts: ClaudeStreamTelemetryOptions & {
465
+ watcherOwnsSubagents: boolean
466
+ /** Where a SUBAGENT conversation's call goes. Absent ⇒ `publish`, the parent's sink. */
467
+ publishSubagent?: (metric: HarnessCallMetric) => void
468
+ },
458
469
  ): ClaudeRunTelemetry {
459
470
  const parent = createClaudeStreamTelemetry(opts)
460
- const subagents = opts.watcherOwnsSubagents ? undefined : createSubagentStreamTelemetry(opts)
471
+ const subagents = opts.watcherOwnsSubagents
472
+ ? undefined
473
+ : createSubagentStreamTelemetry({ ...opts, publish: opts.publishSubagent ?? opts.publish })
461
474
  let sawSubagentTurn = false
462
475
 
463
476
  return {
@@ -24,6 +24,8 @@ import {
24
24
  pushBranch,
25
25
  refreshFromBaseIfClean,
26
26
  remoteBranchExists,
27
+ unpublishedWorkBranchTip,
28
+ workBranchLease,
27
29
  } from './git.js'
28
30
  import { FOLLOW_UPS_FILENAME, FollowUpTailer } from './follow-ups.js'
29
31
  import type { HarnessCallMetric } from './pi.js'
@@ -280,13 +282,35 @@ function followUpPollIntervalMs(): number {
280
282
  * concurrently: overlapping pushes race on the remote ref and can make a push fail with a
281
283
  * ref-lock / non-fast-forward error — which, on the FINAL push, would fail the whole run even
282
284
  * though the work is committed. `pushWorkOnce` coalesces concurrent callers onto one push and only
283
- * pushes once the branch has advanced past `baseSha`.
285
+ * pushes what is UNPUBLISHED ({@link unpublishedWorkBranchTip}: past `baseSha`, and not already the
286
+ * tip the last push published).
284
287
  *
285
- * Only push once the branch has advanced past its pre-run tip: pushing while it still sits at
286
- * `baseSha` would create the work branch at the base commit (a zero-diff branch), which a later
287
- * retry would see via `remoteBranchExists` and treat as resumable work then fail to open a PR
288
- * ("no commits between base and head"). So a run that never commits leaves NO branch behind,
289
- * preserving the clean no-op outcome.
288
+ * Every push after the first LEASES against the sha this pass published (see
289
+ * {@link pushBranch}), because the checkpoint makes the harness its own competing writer: it
290
+ * publishes a commit within a minute of the agent making it, and the agent is then free to amend,
291
+ * reset or rebase that commit, which is perfectly ordinary git hygiene, and the delivery contract
292
+ * asks it to validate AFTER committing, exactly the sequence that produces an amend. Without the
293
+ * lease the final push is refused as a non-fast-forward and the whole run fails with its work
294
+ * already on the branch. The lease is what keeps that recovery from becoming a blanket `--force`:
295
+ * a SECOND writer (a concurrent dispatch for the same block) still refuses the push, which is the
296
+ * "never clobber another run's commits" property the resume design leans on.
297
+ *
298
+ * The lease alone does not bound the force to THIS pass's own commits, and that is the property
299
+ * the design promises, so it is checked rather than assumed: once one checkpoint has landed, a
300
+ * rewrite that drops `baseSha` (the tip the pass started from, which on a RESUMED branch is an
301
+ * earlier run's published work) would lease successfully against our own checkpoint and take the
302
+ * earlier commits with it. So the lease is armed only while the branch still CONTAINS `baseSha`;
303
+ * withheld, the push goes out plain, git refuses it, and the engine re-dispatches onto the branch
304
+ * as it stands. A rewrite this pass cannot prove is its own is never forced away.
305
+ *
306
+ * What is pushable is {@link unpublishedWorkBranchTip}'s question, and both of its answers matter
307
+ * here. A branch still at `baseSha` must not be pushed at all, or a later retry resumes a zero-diff
308
+ * branch and cannot open a PR for it. A branch already at the published tip has nothing to add, and
309
+ * skipping it is what keeps the interval a LOSS WINDOW rather than a push rate: an hour-long run
310
+ * that commits eight times pushes eight times, not sixty. That skip is invisible to the outcome by
311
+ * construction: `finalizeCodingRun` decides `pushed` from the BRANCH (advanced this pass, or
312
+ * resumed), never from whether the final call issued a `git push`, because a tip the checkpoint
313
+ * already published is published.
290
314
  */
291
315
  function createWorkBranchPusher(args: {
292
316
  dir: string
@@ -301,11 +325,31 @@ function createWorkBranchPusher(args: {
301
325
  } {
302
326
  const { dir, spec, baseSha, logger, signal } = args
303
327
  let pushInFlight: Promise<void> | null = null
328
+ // The sha THIS pass last published to the work branch, and the only value it will ever lease a
329
+ // force push against. Starts unset even on a RESUMED branch: the tip we merely cloned is an
330
+ // earlier run's work, so a rewrite of it is refused (and re-driven) rather than forced away.
331
+ let publishedSha: string | undefined
304
332
  const pushWorkOnce = (): Promise<void> => {
305
333
  if (pushInFlight) return pushInFlight
306
334
  pushInFlight = (async () => {
307
- if (!(await branchHasCommitsSince(dir, baseSha, signal))) return
308
- await pushBranch(dir, spec.pushBranch, spec.ghToken, signal)
335
+ if (!(await unpublishedWorkBranchTip({ dir, baseSha, publishedSha, signal }))) return
336
+ // The rule the lease is entitled to lives beside the push ({@link workBranchLease}); the
337
+ // warn is here, because a withheld lease is how a rewrite this pass cannot claim fails the
338
+ // push it is about to make, and the run's log is where that is read.
339
+ const lease = await workBranchLease({
340
+ dir,
341
+ branch: spec.pushBranch,
342
+ baseSha,
343
+ publishedSha,
344
+ signal,
345
+ onWithheld: (probe) =>
346
+ logger.warn('coding-agent: push lease withheld, the branch dropped its pre-run tip', {
347
+ baseSha,
348
+ publishedSha,
349
+ probe,
350
+ }),
351
+ })
352
+ publishedSha = await pushBranch(dir, spec.pushBranch, spec.ghToken, signal, lease)
309
353
  })().finally(() => {
310
354
  pushInFlight = null
311
355
  })
package/src/failure.ts CHANGED
@@ -26,6 +26,12 @@
26
26
  * went quiet, this one says the model rabbit-holed while streaming.
27
27
  * - `agent` — the agent ran but produced an unusable/failed result, or threw.
28
28
  * - `git` — a git operation failed (clone/push/merge/PR).
29
+ * - `branch-contended`: a push to the work branch was REFUSED because the branch carries
30
+ * commits this push would drop (a second writer, or a rewrite of an
31
+ * earlier run's history). Split out of `git` because it is the one git
32
+ * fault the ENGINE can recover from on its own: re-dispatching the step
33
+ * resumes the branch as it now stands, where every other `git` failure
34
+ * would only fail again.
29
35
  * - `api` — an upstream API call failed (e.g. the GitHub/GitLab PR/MR REST call).
30
36
  * - `llm-upstream` — the model provider rejected every call (auth/quota/rate-limit) and Pi
31
37
  * exhausted its retries, so the run never produced a result.
@@ -38,6 +44,7 @@ export const FAILURE_CAUSES = [
38
44
  'no-tool-progress',
39
45
  'agent',
40
46
  'git',
47
+ 'branch-contended',
41
48
  'api',
42
49
  'llm-upstream',
43
50
  'no-usable-output',
package/src/git.ts CHANGED
@@ -112,6 +112,69 @@ function gitSubcommand(args: string[]): string {
112
112
  return args.find((a) => a !== '' && !a.startsWith('-')) ?? 'command'
113
113
  }
114
114
 
115
+ /**
116
+ * Why a push to the work branch was REFUSED. Both mean the branch carries commits this push
117
+ * would drop, and git tells them apart by whether our object database HOLDS the tip the remote
118
+ * reports: it does for a tip our own checkout created, so the two are distinguishable and need
119
+ * different remedies (see {@link PUSH_REJECTION_REMEDIES}).
120
+ *
121
+ * - `local-rewrite`: we HAVE the remote's tip and are no longer descended from it, i.e. this
122
+ * checkout amended / reset / rebased a commit that had already been pushed. Git labels it
123
+ * `(non-fast-forward)`.
124
+ * - `remote-writer`: the remote's tip is a commit this checkout has never seen (`(fetch first)`),
125
+ * or our lease found the branch moved past what we published (`(stale info)`), so a SECOND
126
+ * writer owns the branch.
127
+ */
128
+ export type PushRejection = 'local-rewrite' | 'remote-writer'
129
+
130
+ /**
131
+ * Whether `stderr` is a REFUSED push, and which shape. Ordered: the lease/fetch-first shapes are
132
+ * checked first, because a `(stale info)` refusal also prints the generic "failed to push some
133
+ * refs" line the non-fast-forward shape shares. Pure, so both branches are unit-tested against
134
+ * real git output rather than inferred.
135
+ */
136
+ export function classifyPushRejection(stderr: string): PushRejection | undefined {
137
+ // A HOST-side refusal is not contention, and re-dispatching cannot help: branch protection, a
138
+ // pre-receive hook or a token policy is declining the write itself, and GitHub's protected-branch
139
+ // message says "refusing to allow a non-fast-forward push", which would otherwise read as a
140
+ // rewrite. Git's own labels separate the two cleanly (`! [remote rejected]` is the server
141
+ // declining, `! [rejected]` is git's own fast-forward/lease check), so such a failure stays a
142
+ // plain `git` fault with the write-access remedy below.
143
+ if (/remote rejected|protected branch|hook declined|refusing to allow/i.test(stderr)) {
144
+ return undefined
145
+ }
146
+ if (/\(stale info\)|\(fetch first\)|remote contains work that you do not/i.test(stderr)) {
147
+ return 'remote-writer'
148
+ }
149
+ if (
150
+ /\(non-fast-forward\)|tip of your current branch is behind|branch tip is behind/i.test(stderr)
151
+ ) {
152
+ return 'local-rewrite'
153
+ }
154
+ return undefined
155
+ }
156
+
157
+ /**
158
+ * The remedy each {@link PushRejection} earns. A `Record`, so a new rejection shape cannot be
159
+ * classified without saying what a human should do about it. Neither is "run `git pull`", which is
160
+ * what git's own hint advises and is advice for a person at a terminal, not for an autonomous run.
161
+ */
162
+ const PUSH_REJECTION_REMEDIES: Record<PushRejection, string> = {
163
+ 'local-rewrite':
164
+ 'The push was refused because the commit it publishes is not descended from the one the work ' +
165
+ 'branch already holds: this checkout rewrote history that had already been pushed (an amend, ' +
166
+ "reset or rebase of an existing commit). The platform checkpoint-pushes the agent's commits " +
167
+ 'while it works and lets a run force over its OWN published checkpoint, so what stays refused ' +
168
+ 'is a rewrite it cannot attribute to this pass: commits an earlier run published, or a rewrite ' +
169
+ 'that dropped the branch tip this pass started from. The engine re-dispatches the step to ' +
170
+ 'resume from the branch as it stands; work already on the branch is never dropped.',
171
+ 'remote-writer':
172
+ 'The push was refused because another writer advanced this work branch while the run was ' +
173
+ 'working (a second dispatch for the same block, or a person pushing to it). Nothing is lost: ' +
174
+ "the other writer's commits stay on the branch and the engine re-dispatches the step so the " +
175
+ 'agent resumes on top of them. If it recurs, check whether two runs are active for the same block.',
176
+ }
177
+
115
178
  /**
116
179
  * Classify the common shapes of git's own stderr into an actionable remedy, else undefined
117
180
  * (an unrecognized failure keeps just its raw stderr). This is the FIRST-WRAP-POINT for
@@ -123,6 +186,10 @@ function gitSubcommand(args: string[]): string {
123
186
  */
124
187
  export function describeGitFailure(stderr: string): string | undefined {
125
188
  const s = stderr.toLowerCase()
189
+ // A refused push first: its stderr carries neither an auth nor an access shape, so a miss here
190
+ // would leave the operator git's own "use 'git pull' before pushing again" hint and nothing else.
191
+ const rejection = classifyPushRejection(stderr)
192
+ if (rejection) return PUSH_REJECTION_REMEDIES[rejection]
126
193
  // Rate-limit / abuse-detection first: the host returns these as a 403, which would
127
194
  // otherwise fall into the write-access shape below and be mislabeled as a permission
128
195
  // problem — but the fix is to wait, not to grant access.
@@ -200,13 +267,24 @@ function gitFailure(err: unknown, args: string[], aborted: boolean): HarnessFail
200
267
  }
201
268
  const stderr = typeof e?.stderr === 'string' ? e.stderr : (e?.stderr?.toString() ?? '')
202
269
  const base = e instanceof Error ? e.message : String(err)
203
- const combined = stderr.trim() ? `${base}\n${stderr.trim()}` : base
204
- // Append a cause + fix for the recognized auth/access shapes, keeping the raw (scrubbed)
205
- // stderr above it as the detail. The remedy is static text with no secrets, so it is added
206
- // after redaction.
270
+ // `execFile` builds its rejection message as `Command failed: <cmd>\n<stderr>`, so for the
271
+ // ordinary non-zero exit the stderr is ALREADY in `base`, and appending it again printed every
272
+ // git failure's output twice, which reads as two attempts. Append only what `base` lacks
273
+ // (a killed/other rejection whose message carries no output).
274
+ const tail = stderr.trim()
275
+ const combined = tail && !base.includes(tail) ? `${base}\n${tail}` : base
276
+ // Append a cause + fix for the recognized auth/access/push-rejection shapes, keeping the raw
277
+ // (scrubbed) stderr above it as the detail. The remedy is static text with no secrets, so it is
278
+ // added after redaction.
207
279
  const remedy = describeGitFailure(combined)
208
280
  const message = remedy ? `${redactSecrets(combined)}\n${remedy}` : redactSecrets(combined)
209
- const failure = new HarnessFailure('git', message)
281
+ // A REFUSED push is not a generic `git` fault: the branch moved under this run, which the engine
282
+ // recovers from by re-dispatching the step onto the branch as it now stands. It gets its own
283
+ // structured cause so that recovery keys off a classification rather than this message.
284
+ const failure = new HarnessFailure(
285
+ classifyPushRejection(combined) ? 'branch-contended' : 'git',
286
+ message,
287
+ )
210
288
  if (e?.stack) failure.stack = redactSecrets(e.stack)
211
289
  return failure
212
290
  }
@@ -1054,20 +1132,145 @@ export async function fetchPullRequestHead(opts: {
1054
1132
  }
1055
1133
 
1056
1134
  /**
1057
- * Push the work branch to origin. The remote URL carries only the username, so
1058
- * the token is supplied here via the askpass env (never in argv).
1135
+ * Push the work branch to origin and return the sha it PUBLISHED. The remote URL carries only the
1136
+ * username, so the token is supplied here via the askpass env (never in argv).
1137
+ *
1138
+ * The push names an explicit SOURCE COMMIT (`<sha>:refs/heads/<branch>`) rather than the branch,
1139
+ * which is what makes the return value exact rather than a guess. The agent commits while this
1140
+ * runs, so `git push origin <branch>` publishes whatever the branch ref holds at the moment git
1141
+ * reads it, and a caller that leases against a sha it read either side of that has leased against
1142
+ * the wrong commit. Reading it back from `refs/remotes/origin/<branch>` afterwards is worse than
1143
+ * inexact, it is EMPTY on the production checkout: a fresh coding run clones one branch
1144
+ * (`cloneRepo`), so the remote's fetch refspec covers the base alone and `git push` creates no
1145
+ * tracking ref for the work branch at all. Naming the sha needs no ref and no round trip.
1146
+ *
1147
+ * `-u` goes with it: with a non-branch source git sets no upstream config (verified), nothing in
1148
+ * the harness reads that config, and the agent is told never to push or pull.
1149
+ *
1150
+ * `expectRemoteSha` turns the push into a LEASED force (`--force-with-lease=<branch>:<sha>`), which
1151
+ * is how a run whose own checkpoint push it has since rewritten still lands. It is deliberately NOT
1152
+ * a plain `--force`: the lease succeeds only while the remote still holds the sha THIS run
1153
+ * published, so a second writer's commits refuse the push (`(stale info)`) instead of being
1154
+ * clobbered. Callers therefore pass only a sha this same pass published; leasing against a tip we
1155
+ * merely CLONED would force over an earlier run's work.
1059
1156
  */
1060
1157
  export async function pushBranch(
1061
1158
  dir: string,
1062
1159
  branch: string,
1063
1160
  ghToken: string,
1064
1161
  signal?: AbortSignal,
1065
- ): Promise<void> {
1066
- await git(['push', '-u', 'origin', branch], {
1162
+ opts: { expectRemoteSha?: string } = {},
1163
+ ): Promise<string> {
1164
+ const sha = (
1165
+ await git(['rev-parse', '--verify', `refs/heads/${branch}`], { cwd: dir, signal })
1166
+ ).trim()
1167
+ const lease = opts.expectRemoteSha ? [`--force-with-lease=${branch}:${opts.expectRemoteSha}`] : []
1168
+ await git(['push', ...lease, 'origin', `${sha}:refs/heads/${branch}`], {
1067
1169
  cwd: dir,
1068
1170
  signal,
1069
1171
  env: await authEnv(ghToken),
1070
1172
  })
1173
+ return sha
1174
+ }
1175
+
1176
+ /**
1177
+ * Whether `sha` is still reachable from `branch`'s tip, i.e. the branch CONTAINS it:
1178
+ * `git rev-list --count --max-count=1 <sha> --not refs/heads/<branch>` is 0 when everything
1179
+ * reachable from `sha` is reachable from the branch too (the tip itself counts as contained).
1180
+ *
1181
+ * Phrased as a rev-list rather than `merge-base --is-ancestor` on purpose: the latter answers "no"
1182
+ * by EXITING 1, which is indistinguishable here from a broken checkout, and this probe's whole job
1183
+ * is to be trusted only when it is a definite answer. Tri-state for the same reason (as
1184
+ * {@link branchAheadOfBase} is):
1185
+ *
1186
+ * - `true`: confirmed contained.
1187
+ * - `false`: confirmed dropped, so the branch was rewritten below `sha`.
1188
+ * - `undefined`: could not determine (an unknown object, a rev-list error). A caller must not read
1189
+ * a failed probe as either answer.
1190
+ *
1191
+ * The work-branch lease is gated on this: see {@link workBranchLease}.
1192
+ */
1193
+ export async function branchContainsCommit(
1194
+ dir: string,
1195
+ branch: string,
1196
+ sha: string,
1197
+ signal?: AbortSignal,
1198
+ ): Promise<boolean | undefined> {
1199
+ try {
1200
+ const out = await git(
1201
+ ['rev-list', '--count', '--max-count=1', sha, '--not', `refs/heads/${branch}`],
1202
+ { cwd: dir, signal },
1203
+ )
1204
+ const count = Number(out.trim())
1205
+ return Number.isNaN(count) ? undefined : count === 0
1206
+ } catch {
1207
+ return undefined
1208
+ }
1209
+ }
1210
+
1211
+ /**
1212
+ * The work branch's tip when it holds something UNPUBLISHED, else undefined: the answer to whether a
1213
+ * checkpoint tick has anything to do. Two ways of having nothing:
1214
+ *
1215
+ * - the tip is still `baseSha`, so this pass has committed nothing. Pushing here would create the
1216
+ * work branch at the base commit, and a later retry would see that zero-diff branch via
1217
+ * `remoteBranchExists`, resume it as work, and fail to open a PR ("no commits between base and
1218
+ * head"). A pass that never commits must leave NO branch behind.
1219
+ * - the tip is `publishedSha`, so the last push already published it. Without this the checkpoint
1220
+ * re-pushed an unchanged branch on every tick: an hour-long run committing eight times issued
1221
+ * ~60 pushes, ~52 of them a full authenticated round trip answering "Everything up-to-date",
1222
+ * each one counting against the host's push rate limits.
1223
+ *
1224
+ * That second condition is also what keeps the INTERVAL the right knob. It expresses the acceptable
1225
+ * loss window when a container dies (a property of the deployment's infra churn), not a rate: gated
1226
+ * this way, the tick publishes at most one push per commit the agent makes, whatever the model or
1227
+ * the run's length, so nothing here needs to be tuned per model.
1228
+ */
1229
+ export async function unpublishedWorkBranchTip(args: {
1230
+ dir: string
1231
+ /** The branch tip this pass started from. */
1232
+ baseSha: string
1233
+ /** The sha this pass published, if any ({@link pushBranch}'s return). */
1234
+ publishedSha: string | undefined
1235
+ signal?: AbortSignal
1236
+ }): Promise<string | undefined> {
1237
+ const head = await headCommit(args.dir, args.signal)
1238
+ if (head === args.baseSha || head === args.publishedSha) return undefined
1239
+ return head
1240
+ }
1241
+
1242
+ /**
1243
+ * The lease a work-branch push is entitled to (the `opts` {@link pushBranch} takes): the sha this
1244
+ * pass last published, and nothing at all before it has published one.
1245
+ *
1246
+ * The extra condition is what bounds the force to THIS pass's own commits, which the lease alone
1247
+ * does not do and the design promises. Once one checkpoint has landed, a rewrite that drops
1248
+ * `baseSha` (the tip the pass started from, which on a RESUMED branch is an earlier run's published
1249
+ * work) would still lease successfully against our own checkpoint and carry those earlier commits
1250
+ * away with it. So the lease is withheld unless the branch still CONTAINS `baseSha`: the push then
1251
+ * goes out plain, git refuses it as a non-fast-forward, and the engine re-dispatches onto the
1252
+ * branch as it stands.
1253
+ *
1254
+ * A probe that could not answer withholds it too (`onWithheld('unreadable')`), because the two
1255
+ * mistakes are not symmetric: withholding costs a refused rewrite and one re-dispatch, trusting an
1256
+ * unreadable probe costs commits.
1257
+ */
1258
+ export async function workBranchLease(args: {
1259
+ dir: string
1260
+ branch: string
1261
+ /** The branch tip this pass started from. */
1262
+ baseSha: string
1263
+ /** The sha this pass published, if any (`pushBranch`'s return). */
1264
+ publishedSha: string | undefined
1265
+ signal?: AbortSignal
1266
+ /** Told why the lease was withheld, so the harness can log it with its own logger. */
1267
+ onWithheld?: (probe: 'unreadable' | 'dropped') => void
1268
+ }): Promise<{ expectRemoteSha?: string }> {
1269
+ if (!args.publishedSha) return {}
1270
+ const contains = await branchContainsCommit(args.dir, args.branch, args.baseSha, args.signal)
1271
+ if (contains === true) return { expectRemoteSha: args.publishedSha }
1272
+ args.onWithheld?.(contains === undefined ? 'unreadable' : 'dropped')
1273
+ return {}
1071
1274
  }
1072
1275
 
1073
1276
  /**