@haiyangbg/buildbeat 2.0.0-beta.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/v2/cli/run.js CHANGED
@@ -17,11 +17,11 @@ import { fileURLToPath } from "node:url";
17
17
 
18
18
  import { createShellAdapter } from "../adapters/shell.js";
19
19
  import { loadRiskPreset } from "../engine/risk-preset.js";
20
- import { loadWorkflow } from "../engine/workflow.js";
20
+ import { loadWorkflow, nextStep } from "../engine/workflow.js";
21
21
  import { parseYamlSubset } from "../engine/yaml-subset.js";
22
22
  import { parsePolicyDoc } from "../policy/policy.js";
23
23
  import { observeStatus, runObserveCycle, triageIntent } from "../observe/observe.js";
24
- import { acceptArtifact, approveRun, listInbox, rejectRun } from "../runtime/decisions.js";
24
+ import { acceptArtifact, adoptCandidate, approveRun, listInbox, rejectRun } from "../runtime/decisions.js";
25
25
  import { checkRequires } from "../runtime/env-contract.js";
26
26
  import {
27
27
  adjudicateFinding,
@@ -31,7 +31,7 @@ import {
31
31
  } from "../runtime/findings.js";
32
32
  import { loadEnvelope, nextAttemptId } from "../runtime/envelope.js";
33
33
  import { applyGc, planGc } from "../runtime/gc.js";
34
- import { computeOverview, renderOverview } from "../runtime/overview.js";
34
+ import { artifactStatus, computeOverview, readJsonl, renderOverview } from "../runtime/overview.js";
35
35
  import {
36
36
  DEFAULT_STALL_AFTER_MS,
37
37
  describeLiveness,
@@ -51,7 +51,7 @@ import { writeRunRecord } from "../runtime/run-record.js";
51
51
  import { resumeRun, startRun } from "../runtime/orchestrator.js";
52
52
  import { toRepoRef } from "../runtime/repo-ref.js";
53
53
  import { EventLedger } from "../storage/event-ledger.js";
54
- import { acquireLock, releaseLock } from "../workspace/workspace-manager.js";
54
+ import { acquireLock, listHeldRunLocks, releaseLock } from "../workspace/workspace-manager.js";
55
55
 
56
56
  const KERNEL = { kind: "kernel", id: "cli" };
57
57
 
@@ -59,7 +59,7 @@ const USAGE = `BuildBeat v2 runtime
59
59
 
60
60
  Usage:
61
61
  run.js start --config <run-config.yaml> [--attempt new]
62
- run.js resume --config <run-config.yaml>
62
+ run.js resume --config <run-config.yaml> [--adopt <sha> --by <name>] # --adopt: hand fix committed in the worktree; skip fix, resume at verify
63
63
  run.js status --repo <path> --run <RUN-ID> [--stall-after <minutes>]
64
64
  run.js inbox --repo <path>
65
65
  run.js overview --repo <path> [--work <WORK-ID>] [--json true]
@@ -281,6 +281,43 @@ function loadRunConfig(flags, command) {
281
281
  if (config.stallAfterMs !== undefined && !(Number(config.stallAfterMs) > 0)) {
282
282
  throw new Error(`stallAfterMs must be a positive number, got: ${config.stallAfterMs}`);
283
283
  }
284
+ // budgets: run config beats the preset (the preset's two review rounds
285
+ // could not be raised per run before; a pilot's shipped candidates ended
286
+ // as CANCELLED runs because of it).
287
+ const budgets = {};
288
+ if (config.budgets !== undefined) {
289
+ if (!config.budgets || typeof config.budgets !== "object" || Array.isArray(config.budgets)) {
290
+ throw new Error("budgets must be a map");
291
+ }
292
+ for (const key of Object.keys(config.budgets)) {
293
+ if (!["maxAttempts", "reviewRoundsPerWork"].includes(key)) {
294
+ throw new Error(`unknown budgets key: ${key} (known: maxAttempts, reviewRoundsPerWork)`);
295
+ }
296
+ }
297
+ if (config.budgets.maxAttempts !== undefined) {
298
+ const map = config.budgets.maxAttempts;
299
+ if (!map || typeof map !== "object" || Array.isArray(map)) {
300
+ throw new Error("budgets.maxAttempts must be a map of step -> positive integer");
301
+ }
302
+ budgets.maxAttempts = {};
303
+ for (const [step, value] of Object.entries(map)) {
304
+ if (!workflow.stepIds.has(step)) {
305
+ throw new Error(`budgets.maxAttempts.${step}: step not in workflow`);
306
+ }
307
+ if (!Number.isInteger(Number(value)) || Number(value) < 1) {
308
+ throw new Error(`budgets.maxAttempts.${step} must be a positive integer, got: ${value}`);
309
+ }
310
+ budgets.maxAttempts[step] = Number(value);
311
+ }
312
+ }
313
+ if (config.budgets.reviewRoundsPerWork !== undefined) {
314
+ const value = Number(config.budgets.reviewRoundsPerWork);
315
+ if (!Number.isInteger(value) || value < 1) {
316
+ throw new Error(`budgets.reviewRoundsPerWork must be a positive integer, got: ${config.budgets.reviewRoundsPerWork}`);
317
+ }
318
+ budgets.reviewRoundsPerWork = value;
319
+ }
320
+ }
284
321
  const cache = {};
285
322
  for (const [step, mode] of Object.entries(config.cache ?? {})) {
286
323
  if (mode !== "tree") {
@@ -317,6 +354,7 @@ function loadRunConfig(flags, command) {
317
354
  policies,
318
355
  riskPreset,
319
356
  maxAttemptsPerStep: config.maxAttemptsPerStep ?? 4,
357
+ budgets,
320
358
  stepTimeoutMs: config.stepTimeoutMs,
321
359
  allowedPaths: config.allowedPaths,
322
360
  requires: config.requires ?? [],
@@ -419,7 +457,36 @@ async function commandStart(flags) {
419
457
  if (watching) {
420
458
  console.log(`stall watcher armed (no output for ${formatMs(options.stallAfterMs)} notifies STALLED)`);
421
459
  }
422
- const result = startRun(options);
460
+ let result;
461
+ try {
462
+ result = startRun(options);
463
+ } catch (error) {
464
+ if (/another run is active/.test(error.message ?? "")) {
465
+ // Say who holds the repository and how to watch it: a pilot session
466
+ // waited 3h23m behind another work's run with nothing but the lock
467
+ // message to go on ("二十分钟了哎").
468
+ const label = repoLabelFor(options.repoRoot);
469
+ const holders = listHeldRunLocks(options.repoRoot);
470
+ if (holders.length === 0) {
471
+ console.error("blocked by: a stale active-run lock with no run holding it (a killed process?); `gc` clears locks of terminal runs, or remove .buildbeat/runtime/locks/active-run.lock after checking no driver process is alive");
472
+ }
473
+ for (const holder of holders) {
474
+ const ledgerPath = join(options.repoRoot, ".buildbeat", "runtime", "runs", holder, "events.jsonl");
475
+ let summary = "(no ledger found)";
476
+ if (existsSync(ledgerPath)) {
477
+ const ledger = EventLedger.open(ledgerPath);
478
+ const state = ledger.state;
479
+ const step = state.currentStep ? `step ${state.currentStep} attempt ${state.steps[state.currentStep]?.attempts ?? "?"}` : "between steps";
480
+ const since = ledger.events[ledger.events.length - 1]?.ts;
481
+ summary = `${state.run?.work ?? "?"} ${state.run?.status ?? "?"} ${step}${since ? `, last event ${formatMs(Date.now() - Date.parse(since))} ago` : ""}`;
482
+ }
483
+ console.error(`blocked by ${holder}: ${summary}`);
484
+ console.error(` watch it: buildbeat-v2 status --repo ${label} --run ${holder}`);
485
+ }
486
+ console.error("queue position: next after the holder(s) above stop or wait on a human (the repository allows one driving run at a time; worktrees are already isolated)");
487
+ }
488
+ throw error;
489
+ }
423
490
  const repoLabel = repoLabelFor(options.repoRoot);
424
491
  for (const run of result.superseded ?? []) {
425
492
  console.log(`superseded ${run} (was waiting on a human for the same work; now SUPERSEDED)`);
@@ -439,6 +506,15 @@ async function commandStart(flags) {
439
506
 
440
507
  async function commandResume(flags) {
441
508
  const options = loadRunConfig(flags, "resume");
509
+ if (flags.adopt !== undefined) {
510
+ const resumeAt = nextStep(options.workflow, "fix", "succeeded") ?? "verify";
511
+ const adopted = adoptCandidate(options.repoRoot, options.runId, {
512
+ sha: flags.adopt,
513
+ by: flags.by ?? "human",
514
+ resumeAt,
515
+ });
516
+ console.log(`adopted ${adopted.adopted} as candidate (${adopted.decisionRef}, answers ${adopted.transition}); resuming at ${adopted.resumeAt}`);
517
+ }
442
518
  const watching = spawnStallWatcher(options.repoRoot, options.runId, options.stallAfterMs);
443
519
  if (watching) {
444
520
  console.log(`stall watcher armed (no output for ${formatMs(options.stallAfterMs)} notifies STALLED)`);
@@ -549,6 +625,26 @@ function commandAccept(flags) {
549
625
  console.log(" note: editing the artifact after acceptance makes this acceptance stale");
550
626
  }
551
627
 
628
+ // Artifacts a policy rule requires to be accepted (artifact.accepted leaves
629
+ // anywhere under all/any/not).
630
+ function artifactsRequiredBy(rule, found = []) {
631
+ if (!rule || typeof rule !== "object") {
632
+ return found;
633
+ }
634
+ for (const [key, value] of Object.entries(rule)) {
635
+ if (key === "artifact.accepted" && value && typeof value.artifact === "string") {
636
+ found.push(value.artifact);
637
+ } else if (Array.isArray(value)) {
638
+ for (const item of value) {
639
+ artifactsRequiredBy(item, found);
640
+ }
641
+ } else if (value && typeof value === "object") {
642
+ artifactsRequiredBy(value, found);
643
+ }
644
+ }
645
+ return found;
646
+ }
647
+
552
648
  function commandDoctor(flags) {
553
649
  const options = loadRunConfig(flags, "doctor");
554
650
  console.log(`risk preset: ${options.riskPreset}`);
@@ -593,6 +689,51 @@ function commandDoctor(flags) {
593
689
  console.log("push protection: repository has no remotes (nothing to protect)");
594
690
  }
595
691
  console.log("kernel capabilities: merge/deploy/publish have no call path in the runner (invariant 20)");
692
+ const budgetLines = [];
693
+ for (const step of options.workflow.steps) {
694
+ if (!step.worker) {
695
+ continue;
696
+ }
697
+ const fromRun = options.budgets.maxAttempts?.[step.id];
698
+ const fromPreset = options.workflow.budgets?.maxAttempts?.[step.id];
699
+ const effective = fromRun ?? fromPreset ?? options.maxAttemptsPerStep;
700
+ const source = fromRun !== undefined ? "run config" : fromPreset !== undefined ? "workflow preset" : "default";
701
+ budgetLines.push(`${step.id}=${effective} (${source})`);
702
+ }
703
+ console.log(`budgets (maxAttempts per step; approving resume-<step> after exhaustion grants +1): ${budgetLines.join(", ")}`);
704
+ if (options.budgets.reviewRoundsPerWork !== undefined) {
705
+ console.log(`budgets.reviewRoundsPerWork: ${options.budgets.reviewRoundsPerWork} (counted across every run of the work, superseded ones included)`);
706
+ }
707
+ // Same preconditions start's first gate will read (real incident, twice:
708
+ // doctor passed, start stopped at build because plan.md was not mirrored
709
+ // into the repository the run was started in).
710
+ const workDir = join(options.repoRoot, "delivery", "work", options.workId);
711
+ const decisions = readJsonl(join(workDir, "decisions.jsonl"));
712
+ console.log(`work artifacts in this repository (delivery/work/${options.workId}):`);
713
+ const artifactState = {};
714
+ for (const artifact of ["intent", "plan"]) {
715
+ const status = artifactStatus(workDir, decisions, artifact);
716
+ artifactState[artifact] = status;
717
+ const label = !status.exists
718
+ ? "MISSING"
719
+ : status.stale
720
+ ? "accepted but edited since (stale)"
721
+ : status.accepted
722
+ ? `accepted${status.by ? ` by ${status.by}` : ""}`
723
+ : "draft (not accepted)";
724
+ console.log(` ${artifact}.md: ${label}`);
725
+ }
726
+ for (const policy of options.policies) {
727
+ for (const artifact of artifactsRequiredBy(policy.rule)) {
728
+ const status = artifactState[artifact] ?? artifactStatus(workDir, decisions, artifact);
729
+ if (!status.accepted) {
730
+ console.log(
731
+ ` WARNING policy ${policy.name} (${policy.type} ${policy.appliesTo}) needs an accepted ${artifact}.md; start will stop at ${policy.appliesTo}` +
732
+ (!status.exists ? " (file missing here: mirror it into this repository, then accept)" : " (accept it first)"),
733
+ );
734
+ }
735
+ }
736
+ }
596
737
  if (options.requires.length > 0) {
597
738
  console.log("environment contract (requires):");
598
739
  const check = checkRequires(options.requires);
@@ -33,6 +33,7 @@ export const EVENT_REGISTRY = {
33
33
  TRANSITION: ["from", "to", "cause"],
34
34
  FAILURE_FINGERPRINT: ["step", "command", "exitCode", "errorDigest", "diffDigest"],
35
35
  BUDGET_CONSUMED: ["kind", "amount", "remaining"],
36
+ BUDGET_EXTENDED: ["step", "amount", "maxAttempts"],
36
37
  HUMAN_REQUESTED: ["transition", "subject", "reasons"],
37
38
  DECISION_RECORDED: ["decision", "transition", "subject", "decisionRef"],
38
39
  APPROVAL_STALE: ["approvalRef", "changed"],
@@ -23,6 +23,8 @@ export function initialState() {
23
23
  fingerprints: [],
24
24
  consecutiveSameFailure: 0,
25
25
  budgets: {},
26
+ budgetExtensions: {},
27
+ workReviewGrants: 0,
26
28
  pendingHuman: null,
27
29
  decisions: [],
28
30
  approvals: [],
@@ -107,7 +109,12 @@ export function applyEvent(state, event) {
107
109
  `step ${data.step} expected attempt ${expectedAttempt}, got ${data.attempt}`,
108
110
  );
109
111
  }
110
- next.steps[data.step] = { status: "RUNNING", attempts: data.attempt, detail: null };
112
+ next.steps[data.step] = {
113
+ status: "RUNNING",
114
+ attempts: data.attempt,
115
+ detail: null,
116
+ infraAttempts: state.steps[data.step]?.infraAttempts ?? 0,
117
+ };
111
118
  next.currentStep = data.step;
112
119
  break;
113
120
  }
@@ -120,6 +127,11 @@ export function applyEvent(state, event) {
120
127
  }
121
128
  next.steps[data.step].status = data.status === "succeeded" ? "SUCCEEDED" : "FAILED";
122
129
  next.steps[data.step].detail = data.status;
130
+ if (data.infra === true) {
131
+ // A worker-infrastructure failure (backend outage, timeout, garbage
132
+ // output, exit 75) is not charged to the step's budget.
133
+ next.steps[data.step].infraAttempts = (step.infraAttempts ?? 0) + 1;
134
+ }
123
135
  next.currentStep = null;
124
136
  break;
125
137
  }
@@ -168,6 +180,19 @@ export function applyEvent(state, event) {
168
180
  next.budgets[data.kind] = { consumed: consumed + data.amount, remaining: data.remaining };
169
181
  break;
170
182
  }
183
+ case "BUDGET_EXTENDED": {
184
+ // A human approving resume-<step> after its budget ran out grants
185
+ // exactly one more attempt; the grant is a ledger fact, not a config
186
+ // edit, so the effective cap is replayable.
187
+ if (data.scope === "work") {
188
+ // Work-level review cap (budgets.reviewRoundsPerWork) lifted once by
189
+ // a human for this run.
190
+ next.workReviewGrants = (state.workReviewGrants ?? 0) + data.amount;
191
+ } else {
192
+ next.budgetExtensions[data.step] = (state.budgetExtensions?.[data.step] ?? 0) + data.amount;
193
+ }
194
+ break;
195
+ }
171
196
  case "HUMAN_REQUESTED": {
172
197
  next.pendingHuman = {
173
198
  transition: data.transition,
@@ -194,6 +219,7 @@ export function applyEvent(state, event) {
194
219
  transition: data.transition,
195
220
  subject: data.subject,
196
221
  stale: false,
222
+ ...(data.resumeAt ? { resumeAt: data.resumeAt } : {}),
197
223
  });
198
224
  }
199
225
  next.pendingHuman = null;
@@ -185,6 +185,86 @@ export function approveRun(repoRoot, runId, { by = "human", transition, ts, poli
185
185
  }
186
186
  }
187
187
 
188
+ // A human (or the driving session) supplies the candidate: the fix was made
189
+ // by hand in the run's worktree and committed, so the fixer step has nothing
190
+ // to do. The pending request is answered with the adopted commit as its
191
+ // subject and the run resumes at `resumeAt` (verify, by convention). Real
192
+ // incident: hand fixes inside a run cost a no-op fixer and an extra verify
193
+ // each time (a frontend run reached verify #5 and fix #3 for three hand
194
+ // fixes). The commit must already be the worktree HEAD: git is read back,
195
+ // the claim is not trusted.
196
+ export function adoptCandidate(repoRoot, runId, { sha, by = "human", resumeAt, ts } = {}) {
197
+ if (!sha || typeof sha !== "string" || sha.length < 7) {
198
+ throw new DecisionError("adopt requires a commit sha (at least 7 characters)");
199
+ }
200
+ if (!resumeAt) {
201
+ throw new DecisionError("adopt requires the step to resume at (resumeAt)");
202
+ }
203
+ const ledger = openWaiting(repoRoot, runId);
204
+ const pending = ledger.state.pendingHuman;
205
+ if (pending.kind === "final-decision") {
206
+ throw new DecisionError("adopt is for a run waiting before fix/verify, not at the merge decision");
207
+ }
208
+ acquireLock(repoRoot, runId);
209
+ try {
210
+ const bound = ledger.state.workspaces[runId];
211
+ const worktreePath = bound ? resolveRepoRef(repoRoot, bound.worktreePath) : null;
212
+ if (!bound || !existsSync(worktreePath)) {
213
+ throw new DecisionError(`worktree missing for ${runId}; cannot adopt a candidate`);
214
+ }
215
+ const tree = readback(worktreePath);
216
+ if (tree.dirty) {
217
+ throw new DecisionError("worktree is dirty; commit the hand fix before adopting it");
218
+ }
219
+ if (!tree.head.startsWith(sha)) {
220
+ throw new DecisionError(`worktree HEAD is ${tree.head}, not ${sha}; adopt what git reads back`);
221
+ }
222
+ const when = ts ?? new Date().toISOString();
223
+ const lastEvidence = ledger.state.evidence[ledger.state.evidence.length - 1];
224
+ if (bound.candidate !== tree.head) {
225
+ ledger.append({
226
+ type: "CANDIDATE_PINNED",
227
+ actor: { kind: "human", id: by },
228
+ ts: when,
229
+ data: { workspaceId: runId, base: bound.base, candidate: tree.head, adopted: true },
230
+ });
231
+ }
232
+ const subject = {
233
+ candidate: tree.head,
234
+ planDigest: ledger.state.run.planDigest,
235
+ evidenceDigest: lastEvidence?.digest ?? "UNVERIFIED",
236
+ };
237
+ const decisionRef = `D-${runId}-${ledger.state.decisions.length + 1}`;
238
+ ledger.append({
239
+ type: "DECISION_RECORDED",
240
+ actor: { kind: "human", id: by },
241
+ ts: when,
242
+ data: {
243
+ decision: "approved",
244
+ transition: pending.transition,
245
+ subject,
246
+ decisionRef,
247
+ adopted: tree.head,
248
+ resumeAt,
249
+ },
250
+ });
251
+ recordDecisionFile(repoRoot, ledger.state.run.work, {
252
+ ts: when,
253
+ run: runId,
254
+ decisionRef,
255
+ decision: "approved",
256
+ transition: pending.transition,
257
+ subject,
258
+ by,
259
+ adopted: tree.head,
260
+ resumeAt,
261
+ });
262
+ return { adopted: tree.head, decisionRef, transition: pending.transition, resumeAt, state: ledger.state };
263
+ } finally {
264
+ releaseLock(repoRoot, runId);
265
+ }
266
+ }
267
+
188
268
  // Accepts a work artifact (plan, intent, spec) by binding a decision to the
189
269
  // file's current digest in the Git plane. If the file changes afterwards,
190
270
  // artifact.accepted evaluates false again — acceptance cannot go stale
@@ -26,6 +26,7 @@ import {
26
26
  releaseLock,
27
27
  } from "../workspace/workspace-manager.js";
28
28
  import { writeRunRecord } from "./run-record.js";
29
+ import { computeWorkCost } from "./work-cost.js";
29
30
  import { assertRequires } from "./env-contract.js";
30
31
  import { materialisePrompt } from "./envelope.js";
31
32
  import { cacheKey, findReusableEvidence, lastReviewedCandidate, treeHash } from "./cache.js";
@@ -142,8 +143,18 @@ function makeContext(options, ledger, workspace) {
142
143
  ledger,
143
144
  workspace,
144
145
  };
146
+ // Effective cap per step: run config `budgets:` beats the preset, the
147
+ // preset beats the global default, and every BUDGET_EXTENDED a human
148
+ // granted on this ledger adds to it. Real incident: the preset's two
149
+ // review rounds could not be raised from the run config, and approving
150
+ // resume-review re-asked the same question forever.
151
+ context.runBudgets = options.budgets ?? {};
145
152
  context.maxAttemptsFor = (step) =>
146
- workflow.budgets?.maxAttempts?.[step] ?? maxAttemptsPerStep;
153
+ (context.runBudgets.maxAttempts?.[step] ??
154
+ workflow.budgets?.maxAttempts?.[step] ??
155
+ maxAttemptsPerStep) +
156
+ (ledger.state.budgetExtensions?.[step] ?? 0) +
157
+ (ledger.state.steps[step]?.infraAttempts ?? 0);
147
158
  context.policies = options.policies ?? [];
148
159
  context.allowedPaths = options.allowedPaths ?? null;
149
160
  context.reviewTriage = options.reviewTriage ?? null;
@@ -240,6 +251,7 @@ function settleOutcome(context, step, outcome, tree, exec) {
240
251
  if ((ledger.state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)) {
241
252
  context.waitHuman(`resume-${step}`, [
242
253
  `budget exhausted: ${step} failed its final attempt (maxAttempts=${context.maxAttemptsFor(step)}); not routing to fix`,
254
+ `approving resume-${step} grants one more attempt; rejecting ends the run`,
243
255
  ]);
244
256
  return null;
245
257
  }
@@ -266,13 +278,13 @@ function settleOutcome(context, step, outcome, tree, exec) {
266
278
  },
267
279
  });
268
280
  if (!to) {
269
- ledger.append({
270
- type: "RUN_TERMINAL",
271
- actor: KERNEL,
272
- ts: now(),
273
- data: { status: "FAILED", reason: `no transition for (${step}, ${outcome})` },
274
- });
275
- writeRunRecord({ repoRoot: context.repoRoot, ledger, ts: now() });
281
+ // A workflow without an edge for this outcome is not a verdict on the
282
+ // candidate; the person decides whether to rerun the step or end the
283
+ // run. Terminal FAILED here used to kill runs whose reviewer had merely
284
+ // errored out.
285
+ context.waitHuman(`resume-${step}`, [
286
+ `no transition for (${step}, ${outcome}); approve resume-${step} to rerun the step, reject to end the run`,
287
+ ]);
276
288
  return null;
277
289
  }
278
290
  ledger.append({
@@ -320,6 +332,30 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
320
332
  return;
321
333
  }
322
334
 
335
+ // Work-level review cap (iteration 09): review rounds are counted across
336
+ // every run of the work, superseded ones included, so "one run per
337
+ // round" cannot slip past the per-run budget. Reaching the cap is a
338
+ // human decision (review once more, or merge/close as-is), not a stop.
339
+ const workCap = context.runBudgets.reviewRoundsPerWork;
340
+ if (workCap !== undefined && (stepDef.worker === "reviewer" || step === "review")) {
341
+ const prior = computeWorkCost(context.repoRoot, ledger.state.run.work, {
342
+ excludeRun: ledger.state.run.id,
343
+ });
344
+ const rounds = prior.reviewRounds + (ledger.state.steps[step]?.attempts ?? 0);
345
+ const allowed = workCap + (ledger.state.workReviewGrants ?? 0);
346
+ if (rounds >= allowed) {
347
+ context.waitHuman(
348
+ `enter-${step}`,
349
+ [
350
+ `work review cap reached: ${rounds} review round(s) across ${prior.runs + 1} run(s) of ${ledger.state.run.work} (budgets.reviewRoundsPerWork=${workCap})`,
351
+ `approve enter-${step} to review once more, or reject and merge/close the work on the evidence you have`,
352
+ ],
353
+ "work-review-cap",
354
+ );
355
+ return;
356
+ }
357
+ }
358
+
323
359
  const preGate = runPolicyGate(context, "pre", step);
324
360
  if (preGate.action === "block") {
325
361
  ledger.append({
@@ -341,6 +377,7 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
341
377
  if (attempt > maxAttempts) {
342
378
  context.waitHuman(`resume-${step}`, [
343
379
  `budget exhausted: ${step} would exceed maxAttempts=${maxAttempts}`,
380
+ `approving resume-${step} grants one more attempt; rejecting ends the run`,
344
381
  ]);
345
382
  return;
346
383
  }
@@ -531,18 +568,53 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
531
568
  } else {
532
569
  stepStatus = "succeeded";
533
570
  }
571
+ // Infrastructure failure vs candidate failure. A timeout, a crash,
572
+ // garbage output or the worker's own "environment unavailable" signal
573
+ // (exit 75, EX_TEMPFAIL) says nothing about the candidate: no failure
574
+ // fingerprint, no fixer, the attempt is refunded, and a human decides
575
+ // when the backend is back. Real incidents: a worker backend outage
576
+ // (review exit 97) and non-JSON reviewer output killed five runs in two
577
+ // days as "no transition for (review, failed)"; PATH, port and host-load
578
+ // verify failures dispatched fixers five times.
579
+ const infra =
580
+ stepStatus === "timeout" ||
581
+ stepStatus === "crashed" ||
582
+ stepStatus === "invalid-output" ||
583
+ (stepStatus === "failed" && exec.exitCode === 75);
534
584
  ledger.append({
535
585
  type: "STEP_FINISHED",
536
586
  actor: KERNEL,
537
587
  ts: now(),
538
- data: { step, attempt, status: stepStatus, exitCode: exec.exitCode },
588
+ data: { step, attempt, status: stepStatus, exitCode: exec.exitCode, ...(infra ? { infra: true } : {}) },
539
589
  });
540
590
  ledger.append({
541
591
  type: "BUDGET_CONSUMED",
542
592
  actor: KERNEL,
543
593
  ts: now(),
544
- data: { kind: "attempts", amount: 1, remaining: maxAttempts - attempt },
594
+ data: {
595
+ kind: "attempts",
596
+ amount: infra ? 0 : 1,
597
+ remaining: context.maxAttemptsFor(step) - attempt,
598
+ },
545
599
  });
600
+ if (infra) {
601
+ const cause =
602
+ stepStatus === "failed"
603
+ ? "exit 75 (worker reports its environment unavailable)"
604
+ : stepStatus === "invalid-output"
605
+ ? "output is not a worker envelope"
606
+ : stepStatus;
607
+ context.waitHuman(
608
+ `resume-${step}`,
609
+ [
610
+ `worker infrastructure failure at ${step}: ${cause}; not a candidate defect, attempt not charged`,
611
+ ...(tree.dirty ? [`the failed worker left the worktree dirty; inspect before rerunning`] : []),
612
+ `approve resume-${step} to rerun once the backend/environment is back; reject to end the run`,
613
+ ],
614
+ "infra",
615
+ );
616
+ return;
617
+ }
546
618
 
547
619
  let blockingFindings = [];
548
620
  if (envelope?.findings) {
@@ -920,12 +992,51 @@ export function resumeRun(options) {
920
992
  });
921
993
  return { runId, ledgerPath, state: ledger.state, resumed: true, stale: true, reason: null };
922
994
  }
923
- const step = resumeStepFromTransition(approval.transition);
995
+ // An adopted candidate names where to resume (verify, by convention):
996
+ // the fixer step the request was waiting on has nothing left to do.
997
+ const step = approval.resumeAt ?? resumeStepFromTransition(approval.transition);
924
998
  if (!step || !context.workflow.stepIds.has(step)) {
925
999
  throw new OrchestratorError(
926
1000
  `cannot derive a resume step from approved transition ${approval.transition}`,
927
1001
  );
928
1002
  }
1003
+ // An approved resume-<step> on an exhausted budget is the human saying
1004
+ // "one more"; record the grant before driving or the same request
1005
+ // comes straight back (the pilot's app-login runs ended CANCELLED
1006
+ // with their candidates in production because of exactly that).
1007
+ const requestKind = [...ledger.events]
1008
+ .reverse()
1009
+ .find((event) => event.type === "HUMAN_REQUESTED" && event.data.transition === approval.transition)
1010
+ ?.data.kind;
1011
+ if (requestKind === "work-review-cap") {
1012
+ ledger.append({
1013
+ type: "BUDGET_EXTENDED",
1014
+ actor: KERNEL,
1015
+ ts: now(),
1016
+ data: {
1017
+ step,
1018
+ amount: 1,
1019
+ scope: "work",
1020
+ maxAttempts: (context.runBudgets.reviewRoundsPerWork ?? 0) + (state.workReviewGrants ?? 0) + 1,
1021
+ approvalRef: approval.decisionRef,
1022
+ },
1023
+ });
1024
+ } else if (
1025
+ approval.transition.startsWith("resume-") &&
1026
+ (state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)
1027
+ ) {
1028
+ ledger.append({
1029
+ type: "BUDGET_EXTENDED",
1030
+ actor: KERNEL,
1031
+ ts: now(),
1032
+ data: {
1033
+ step,
1034
+ amount: 1,
1035
+ maxAttempts: context.maxAttemptsFor(step) + 1,
1036
+ approvalRef: approval.decisionRef,
1037
+ },
1038
+ });
1039
+ }
929
1040
  ledger.append({ type: "RUN_STARTED", actor: KERNEL, ts: now(), data: {} });
930
1041
  drive(context, step, { skipBoundaryOnce: true });
931
1042
  return { runId, ledgerPath, state: ledger.state, resumed: true, reason: null };