@haiyangbg/buildbeat 3.3.1 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.en.md +20 -124
- package/README.md +20 -124
- package/SKILL.md +73 -84
- package/docs/CAPABILITY-MATRIX.md +30 -39
- package/docs/MIGRATION.md +93 -0
- package/docs/README.md +8 -37
- package/docs/RELEASING.md +4 -3
- package/docs/v2/RFC-0001-product-definition.md +3 -0
- package/docs/v2/RFC-0002-domain-model.md +3 -0
- package/docs/v2/RFC-0003-workflow-policy.md +3 -0
- package/docs/v2/SPEC-0001-events-v1.md +3 -0
- package/docs/v2/guide/00-how-to-talk.en.md +12 -56
- package/docs/v2/guide/00-how-to-talk.md +12 -56
- package/docs/v2/guide/01-quickstart.en.md +42 -111
- package/docs/v2/guide/01-quickstart.md +42 -111
- package/docs/v2/guide/02-workflow-guide.en.md +13 -123
- package/docs/v2/guide/02-workflow-guide.md +14 -122
- package/docs/v2/guide/03-policy-guide.en.md +6 -50
- package/docs/v2/guide/03-policy-guide.md +6 -50
- package/docs/v2/guide/04-adapter-guide.en.md +7 -60
- package/docs/v2/guide/04-adapter-guide.md +7 -60
- package/docs/v2/guide/05-worker-contract.en.md +6 -53
- package/docs/v2/guide/05-worker-contract.md +7 -52
- package/docs/v2/guide/06-evidence-guide.en.md +8 -44
- package/docs/v2/guide/06-evidence-guide.md +9 -43
- package/docs/v2/guide/07-approval-guide.en.md +8 -110
- package/docs/v2/guide/07-approval-guide.md +8 -121
- package/docs/v2/guide/09-security-boundaries.en.md +13 -37
- package/docs/v2/guide/09-security-boundaries.md +13 -37
- package/docs/v2/guide/10-recovery.en.md +12 -96
- package/docs/v2/guide/10-recovery.md +12 -96
- package/docs/v2/guide/11-session-handoff.en.md +10 -75
- package/docs/v2/guide/11-session-handoff.md +10 -75
- package/docs/v2/guide/12-without-runtime.en.md +46 -0
- package/docs/v2/guide/12-without-runtime.md +46 -0
- package/docs/v2/guide/README.en.md +16 -36
- package/docs/v2/guide/README.md +16 -36
- package/example/AGENTS.md +15 -69
- package/example/README.md +8 -23
- package/example/delivery/envelope/prompts/builder.md +3 -1
- package/example/delivery/envelope/prompts/fixer.md +3 -1
- package/example/delivery/envelope/prompts/reviewer.md +3 -1
- package/example/delivery/work/WORK-EXPORT-DATE-FILTER/run-config.yaml +1 -1
- package/example//346/214/207/346/214/245/345/217/260.md +8 -40
- package/package.json +2 -1
- package/src/v2/adapters/notification-payload.js +40 -0
- package/src/v2/adapters/shell.js +3 -1
- package/src/v2/cli/run-config-check.js +129 -45
- package/src/v2/cli/run.js +853 -254
- package/src/v2/engine/reducer.js +2 -0
- package/src/v2/engine/risk-preset.js +14 -47
- package/src/v2/engine/workflow.js +177 -185
- package/src/v2/evidence/collector.js +12 -1
- package/src/v2/evidence/image.js +346 -0
- package/src/v2/policy/policy.js +219 -286
- package/src/v2/presets/software-delivery.yaml +1 -16
- package/src/v2/runtime/cache.js +22 -1
- package/src/v2/runtime/decisions.js +185 -31
- package/src/v2/runtime/findings.js +4 -1
- package/src/v2/runtime/notify.js +42 -36
- package/src/v2/runtime/orchestrator.js +811 -297
- package/src/v2/runtime/overview-repos.js +79 -0
- package/src/v2/runtime/overview.js +279 -47
- package/src/v2/runtime/release.js +233 -0
- package/src/v2/runtime/run-record.js +1 -0
- package/src/v2/workspace/workspace-manager.js +8 -0
- package/templates/v2/AGENTS.md +20 -73
- package/templates/v2/BUILDBEAT.md +2 -13
- package/templates/v2/envelope/prompts/builder.md +3 -1
- package/templates/v2/envelope/prompts/fixer.md +3 -1
- package/templates/v2/envelope/prompts/reviewer.md +3 -1
- package/templates/v2/run-config.example.yaml +9 -7
- package/templates/v2/work.example.md +16 -0
- package/templates/v2//346/214/207/346/214/245/345/217/260.md +8 -40
- package/docs/v2/skill/01-principles.md +0 -26
- package/docs/v2/skill/02-project-layout.md +0 -31
- package/docs/v2/skill/03-collaboration-rules.md +0 -45
- package/docs/v2/skill/04-rhythm-and-rituals.md +0 -104
- package/docs/v2/skill/05-red-lines.md +0 -13
- package/docs/v2/skill/06-bootstrap-and-takeover.md +0 -84
- package/docs/v2/skill/07-templates-and-lessons.md +0 -24
- package/example/.buildbeat/observe.yaml +0 -31
- package/src/v2/observe/observe-config.js +0 -194
- package/src/v2/observe/observe-reducer.js +0 -117
- package/src/v2/observe/observe.js +0 -427
- package/src/v2/presets/observe.yaml +0 -45
- package/src/v2/presets/policies/ui-render-gate.yaml +0 -13
- package/src/v2/presets/release-readback.yaml +0 -36
- package/src/v2/presets/risk/controlled.yaml +0 -39
- package/src/v2/presets/risk/fast.yaml +0 -19
- package/src/v2/presets/risk/release.yaml +0 -21
- package/src/v2/presets/risk/standard.yaml +0 -28
- package/templates/ARCHITECTURE.md +0 -50
- package/templates/contracts/PROTOCOL.md +0 -28
- package/templates/pm/adr/ADR-0000-template.md +0 -25
- package/templates/pm/adr/README.md +0 -15
- package/templates/pm/decisions.md +0 -11
- package/templates/standards/CODE.md +0 -23
- package/templates/standards/DESIGN.md +0 -36
- package/templates/standards/REVIEW.md +0 -20
- package/templates/standards/STACK.md +0 -31
|
@@ -11,17 +11,26 @@
|
|
|
11
11
|
// anything changed goes APPROVAL_STALE and back to WAITING_HUMAN.
|
|
12
12
|
|
|
13
13
|
import { createHash } from "node:crypto";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
existsSync,
|
|
16
|
+
mkdirSync,
|
|
17
|
+
readdirSync,
|
|
18
|
+
readFileSync,
|
|
19
|
+
rmSync,
|
|
20
|
+
writeFileSync,
|
|
21
|
+
} from "node:fs";
|
|
15
22
|
import { join } from "node:path";
|
|
16
23
|
|
|
17
24
|
import { nextStep } from "../engine/workflow.js";
|
|
18
25
|
import { collectCommandEvidence } from "../evidence/collector.js";
|
|
19
|
-
import {
|
|
26
|
+
import { checkScreenshotFile, screenshotFormat } from "../evidence/image.js";
|
|
27
|
+
import { currentScreenshots, evaluatePolicies } from "../policy/policy.js";
|
|
20
28
|
import { EventLedger, canonicalJson } from "../storage/event-ledger.js";
|
|
21
29
|
import {
|
|
22
30
|
acquireLock,
|
|
23
31
|
createWorkspace,
|
|
24
32
|
describeLockOwner,
|
|
33
|
+
discardWorkspace,
|
|
25
34
|
listChangedPaths,
|
|
26
35
|
liveParallelMarkers,
|
|
27
36
|
readback,
|
|
@@ -32,7 +41,13 @@ import { writeRunRecord } from "./run-record.js";
|
|
|
32
41
|
import { computeWorkCost } from "./work-cost.js";
|
|
33
42
|
import { assertRequires } from "./env-contract.js";
|
|
34
43
|
import { materialisePrompt } from "./envelope.js";
|
|
35
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
cacheKey,
|
|
46
|
+
findReusableEvidence,
|
|
47
|
+
lastReviewedCandidate,
|
|
48
|
+
linkedScreenshots,
|
|
49
|
+
treeHash,
|
|
50
|
+
} from "./cache.js";
|
|
36
51
|
import {
|
|
37
52
|
buildAnchor,
|
|
38
53
|
fingerprintFinding,
|
|
@@ -106,7 +121,12 @@ function holdRunLock(repoRoot, runId, fn) {
|
|
|
106
121
|
}
|
|
107
122
|
}
|
|
108
123
|
|
|
109
|
-
function withRunLocks(
|
|
124
|
+
function withRunLocks(
|
|
125
|
+
repoRoot,
|
|
126
|
+
runId,
|
|
127
|
+
fn,
|
|
128
|
+
{ workId = null, parallel = false } = {},
|
|
129
|
+
) {
|
|
110
130
|
if (!parallel) {
|
|
111
131
|
lockActive(repoRoot);
|
|
112
132
|
try {
|
|
@@ -114,8 +134,14 @@ function withRunLocks(repoRoot, runId, fn, { workId = null, parallel = false } =
|
|
|
114
134
|
if (running.length > 0) {
|
|
115
135
|
throw new OrchestratorError(
|
|
116
136
|
`another run is active in this repository (parallel run(s) ${running
|
|
117
|
-
.map((marker) =>
|
|
118
|
-
|
|
137
|
+
.map((marker) =>
|
|
138
|
+
marker.owner
|
|
139
|
+
? `${marker.run}, ${describeLockOwner(marker.owner)}`
|
|
140
|
+
: marker.run,
|
|
141
|
+
)
|
|
142
|
+
.join(
|
|
143
|
+
"; ",
|
|
144
|
+
)}); this run is exclusive (set parallel: true in its run config to drive alongside other works)`,
|
|
119
145
|
);
|
|
120
146
|
}
|
|
121
147
|
return holdRunLock(repoRoot, runId, fn);
|
|
@@ -192,7 +218,10 @@ export function parseEnvelope(raw) {
|
|
|
192
218
|
!/^P[0-3]$/.test(finding.severity ?? "") ||
|
|
193
219
|
typeof finding.summary !== "string"
|
|
194
220
|
) {
|
|
195
|
-
return {
|
|
221
|
+
return {
|
|
222
|
+
envelope: null,
|
|
223
|
+
error: "each finding needs severity P0-P3 and a summary",
|
|
224
|
+
};
|
|
196
225
|
}
|
|
197
226
|
}
|
|
198
227
|
}
|
|
@@ -230,15 +259,19 @@ function makeContext(options, ledger, workspace) {
|
|
|
230
259
|
// A review step without an explicit cap takes the work-level cap as its run
|
|
231
260
|
// cap, so review rounds are counted in one unit: the work.
|
|
232
261
|
context.runBudgets = options.budgets ?? {};
|
|
233
|
-
context.reviewRoundsPerWork =
|
|
262
|
+
context.reviewRoundsPerWork =
|
|
263
|
+
context.runBudgets.reviewRoundsPerWork ?? DEFAULT_REVIEW_ROUNDS_PER_WORK;
|
|
234
264
|
context.budgetLimitFor = (step) =>
|
|
235
265
|
(context.runBudgets.maxAttempts?.[step] ??
|
|
236
266
|
workflow.budgets?.maxAttempts?.[step] ??
|
|
237
|
-
(isReviewStep(
|
|
267
|
+
(isReviewStep(
|
|
268
|
+
step,
|
|
269
|
+
workflow.steps.find((item) => item.id === step),
|
|
270
|
+
)
|
|
238
271
|
? context.reviewRoundsPerWork
|
|
239
|
-
: maxAttemptsPerStep)) +
|
|
240
|
-
|
|
241
|
-
|
|
272
|
+
: maxAttemptsPerStep)) + (ledger.state.budgetExtensions?.[step] ?? 0);
|
|
273
|
+
context.maxAttemptsFor = (step) =>
|
|
274
|
+
context.budgetLimitFor(step) +
|
|
242
275
|
(ledger.state.steps[step]?.infraAttempts ?? 0) +
|
|
243
276
|
(ledger.state.steps[step]?.freeAttempts ?? 0);
|
|
244
277
|
// Refunds must not move this ceiling: otherwise a success-only loop has
|
|
@@ -252,16 +285,23 @@ function makeContext(options, ledger, workspace) {
|
|
|
252
285
|
context.redact = options.redact ?? [];
|
|
253
286
|
context.adapterConfigs = options.adapterConfigs ?? {};
|
|
254
287
|
context.policyCtx = () => ({
|
|
288
|
+
repoRoot,
|
|
255
289
|
state: ledger.state,
|
|
256
|
-
candidate:
|
|
290
|
+
candidate:
|
|
291
|
+
ledger.state.workspaces[workspace.workspaceId]?.candidate ?? null,
|
|
257
292
|
workDir: join(repoRoot, "delivery", "work", ledger.state.run?.work ?? ""),
|
|
258
293
|
worktreePath: workspace.worktreePath,
|
|
259
294
|
readWorktree: () =>
|
|
260
|
-
existsSync(workspace.worktreePath)
|
|
295
|
+
existsSync(workspace.worktreePath)
|
|
296
|
+
? readback(workspace.worktreePath)
|
|
297
|
+
: null,
|
|
261
298
|
});
|
|
262
299
|
context.subjectNow = () => {
|
|
263
|
-
const candidate =
|
|
264
|
-
|
|
300
|
+
const candidate =
|
|
301
|
+
ledger.state.workspaces[workspace.workspaceId]?.candidate ??
|
|
302
|
+
workspace.base;
|
|
303
|
+
const lastEvidence =
|
|
304
|
+
ledger.state.evidence[ledger.state.evidence.length - 1];
|
|
265
305
|
return {
|
|
266
306
|
candidate,
|
|
267
307
|
planDigest: ledger.state.run?.planDigest ?? "UNVERIFIED",
|
|
@@ -273,7 +313,13 @@ function makeContext(options, ledger, workspace) {
|
|
|
273
313
|
type: "HUMAN_REQUESTED",
|
|
274
314
|
actor: KERNEL,
|
|
275
315
|
ts: context.now(),
|
|
276
|
-
data: {
|
|
316
|
+
data: {
|
|
317
|
+
transition,
|
|
318
|
+
subject: context.subjectNow(),
|
|
319
|
+
reasons,
|
|
320
|
+
kind,
|
|
321
|
+
...(grants.length ? { grants } : {}),
|
|
322
|
+
},
|
|
277
323
|
});
|
|
278
324
|
};
|
|
279
325
|
return context;
|
|
@@ -283,17 +329,24 @@ function makeContext(options, ledger, workspace) {
|
|
|
283
329
|
function budgetUsage(context, step) {
|
|
284
330
|
const state = context.ledger.state.steps[step];
|
|
285
331
|
const attempts = state?.attempts ?? 0;
|
|
286
|
-
const used =
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
event
|
|
332
|
+
const used =
|
|
333
|
+
attempts - (state?.infraAttempts ?? 0) - (state?.freeAttempts ?? 0);
|
|
334
|
+
const failures = context.ledger.events.filter(
|
|
335
|
+
(event) =>
|
|
336
|
+
event.type === "STEP_FINISHED" &&
|
|
337
|
+
event.data.step === step &&
|
|
338
|
+
event.data.status !== "succeeded" &&
|
|
339
|
+
event.data.infra !== true,
|
|
340
|
+
).length;
|
|
290
341
|
return { attempts, used, failures, limit: context.budgetLimitFor(step) };
|
|
291
342
|
}
|
|
292
343
|
|
|
293
344
|
function budgetReasons(context, step, safeguard = false) {
|
|
294
345
|
const { attempts, used, failures, limit } = budgetUsage(context, step);
|
|
295
|
-
const charging = context.workflow.steps.find((item) => item.id === step)
|
|
296
|
-
|
|
346
|
+
const charging = context.workflow.steps.find((item) => item.id === step)
|
|
347
|
+
?.readonly
|
|
348
|
+
? "each round is charged"
|
|
349
|
+
: "successful attempts are not charged";
|
|
297
350
|
return [
|
|
298
351
|
safeguard
|
|
299
352
|
? `${step} budget exhausted (runaway safeguard): ${attempts}/${context.totalAttemptsFor(step)} total attempt(s), ${failures} real failure(s)`
|
|
@@ -306,10 +359,15 @@ function workReviewBudget(context, step) {
|
|
|
306
359
|
const stepDef = context.workflow.steps.find((item) => item.id === step);
|
|
307
360
|
const cap = context.reviewRoundsPerWork;
|
|
308
361
|
if (!isReviewStep(step, stepDef)) return null;
|
|
309
|
-
const prior = computeWorkCost(
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
362
|
+
const prior = computeWorkCost(
|
|
363
|
+
context.repoRoot,
|
|
364
|
+
context.ledger.state.run.work,
|
|
365
|
+
{
|
|
366
|
+
excludeRun: context.ledger.state.run.id,
|
|
367
|
+
},
|
|
368
|
+
);
|
|
369
|
+
const rounds =
|
|
370
|
+
prior.reviewRounds + (context.ledger.state.steps[step]?.attempts ?? 0);
|
|
313
371
|
const allowed = cap + (context.ledger.state.workReviewGrants ?? 0);
|
|
314
372
|
return { rounds, allowed, exhausted: rounds >= allowed };
|
|
315
373
|
}
|
|
@@ -318,11 +376,16 @@ function workReviewBudget(context, step) {
|
|
|
318
376
|
// at stop time, so one approval lifts both layers (run and work) at once.
|
|
319
377
|
function budgetGrants(context, step) {
|
|
320
378
|
const grants = [];
|
|
321
|
-
if (
|
|
322
|
-
|
|
379
|
+
if (
|
|
380
|
+
(context.ledger.state.steps[step]?.attempts ?? 0) >=
|
|
381
|
+
context.maxAttemptsFor(step) ||
|
|
382
|
+
(context.ledger.state.steps[step]?.attempts ?? 0) >=
|
|
383
|
+
context.totalAttemptsFor(step)
|
|
384
|
+
) {
|
|
323
385
|
grants.push({ step, scope: "run" });
|
|
324
386
|
}
|
|
325
|
-
if (workReviewBudget(context, step)?.exhausted)
|
|
387
|
+
if (workReviewBudget(context, step)?.exhausted)
|
|
388
|
+
grants.push({ step, scope: "work" });
|
|
326
389
|
return grants;
|
|
327
390
|
}
|
|
328
391
|
|
|
@@ -340,25 +403,44 @@ function reviewRoundAttempt(ref, step) {
|
|
|
340
403
|
// converge; a shrinking set of new findings keeps going without asking
|
|
341
404
|
// anyone. Returns the stop reason, or null.
|
|
342
405
|
function reviewNotConverging(context, step, blockingFindings) {
|
|
343
|
-
const rounds = context.ledger.state.evidence.filter(
|
|
344
|
-
|
|
406
|
+
const rounds = context.ledger.state.evidence.filter(
|
|
407
|
+
(item) =>
|
|
408
|
+
item.kind === "review" &&
|
|
409
|
+
reviewRoundAttempt(item.ref ?? "", step) !== null,
|
|
410
|
+
);
|
|
345
411
|
if (rounds.length < 2) return null;
|
|
346
412
|
const adjudicated = latestAdjudications(
|
|
347
413
|
readFindingsAccount(context.repoRoot, context.ledger.state.run.work),
|
|
348
414
|
);
|
|
349
|
-
const blocking = (round) =>
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
415
|
+
const blocking = (round) =>
|
|
416
|
+
(round.findings ?? [])
|
|
417
|
+
.filter(
|
|
418
|
+
(finding) => finding.severity === "P0" || finding.severity === "P1",
|
|
419
|
+
)
|
|
420
|
+
.filter(
|
|
421
|
+
(finding) =>
|
|
422
|
+
adjudicated.get(fingerprintFinding(finding))?.action !== "dismiss",
|
|
423
|
+
);
|
|
424
|
+
const earlier = rounds
|
|
425
|
+
.slice(0, -1)
|
|
426
|
+
.flatMap((round, index) =>
|
|
427
|
+
blocking(round).map((finding) => ({ finding, round: index + 1 })),
|
|
428
|
+
);
|
|
429
|
+
const lastCount = new Set(blocking(rounds.at(-2)).map(fingerprintFinding))
|
|
430
|
+
.size;
|
|
431
|
+
const current = [
|
|
432
|
+
...new Map(
|
|
433
|
+
blockingFindings.map((finding) => [fingerprintFinding(finding), finding]),
|
|
434
|
+
).values(),
|
|
435
|
+
];
|
|
356
436
|
const repeated = current.flatMap((finding) => {
|
|
357
437
|
const match = earlier.find((item) => sameIssue(finding, item.finding));
|
|
358
438
|
if (!match) return [];
|
|
359
439
|
const now = fingerprintFinding(finding);
|
|
360
440
|
const before = fingerprintFinding(match.finding);
|
|
361
|
-
return [
|
|
441
|
+
return [
|
|
442
|
+
now === before ? now : `${now} restates round ${match.round} ${before}`,
|
|
443
|
+
];
|
|
362
444
|
});
|
|
363
445
|
if (repeated.length > 0) {
|
|
364
446
|
return `${step} is not converging: ${repeated.length} blocking finding(s) came back after fix (${repeated.join(", ")}); approving runs fix + re-verify + one more review round, dismissing a finding stops it blocking`;
|
|
@@ -373,7 +455,11 @@ function reviewNotConverging(context, step, blockingFindings) {
|
|
|
373
455
|
// verdict as a POLICY_EVALUATED event, and reports what the kernel must do.
|
|
374
456
|
// ADVISORY failures are recorded but never gate (doctor reports the gap).
|
|
375
457
|
function runPolicyGate(context, type, appliesTo) {
|
|
376
|
-
const rows = evaluatePolicies(
|
|
458
|
+
const rows = evaluatePolicies(
|
|
459
|
+
context.policies,
|
|
460
|
+
{ type, appliesTo },
|
|
461
|
+
context.policyCtx(),
|
|
462
|
+
);
|
|
377
463
|
for (const row of rows) {
|
|
378
464
|
context.ledger.append({
|
|
379
465
|
type: "POLICY_EVALUATED",
|
|
@@ -388,7 +474,9 @@ function runPolicyGate(context, type, appliesTo) {
|
|
|
388
474
|
},
|
|
389
475
|
});
|
|
390
476
|
}
|
|
391
|
-
const enforced = rows.filter(
|
|
477
|
+
const enforced = rows.filter(
|
|
478
|
+
(row) => row.enforcement !== "ADVISORY" && row.result !== "PASS",
|
|
479
|
+
);
|
|
392
480
|
if (enforced.length === 0) {
|
|
393
481
|
return { action: "continue", rows };
|
|
394
482
|
}
|
|
@@ -427,8 +515,15 @@ function settleOutcome(context, step, outcome, tree, exec) {
|
|
|
427
515
|
}
|
|
428
516
|
// A step that failed its final attempt can never run again, so routing
|
|
429
517
|
// to fix would spend a worker on a candidate nothing can verify.
|
|
430
|
-
if (
|
|
431
|
-
|
|
518
|
+
if (
|
|
519
|
+
(ledger.state.steps[step]?.attempts ?? 0) >= context.maxAttemptsFor(step)
|
|
520
|
+
) {
|
|
521
|
+
context.waitHuman(
|
|
522
|
+
`resume-${step}`,
|
|
523
|
+
budgetReasons(context, step),
|
|
524
|
+
"budget",
|
|
525
|
+
budgetGrants(context, step),
|
|
526
|
+
);
|
|
432
527
|
return null;
|
|
433
528
|
}
|
|
434
529
|
}
|
|
@@ -450,7 +545,9 @@ function settleOutcome(context, step, outcome, tree, exec) {
|
|
|
450
545
|
phase: "transition",
|
|
451
546
|
result,
|
|
452
547
|
enforcement: "LOCAL_ENFORCED",
|
|
453
|
-
reason: to
|
|
548
|
+
reason: to
|
|
549
|
+
? `(${step}, ${outcome}) -> ${to}`
|
|
550
|
+
: `no transition for (${step}, ${outcome})`,
|
|
454
551
|
},
|
|
455
552
|
});
|
|
456
553
|
if (!to) {
|
|
@@ -474,8 +571,17 @@ function settleOutcome(context, step, outcome, tree, exec) {
|
|
|
474
571
|
actor: KERNEL,
|
|
475
572
|
ts: now(),
|
|
476
573
|
data: {
|
|
477
|
-
resumePoint: {
|
|
478
|
-
|
|
574
|
+
resumePoint: {
|
|
575
|
+
step: to,
|
|
576
|
+
attempt: (ledger.state.steps[to]?.attempts ?? 0) + 1,
|
|
577
|
+
},
|
|
578
|
+
workspaceStates: [
|
|
579
|
+
{
|
|
580
|
+
workspaceId: workspace.workspaceId,
|
|
581
|
+
head: tree.head,
|
|
582
|
+
dirty: tree.dirty,
|
|
583
|
+
},
|
|
584
|
+
],
|
|
479
585
|
},
|
|
480
586
|
});
|
|
481
587
|
return to;
|
|
@@ -501,7 +607,9 @@ function checkBeforeStep(context, step, { skipBoundary }) {
|
|
|
501
607
|
return null;
|
|
502
608
|
}
|
|
503
609
|
if (context.stopAt.includes(step) && !skipBoundary) {
|
|
504
|
-
context.waitHuman(`enter-${step}`, [
|
|
610
|
+
context.waitHuman(`enter-${step}`, [
|
|
611
|
+
`automation boundary: stopAt includes ${step}`,
|
|
612
|
+
]);
|
|
505
613
|
return null;
|
|
506
614
|
}
|
|
507
615
|
const stepDef = workflow.steps.find((candidate) => candidate.id === step);
|
|
@@ -551,12 +659,22 @@ function checkBeforeStep(context, step, { skipBoundary }) {
|
|
|
551
659
|
const attempt = (ledger.state.steps[step]?.attempts ?? 0) + 1;
|
|
552
660
|
const maxAttempts = context.maxAttemptsFor(step);
|
|
553
661
|
if (attempt > maxAttempts) {
|
|
554
|
-
context.waitHuman(
|
|
662
|
+
context.waitHuman(
|
|
663
|
+
`resume-${step}`,
|
|
664
|
+
budgetReasons(context, step),
|
|
665
|
+
"budget",
|
|
666
|
+
budgetGrants(context, step),
|
|
667
|
+
);
|
|
555
668
|
return null;
|
|
556
669
|
}
|
|
557
670
|
|
|
558
671
|
if (attempt > context.totalAttemptsFor(step)) {
|
|
559
|
-
context.waitHuman(
|
|
672
|
+
context.waitHuman(
|
|
673
|
+
`resume-${step}`,
|
|
674
|
+
budgetReasons(context, step, true),
|
|
675
|
+
"budget",
|
|
676
|
+
budgetGrants(context, step),
|
|
677
|
+
);
|
|
560
678
|
return null;
|
|
561
679
|
}
|
|
562
680
|
|
|
@@ -582,14 +700,30 @@ function beginStep(context, step, stepDef, attempt) {
|
|
|
582
700
|
},
|
|
583
701
|
});
|
|
584
702
|
const before = stepDef.readonly ? readback(workspace.worktreePath) : null;
|
|
585
|
-
const outputsDir = join(
|
|
703
|
+
const outputsDir = join(
|
|
704
|
+
context.runtimeDir,
|
|
705
|
+
"runs",
|
|
706
|
+
ledger.state.run.id,
|
|
707
|
+
"outputs",
|
|
708
|
+
);
|
|
586
709
|
mkdirSync(outputsDir, { recursive: true });
|
|
587
710
|
const outputPath = join(outputsDir, `${step}-${attempt}.json`);
|
|
588
711
|
// Anchored review: readonly (reviewer) steps receive the adjudicated
|
|
589
712
|
// findings history so a fresh reviewer inherits settled verdicts instead
|
|
590
713
|
// of re-litigating them; writing steps get the latest review findings
|
|
591
714
|
// with their adjudication status (the fixer's worklist).
|
|
592
|
-
const input = {
|
|
715
|
+
const input = {
|
|
716
|
+
workId: ledger.state.run.work,
|
|
717
|
+
runId: ledger.state.run.id,
|
|
718
|
+
step,
|
|
719
|
+
attempt,
|
|
720
|
+
};
|
|
721
|
+
const artifact = ledger.state.run.deliveryChecks?.artifact;
|
|
722
|
+
if (artifact)
|
|
723
|
+
input.workArtifact = {
|
|
724
|
+
ref: `delivery/work/${ledger.state.run.work}/${artifact}.md`,
|
|
725
|
+
digest: ledger.state.run.planDigest,
|
|
726
|
+
};
|
|
593
727
|
const anchor = buildAnchor(context.repoRoot, ledger.state.run.work);
|
|
594
728
|
if (anchor && stepDef.readonly) {
|
|
595
729
|
input.anchor = anchor;
|
|
@@ -605,7 +739,8 @@ function beginStep(context, step, stepDef, attempt) {
|
|
|
605
739
|
severity: finding.severity,
|
|
606
740
|
summary: finding.summary,
|
|
607
741
|
fingerprint: fingerprintFinding(finding),
|
|
608
|
-
adjudication:
|
|
742
|
+
adjudication:
|
|
743
|
+
adjudicated.get(fingerprintFinding(finding))?.action ?? "open",
|
|
609
744
|
}));
|
|
610
745
|
}
|
|
611
746
|
}
|
|
@@ -621,24 +756,75 @@ function beginStep(context, step, stepDef, attempt) {
|
|
|
621
756
|
repoRoot: context.repoRoot,
|
|
622
757
|
});
|
|
623
758
|
if (prompt) {
|
|
624
|
-
input.envelope = {
|
|
759
|
+
input.envelope = {
|
|
760
|
+
promptRef: prompt.ref,
|
|
761
|
+
file: prompt.file,
|
|
762
|
+
digest: context.envelope.digest,
|
|
763
|
+
vars: context.envelope.vars,
|
|
764
|
+
};
|
|
625
765
|
}
|
|
626
766
|
// Incremental review (C7): tell a reviewer which candidate the last
|
|
627
767
|
// review saw when it is an ancestor of this one.
|
|
628
768
|
if (stepDef.readonly) {
|
|
629
769
|
const head = before.head;
|
|
630
|
-
const lastReviewed = lastReviewedCandidate(
|
|
770
|
+
const lastReviewed = lastReviewedCandidate(
|
|
771
|
+
context.repoRoot,
|
|
772
|
+
ledger.state.run.work,
|
|
773
|
+
workspace.worktreePath,
|
|
774
|
+
head,
|
|
775
|
+
);
|
|
631
776
|
if (lastReviewed) {
|
|
632
777
|
input.lastReviewed = lastReviewed;
|
|
633
778
|
}
|
|
779
|
+
// The reviewer judges the real render it is shown, not a description.
|
|
780
|
+
const screenshots = currentScreenshots(ledger.state.evidence, head);
|
|
781
|
+
if (screenshots.length > 0) {
|
|
782
|
+
input.screenshots = screenshots.map((item) => ({
|
|
783
|
+
path: resolveRepoRef(context.repoRoot, item.ref),
|
|
784
|
+
digest: item.digest,
|
|
785
|
+
}));
|
|
786
|
+
}
|
|
634
787
|
}
|
|
635
788
|
return { before, outputPath, input, prompt };
|
|
636
789
|
}
|
|
637
790
|
|
|
791
|
+
// An image counts only as a regular file (no symlink that could point at
|
|
792
|
+
// something removed later) that is structurally complete for its format.
|
|
793
|
+
function screenshotFiles(dir) {
|
|
794
|
+
const accepted = [];
|
|
795
|
+
const rejected = [];
|
|
796
|
+
for (const name of readdirSync(dir).sort()) {
|
|
797
|
+
if (!screenshotFormat(name)) {
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const path = join(dir, name);
|
|
801
|
+
const checked = checkScreenshotFile(path);
|
|
802
|
+
if (checked.ok) {
|
|
803
|
+
accepted.push({ file: path, digest: checked.digest });
|
|
804
|
+
} else {
|
|
805
|
+
rejected.push(checked.reason);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
return { accepted, rejected };
|
|
809
|
+
}
|
|
810
|
+
|
|
638
811
|
// Runs the worker, or references identical passed evidence (verification
|
|
639
812
|
// reuse), then records the command evidence for the tree git reads back.
|
|
640
|
-
function executeOrReuse(
|
|
813
|
+
function executeOrReuse(
|
|
814
|
+
context,
|
|
815
|
+
step,
|
|
816
|
+
stepDef,
|
|
817
|
+
adapter,
|
|
818
|
+
attempt,
|
|
819
|
+
{ before, outputPath, input, prompt },
|
|
820
|
+
) {
|
|
641
821
|
const { ledger, workspace, now } = context;
|
|
822
|
+
// UI work (requireScreenshot): verify leaves screenshots of the real
|
|
823
|
+
// render in a fresh directory outside the worktree; each image becomes
|
|
824
|
+
// screenshot evidence for this candidate.
|
|
825
|
+
const wantsScreenshots =
|
|
826
|
+
step === "verify" &&
|
|
827
|
+
ledger.state.run.deliveryChecks?.requireScreenshot === true;
|
|
642
828
|
// Verification reuse (C7): same tree + same worker + same envelope that
|
|
643
829
|
// already passed is referenced, not re-run. Failures always re-run.
|
|
644
830
|
let stepCacheKey = null;
|
|
@@ -652,10 +838,44 @@ function executeOrReuse(context, step, stepDef, adapter, attempt, { before, outp
|
|
|
652
838
|
adapterSpec: context.adapterConfigs[stepDef.worker] ?? null,
|
|
653
839
|
adapterName: adapter.name,
|
|
654
840
|
envelopeDigest: context.envelope?.digest ?? null,
|
|
841
|
+
screenshots: wantsScreenshots,
|
|
655
842
|
});
|
|
656
843
|
reused = findReusableEvidence(context.repoRoot, stepCacheKey);
|
|
657
844
|
}
|
|
658
845
|
}
|
|
846
|
+
// A reused verify carries its screenshots; one without any runs again.
|
|
847
|
+
let reusedScreenshots = [];
|
|
848
|
+
if (reused && wantsScreenshots) {
|
|
849
|
+
reusedScreenshots = linkedScreenshots(
|
|
850
|
+
context.repoRoot,
|
|
851
|
+
reused.run,
|
|
852
|
+
reused.evidenceRef,
|
|
853
|
+
);
|
|
854
|
+
// Only files that still pass today's screenshot check with their
|
|
855
|
+
// recorded bytes carry over; anything else means verify runs again.
|
|
856
|
+
const intact = reusedScreenshots.every((item) => {
|
|
857
|
+
const checked = checkScreenshotFile(
|
|
858
|
+
resolveRepoRef(context.repoRoot, item.evidenceRef),
|
|
859
|
+
);
|
|
860
|
+
return checked.ok && checked.digest === item.digest;
|
|
861
|
+
});
|
|
862
|
+
if (reusedScreenshots.length === 0 || !intact) {
|
|
863
|
+
reused = null;
|
|
864
|
+
reusedScreenshots = [];
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
let screenshotDir = null;
|
|
868
|
+
if (wantsScreenshots && !reused) {
|
|
869
|
+
screenshotDir = join(
|
|
870
|
+
context.runtimeDir,
|
|
871
|
+
"runs",
|
|
872
|
+
ledger.state.run.id,
|
|
873
|
+
"screenshots",
|
|
874
|
+
`${step}-${attempt}`,
|
|
875
|
+
);
|
|
876
|
+
rmSync(screenshotDir, { recursive: true, force: true });
|
|
877
|
+
mkdirSync(screenshotDir, { recursive: true });
|
|
878
|
+
}
|
|
659
879
|
let exec;
|
|
660
880
|
if (reused) {
|
|
661
881
|
const at = now();
|
|
@@ -684,8 +904,36 @@ function executeOrReuse(context, step, stepDef, adapter, attempt, { before, outp
|
|
|
684
904
|
liveDir: join(context.runtimeDir, "runs", ledger.state.run.id),
|
|
685
905
|
promptPath: prompt?.path ?? null,
|
|
686
906
|
vars: context.envelope?.vars ?? null,
|
|
907
|
+
extraEnv: screenshotDir
|
|
908
|
+
? { BUILDBEAT_SCREENSHOT_DIR: screenshotDir }
|
|
909
|
+
: undefined,
|
|
687
910
|
});
|
|
688
911
|
}
|
|
912
|
+
const found = screenshotDir
|
|
913
|
+
? screenshotFiles(screenshotDir)
|
|
914
|
+
: { accepted: [], rejected: [] };
|
|
915
|
+
const shots = found.accepted;
|
|
916
|
+
// Rejections are reported whether or not another screenshot passed.
|
|
917
|
+
if (found.rejected.length > 0) {
|
|
918
|
+
exec = { ...exec, screenshotsRejected: found.rejected };
|
|
919
|
+
}
|
|
920
|
+
if (
|
|
921
|
+
screenshotDir &&
|
|
922
|
+
shots.length === 0 &&
|
|
923
|
+
exec.exitCode === 0 &&
|
|
924
|
+
!exec.timedOut &&
|
|
925
|
+
!exec.signal &&
|
|
926
|
+
!exec.spawnError
|
|
927
|
+
) {
|
|
928
|
+
exec = {
|
|
929
|
+
...exec,
|
|
930
|
+
requirementFailure:
|
|
931
|
+
"requireScreenshot is on, but verify left no PNG screenshot in BUILDBEAT_SCREENSHOT_DIR" +
|
|
932
|
+
(found.rejected.length > 0
|
|
933
|
+
? `; rejected: ${found.rejected.join(", ")}`
|
|
934
|
+
: ""),
|
|
935
|
+
};
|
|
936
|
+
}
|
|
689
937
|
const tree = readback(workspace.worktreePath);
|
|
690
938
|
const evidence = collectCommandEvidence({
|
|
691
939
|
runtimeDir: context.runtimeDir,
|
|
@@ -694,7 +942,7 @@ function executeOrReuse(context, step, stepDef, adapter, attempt, { before, outp
|
|
|
694
942
|
attempt,
|
|
695
943
|
execResult: exec,
|
|
696
944
|
subject: tree.head,
|
|
697
|
-
grade: reused ? reused.grade : stepDef.grade ?? "L2",
|
|
945
|
+
grade: reused ? reused.grade : (stepDef.grade ?? "L2"),
|
|
698
946
|
redact: context.redact,
|
|
699
947
|
});
|
|
700
948
|
ledger.append({
|
|
@@ -709,9 +957,49 @@ function executeOrReuse(context, step, stepDef, adapter, attempt, { before, outp
|
|
|
709
957
|
status: evidence.status,
|
|
710
958
|
grade: evidence.grade,
|
|
711
959
|
...(stepCacheKey ? { cacheKey: stepCacheKey } : {}),
|
|
712
|
-
...(reused
|
|
960
|
+
...(reused
|
|
961
|
+
? {
|
|
962
|
+
reused: {
|
|
963
|
+
run: reused.run,
|
|
964
|
+
evidenceRef: reused.evidenceRef,
|
|
965
|
+
digest: reused.digest,
|
|
966
|
+
},
|
|
967
|
+
}
|
|
968
|
+
: {}),
|
|
713
969
|
},
|
|
714
970
|
});
|
|
971
|
+
if (evidence.status === "passed") {
|
|
972
|
+
const source = toRepoRef(context.repoRoot, evidence.location);
|
|
973
|
+
const recorded = reused
|
|
974
|
+
? reusedScreenshots.map((item) => ({
|
|
975
|
+
ref: item.evidenceRef,
|
|
976
|
+
digest: item.digest,
|
|
977
|
+
}))
|
|
978
|
+
: shots.map((shot) => ({
|
|
979
|
+
ref: toRepoRef(context.repoRoot, shot.file),
|
|
980
|
+
digest: shot.digest,
|
|
981
|
+
}));
|
|
982
|
+
for (const shot of recorded) {
|
|
983
|
+
ledger.append({
|
|
984
|
+
type: "EVIDENCE_RECORDED",
|
|
985
|
+
actor: KERNEL,
|
|
986
|
+
ts: now(),
|
|
987
|
+
data: {
|
|
988
|
+
evidenceRef: shot.ref,
|
|
989
|
+
kind: "screenshot",
|
|
990
|
+
subject: tree.head,
|
|
991
|
+
digest: shot.digest,
|
|
992
|
+
status: "passed",
|
|
993
|
+
grade: "L2",
|
|
994
|
+
source,
|
|
995
|
+
...(stepCacheKey ? { cacheKey: stepCacheKey } : {}),
|
|
996
|
+
...(reused
|
|
997
|
+
? { reused: { run: reused.run, evidenceRef: shot.ref } }
|
|
998
|
+
: {}),
|
|
999
|
+
},
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
715
1003
|
|
|
716
1004
|
return { exec, tree };
|
|
717
1005
|
}
|
|
@@ -720,11 +1008,21 @@ function executeOrReuse(context, step, stepDef, adapter, attempt, { before, outp
|
|
|
720
1008
|
// worker envelope, status and infrastructure failures, review findings,
|
|
721
1009
|
// scope, the pinned candidate and post policies. Returns the result to
|
|
722
1010
|
// route, or null when the run stopped.
|
|
723
|
-
function recordStepResult(
|
|
1011
|
+
function recordStepResult(
|
|
1012
|
+
context,
|
|
1013
|
+
step,
|
|
1014
|
+
stepDef,
|
|
1015
|
+
attempt,
|
|
1016
|
+
{ before, outputPath },
|
|
1017
|
+
{ exec, tree },
|
|
1018
|
+
) {
|
|
724
1019
|
const { ledger, workspace, now } = context;
|
|
725
1020
|
// Read-only enforcement: a reviewer that changed the workspace is a
|
|
726
1021
|
// policy violation, not a candidate (invariants 9/17).
|
|
727
|
-
if (
|
|
1022
|
+
if (
|
|
1023
|
+
stepDef.readonly &&
|
|
1024
|
+
(tree.head !== before.head || tree.dirty !== before.dirty)
|
|
1025
|
+
) {
|
|
728
1026
|
ledger.append({
|
|
729
1027
|
type: "POLICY_EVALUATED",
|
|
730
1028
|
actor: KERNEL,
|
|
@@ -751,7 +1049,11 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
751
1049
|
|
|
752
1050
|
let envelopeRaw = exec.envelope;
|
|
753
1051
|
if (exec.envelope !== undefined && exec.envelope !== null) {
|
|
754
|
-
writeFileSync(
|
|
1052
|
+
writeFileSync(
|
|
1053
|
+
outputPath,
|
|
1054
|
+
`${JSON.stringify(exec.envelope, null, 2)}\n`,
|
|
1055
|
+
"utf8",
|
|
1056
|
+
);
|
|
755
1057
|
} else if (existsSync(outputPath)) {
|
|
756
1058
|
envelopeRaw = readFileSync(outputPath, "utf8");
|
|
757
1059
|
}
|
|
@@ -766,6 +1068,10 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
766
1068
|
stepStatus = "crashed";
|
|
767
1069
|
} else if (exec.exitCode !== 0) {
|
|
768
1070
|
stepStatus = "failed";
|
|
1071
|
+
} else if (exec.requirementFailure) {
|
|
1072
|
+
// The command passed but a requirement the runner checks did not
|
|
1073
|
+
// (no screenshots): a candidate failure, routed like any other.
|
|
1074
|
+
stepStatus = "failed";
|
|
769
1075
|
} else if (envelopeError) {
|
|
770
1076
|
stepStatus = "invalid-output";
|
|
771
1077
|
} else {
|
|
@@ -789,8 +1095,18 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
789
1095
|
type: "STEP_FINISHED",
|
|
790
1096
|
actor: KERNEL,
|
|
791
1097
|
ts: now(),
|
|
792
|
-
data: {
|
|
793
|
-
|
|
1098
|
+
data: {
|
|
1099
|
+
step,
|
|
1100
|
+
attempt,
|
|
1101
|
+
status: stepStatus,
|
|
1102
|
+
exitCode: exec.exitCode,
|
|
1103
|
+
...(exec.requirementFailure ? { reason: exec.requirementFailure } : {}),
|
|
1104
|
+
...(exec.screenshotsRejected
|
|
1105
|
+
? { screenshotsRejected: exec.screenshotsRejected }
|
|
1106
|
+
: {}),
|
|
1107
|
+
...(infra ? { infra: true } : {}),
|
|
1108
|
+
...(free ? { free: true } : {}),
|
|
1109
|
+
},
|
|
794
1110
|
});
|
|
795
1111
|
ledger.append({
|
|
796
1112
|
type: "BUDGET_CONSUMED",
|
|
@@ -813,7 +1129,11 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
813
1129
|
`resume-${step}`,
|
|
814
1130
|
[
|
|
815
1131
|
`worker infrastructure failure at ${step}: ${cause}; not a candidate defect, attempt not charged`,
|
|
816
|
-
...(tree.dirty
|
|
1132
|
+
...(tree.dirty
|
|
1133
|
+
? [
|
|
1134
|
+
`the failed worker left the worktree dirty; inspect before rerunning`,
|
|
1135
|
+
]
|
|
1136
|
+
: []),
|
|
817
1137
|
`approve resume-${step} to rerun once the backend/environment is back; reject to end the run`,
|
|
818
1138
|
],
|
|
819
1139
|
"infra",
|
|
@@ -855,7 +1175,9 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
855
1175
|
status: blockingFindings.length > 0 ? "failed" : "passed",
|
|
856
1176
|
grade: "L2",
|
|
857
1177
|
findings: envelope.findings,
|
|
858
|
-
...(suppressed.length > 0
|
|
1178
|
+
...(suppressed.length > 0
|
|
1179
|
+
? { suppressedFingerprints: suppressed }
|
|
1180
|
+
: {}),
|
|
859
1181
|
},
|
|
860
1182
|
});
|
|
861
1183
|
}
|
|
@@ -869,7 +1191,8 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
869
1191
|
(path) =>
|
|
870
1192
|
!context.allowedPaths.some(
|
|
871
1193
|
(prefix) =>
|
|
872
|
-
path === prefix ||
|
|
1194
|
+
path === prefix ||
|
|
1195
|
+
path.startsWith(prefix.endsWith("/") ? prefix : `${prefix}/`),
|
|
873
1196
|
),
|
|
874
1197
|
);
|
|
875
1198
|
if (violations.length > 0) {
|
|
@@ -938,7 +1261,12 @@ function recordStepResult(context, step, stepDef, attempt, { before, outputPath
|
|
|
938
1261
|
// Settles the outcome and picks the next step; blocking findings stop once
|
|
939
1262
|
// for triage, the review budget or a review that is not converging before
|
|
940
1263
|
// any fixer runs. Returns the next step, or null when the run stopped.
|
|
941
|
-
function routeAfterStep(
|
|
1264
|
+
function routeAfterStep(
|
|
1265
|
+
context,
|
|
1266
|
+
step,
|
|
1267
|
+
stepDef,
|
|
1268
|
+
{ stepStatus, blockingFindings, tree, exec },
|
|
1269
|
+
) {
|
|
942
1270
|
let outcome;
|
|
943
1271
|
if (stepStatus !== "succeeded") {
|
|
944
1272
|
outcome = "failed";
|
|
@@ -953,7 +1281,9 @@ function routeAfterStep(context, step, stepDef, { stepStatus, blockingFindings,
|
|
|
953
1281
|
if (routed && outcome === "findings-blocking") {
|
|
954
1282
|
const review = isReviewStep(step, stepDef);
|
|
955
1283
|
const grants = review ? budgetGrants(context, step) : [];
|
|
956
|
-
const stalled = review
|
|
1284
|
+
const stalled = review
|
|
1285
|
+
? reviewNotConverging(context, step, blockingFindings)
|
|
1286
|
+
: null;
|
|
957
1287
|
const triage = context.reviewTriage === "required";
|
|
958
1288
|
if (triage || grants.length || stalled) {
|
|
959
1289
|
const { used, limit, failures } = budgetUsage(context, step);
|
|
@@ -961,16 +1291,26 @@ function routeAfterStep(context, step, stepDef, { stepStatus, blockingFindings,
|
|
|
961
1291
|
context.waitHuman(
|
|
962
1292
|
`enter-${routed}`,
|
|
963
1293
|
[
|
|
964
|
-
...(grants.length
|
|
965
|
-
|
|
966
|
-
|
|
1294
|
+
...(grants.length
|
|
1295
|
+
? [
|
|
1296
|
+
`${step} budget exhausted: ${used}/${limit} review round(s) used in this run${workBudget ? `, ${workBudget.rounds}/${workBudget.allowed} across the work` : ""}, ${failures} real failure(s); approve enter-${routed} = fix + re-verify + one more review round; reject = end this run and decide the merge on the evidence you have`,
|
|
1297
|
+
]
|
|
1298
|
+
: []),
|
|
967
1299
|
...(stalled ? [stalled] : []),
|
|
968
1300
|
`review found ${blockingFindings.length} blocking finding(s); ${triage ? "triage" : "approve another round"} before ${routed} runs`,
|
|
969
|
-
...blockingFindings
|
|
970
|
-
|
|
1301
|
+
...blockingFindings
|
|
1302
|
+
.slice(0, 5)
|
|
1303
|
+
.map(
|
|
1304
|
+
(finding) =>
|
|
1305
|
+
`[${finding.severity} ${fingerprintFinding(finding)}] ${finding.summary.slice(0, 200)}`,
|
|
1306
|
+
),
|
|
971
1307
|
`adjudicate fingerprints (findings adjudicate), then approve enter-${routed} or reject the run`,
|
|
972
1308
|
],
|
|
973
|
-
triage
|
|
1309
|
+
triage
|
|
1310
|
+
? "finding-triage"
|
|
1311
|
+
: grants.length
|
|
1312
|
+
? "budget"
|
|
1313
|
+
: "review-not-converging",
|
|
974
1314
|
grants,
|
|
975
1315
|
);
|
|
976
1316
|
return null;
|
|
@@ -983,15 +1323,31 @@ function drive(context, startStep, { skipBoundaryOnce = false } = {}) {
|
|
|
983
1323
|
let step = startStep;
|
|
984
1324
|
let firstStep = true;
|
|
985
1325
|
while (step) {
|
|
986
|
-
const entry = checkBeforeStep(context, step, {
|
|
1326
|
+
const entry = checkBeforeStep(context, step, {
|
|
1327
|
+
skipBoundary: skipBoundaryOnce && firstStep,
|
|
1328
|
+
});
|
|
987
1329
|
firstStep = false;
|
|
988
1330
|
if (!entry) {
|
|
989
1331
|
return;
|
|
990
1332
|
}
|
|
991
1333
|
const { stepDef, adapter, attempt } = entry;
|
|
992
1334
|
const started = beginStep(context, step, stepDef, attempt);
|
|
993
|
-
const ran = executeOrReuse(
|
|
994
|
-
|
|
1335
|
+
const ran = executeOrReuse(
|
|
1336
|
+
context,
|
|
1337
|
+
step,
|
|
1338
|
+
stepDef,
|
|
1339
|
+
adapter,
|
|
1340
|
+
attempt,
|
|
1341
|
+
started,
|
|
1342
|
+
);
|
|
1343
|
+
const result = recordStepResult(
|
|
1344
|
+
context,
|
|
1345
|
+
step,
|
|
1346
|
+
stepDef,
|
|
1347
|
+
attempt,
|
|
1348
|
+
started,
|
|
1349
|
+
ran,
|
|
1350
|
+
);
|
|
995
1351
|
if (!result) {
|
|
996
1352
|
return;
|
|
997
1353
|
}
|
|
@@ -1052,7 +1408,10 @@ function supersedeWaitingRuns(repoRoot, workId, newRunId, now) {
|
|
|
1052
1408
|
type: "RUN_TERMINAL",
|
|
1053
1409
|
actor: KERNEL,
|
|
1054
1410
|
ts: now(),
|
|
1055
|
-
data: {
|
|
1411
|
+
data: {
|
|
1412
|
+
status: "SUPERSEDED",
|
|
1413
|
+
reason: `superseded by ${newRunId} (same work ${workId})`,
|
|
1414
|
+
},
|
|
1056
1415
|
});
|
|
1057
1416
|
writeRunRecord({ repoRoot, ledger: locked, ts: now() });
|
|
1058
1417
|
superseded.push(entry);
|
|
@@ -1064,7 +1423,14 @@ function supersedeWaitingRuns(repoRoot, workId, newRunId, now) {
|
|
|
1064
1423
|
}
|
|
1065
1424
|
|
|
1066
1425
|
function openLedgerFor(repoRoot, runId) {
|
|
1067
|
-
const ledgerPath = join(
|
|
1426
|
+
const ledgerPath = join(
|
|
1427
|
+
repoRoot,
|
|
1428
|
+
".buildbeat",
|
|
1429
|
+
"runtime",
|
|
1430
|
+
"runs",
|
|
1431
|
+
runId,
|
|
1432
|
+
"events.jsonl",
|
|
1433
|
+
);
|
|
1068
1434
|
const ledger = EventLedger.open(ledgerPath);
|
|
1069
1435
|
if (ledger.corruption) {
|
|
1070
1436
|
throw new OrchestratorError(
|
|
@@ -1094,7 +1460,9 @@ export function startRun(options) {
|
|
|
1094
1460
|
throw new OrchestratorError(`entry step not in workflow: ${entry}`);
|
|
1095
1461
|
}
|
|
1096
1462
|
if (!workflowDigest) {
|
|
1097
|
-
throw new OrchestratorError(
|
|
1463
|
+
throw new OrchestratorError(
|
|
1464
|
+
"workflowDigest is required (pin what you run)",
|
|
1465
|
+
);
|
|
1098
1466
|
}
|
|
1099
1467
|
// Environment contract first: a missing or too-old binary fails the start
|
|
1100
1468
|
// with a readable cause instead of burning a run on an implicit PATH fact.
|
|
@@ -1103,59 +1471,93 @@ export function startRun(options) {
|
|
|
1103
1471
|
}
|
|
1104
1472
|
const { ledger, ledgerPath } = openLedgerFor(repoRoot, runId);
|
|
1105
1473
|
if (ledger.events.length > 0) {
|
|
1106
|
-
throw new OrchestratorError(
|
|
1474
|
+
throw new OrchestratorError(
|
|
1475
|
+
`run ${runId} already has a ledger; use resumeRun`,
|
|
1476
|
+
);
|
|
1107
1477
|
}
|
|
1108
1478
|
|
|
1109
|
-
return withRunLocks(
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1479
|
+
return withRunLocks(
|
|
1480
|
+
repoRoot,
|
|
1481
|
+
runId,
|
|
1482
|
+
() => {
|
|
1483
|
+
const workspace = withRepoGitLock(repoRoot, () =>
|
|
1484
|
+
createWorkspace({ repoRoot, runId, base }),
|
|
1485
|
+
);
|
|
1486
|
+
// The checkout workers will read is the final word on what they were
|
|
1487
|
+
// given; a run whose checkout fails the caller's check never starts.
|
|
1488
|
+
if (options.verifyCheckout) {
|
|
1489
|
+
try {
|
|
1490
|
+
options.verifyCheckout(workspace.worktreePath);
|
|
1491
|
+
} catch (error) {
|
|
1492
|
+
withRepoGitLock(repoRoot, () => discardWorkspace(workspace));
|
|
1493
|
+
throw error;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
const context = makeContext(options, ledger, workspace);
|
|
1497
|
+
const now = context.now;
|
|
1498
|
+
const supersession =
|
|
1499
|
+
options.supersede === "off"
|
|
1500
|
+
? { superseded: [], skipped: [] }
|
|
1501
|
+
: supersedeWaitingRuns(repoRoot, workId, runId, now);
|
|
1502
|
+
ledger.append({
|
|
1503
|
+
type: "RUN_CREATED",
|
|
1504
|
+
actor: KERNEL,
|
|
1505
|
+
ts: now(),
|
|
1506
|
+
run: runId,
|
|
1507
|
+
work: workId,
|
|
1508
|
+
data: {
|
|
1509
|
+
workflowRef: workflow.name,
|
|
1510
|
+
workflowDigest,
|
|
1511
|
+
base: workspace.base,
|
|
1512
|
+
riskPreset,
|
|
1513
|
+
entry,
|
|
1514
|
+
planDigest: planDigest ?? "UNVERIFIED",
|
|
1515
|
+
intentDigest: intentDigest ?? "UNVERIFIED",
|
|
1516
|
+
...(options.deliveryChecks
|
|
1517
|
+
? { deliveryChecks: options.deliveryChecks }
|
|
1518
|
+
: {}),
|
|
1519
|
+
...(supersession.superseded.length > 0
|
|
1520
|
+
? { supersedes: supersession.superseded }
|
|
1521
|
+
: {}),
|
|
1522
|
+
...(options.envelope
|
|
1523
|
+
? {
|
|
1524
|
+
envelopeDigest: options.envelope.digest,
|
|
1525
|
+
envelopeSource: options.envelope.source,
|
|
1526
|
+
}
|
|
1527
|
+
: {}),
|
|
1528
|
+
},
|
|
1529
|
+
});
|
|
1530
|
+
ledger.append({
|
|
1531
|
+
type: "RUN_STARTED",
|
|
1532
|
+
actor: KERNEL,
|
|
1533
|
+
ts: now(),
|
|
1534
|
+
data: {},
|
|
1535
|
+
});
|
|
1536
|
+
ledger.append({
|
|
1537
|
+
type: "WORKSPACE_BOUND",
|
|
1538
|
+
actor: KERNEL,
|
|
1539
|
+
ts: now(),
|
|
1540
|
+
data: {
|
|
1541
|
+
workspaceId: workspace.workspaceId,
|
|
1542
|
+
repo: toRepoRef(repoRoot, repoRoot),
|
|
1543
|
+
branch: workspace.branch,
|
|
1544
|
+
worktreePath: toRepoRef(repoRoot, workspace.worktreePath),
|
|
1545
|
+
base: workspace.base,
|
|
1546
|
+
},
|
|
1547
|
+
});
|
|
1548
|
+
drive(context, entry);
|
|
1549
|
+
return {
|
|
1550
|
+
runId,
|
|
1551
|
+
workId,
|
|
1552
|
+
ledgerPath,
|
|
1553
|
+
state: ledger.state,
|
|
1554
|
+
workspace,
|
|
1555
|
+
superseded: supersession.superseded,
|
|
1556
|
+
supersedeSkipped: supersession.skipped,
|
|
1557
|
+
};
|
|
1558
|
+
},
|
|
1559
|
+
{ workId, parallel: options.parallel === true },
|
|
1560
|
+
);
|
|
1159
1561
|
}
|
|
1160
1562
|
|
|
1161
1563
|
function resumeStepFromTransition(transition) {
|
|
@@ -1168,26 +1570,61 @@ function resumeStepFromTransition(transition) {
|
|
|
1168
1570
|
return null;
|
|
1169
1571
|
}
|
|
1170
1572
|
|
|
1171
|
-
function
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1573
|
+
export function assertRunConfiguration(state, options) {
|
|
1574
|
+
if (!state.run) throw new OrchestratorError("run has no creation record");
|
|
1575
|
+
if (options.workId && options.workId !== state.run.work)
|
|
1576
|
+
throw new OrchestratorError("run belongs to a different work");
|
|
1577
|
+
const { workflowDigest } = options;
|
|
1578
|
+
if (
|
|
1579
|
+
state.run.deliveryChecks &&
|
|
1580
|
+
canonicalJson(state.run.deliveryChecks) !==
|
|
1581
|
+
canonicalJson(options.deliveryChecks ?? null)
|
|
1582
|
+
) {
|
|
1583
|
+
throw new OrchestratorError(
|
|
1584
|
+
"delivery safeguards changed since the run was created; refusing to resume (docs/MIGRATION.md)",
|
|
1585
|
+
);
|
|
1176
1586
|
}
|
|
1177
1587
|
if (workflowDigest && state.run.workflowDigest !== workflowDigest) {
|
|
1178
1588
|
throw new OrchestratorError(
|
|
1179
1589
|
`workflow changed since the run was created (${state.run.workflowDigest} != ${workflowDigest}); refusing to resume`,
|
|
1180
1590
|
);
|
|
1181
1591
|
}
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
function resumeTarget(options, { ledger, ledgerPath }) {
|
|
1595
|
+
const { repoRoot, runId } = options;
|
|
1596
|
+
const state = ledger.state;
|
|
1597
|
+
if (!state.run) {
|
|
1598
|
+
throw new OrchestratorError(`no ledger for run ${runId}; use startRun`);
|
|
1599
|
+
}
|
|
1182
1600
|
if (state.terminal) {
|
|
1183
|
-
return {
|
|
1601
|
+
return {
|
|
1602
|
+
early: {
|
|
1603
|
+
runId,
|
|
1604
|
+
ledgerPath,
|
|
1605
|
+
state,
|
|
1606
|
+
resumed: false,
|
|
1607
|
+
reason: "run is terminal",
|
|
1608
|
+
},
|
|
1609
|
+
};
|
|
1184
1610
|
}
|
|
1611
|
+
assertRunConfiguration(state, options);
|
|
1185
1612
|
if (state.run.status === "WAITING_HUMAN" && state.pendingHuman) {
|
|
1186
|
-
return {
|
|
1613
|
+
return {
|
|
1614
|
+
early: {
|
|
1615
|
+
runId,
|
|
1616
|
+
ledgerPath,
|
|
1617
|
+
state,
|
|
1618
|
+
resumed: false,
|
|
1619
|
+
reason: "waiting on a human decision",
|
|
1620
|
+
},
|
|
1621
|
+
};
|
|
1187
1622
|
}
|
|
1188
1623
|
const bound = state.workspaces[runId];
|
|
1189
1624
|
if (!bound) {
|
|
1190
|
-
throw new OrchestratorError(
|
|
1625
|
+
throw new OrchestratorError(
|
|
1626
|
+
`run ${runId} has no bound workspace; cannot resume`,
|
|
1627
|
+
);
|
|
1191
1628
|
}
|
|
1192
1629
|
const worktreePath = resolveRepoRef(repoRoot, bound.worktreePath);
|
|
1193
1630
|
if (!existsSync(worktreePath)) {
|
|
@@ -1226,178 +1663,255 @@ export function resumeRun(options) {
|
|
|
1226
1663
|
return outside.early;
|
|
1227
1664
|
}
|
|
1228
1665
|
|
|
1229
|
-
return withRunLocks(
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
const context = makeContext(options, ledger, workspace);
|
|
1238
|
-
const now = context.now;
|
|
1239
|
-
|
|
1240
|
-
if (state.run.status === "WAITING_HUMAN") {
|
|
1241
|
-
// Pending request already resolved: continue only if the approval's
|
|
1242
|
-
// subject is still exactly what the human saw (F6 machine closure).
|
|
1243
|
-
const approval = [...state.approvals].reverse().find((entry) => !entry.stale);
|
|
1244
|
-
if (!approval) {
|
|
1245
|
-
return { runId, ledgerPath, state, resumed: false, reason: "no active approval to act on" };
|
|
1246
|
-
}
|
|
1247
|
-
const tree = readback(workspace.worktreePath);
|
|
1248
|
-
const changed = [];
|
|
1249
|
-
if (tree.head !== approval.subject.candidate) {
|
|
1250
|
-
changed.push("candidate");
|
|
1251
|
-
}
|
|
1252
|
-
if (
|
|
1253
|
-
planDigest &&
|
|
1254
|
-
approval.subject.planDigest !== "UNVERIFIED" &&
|
|
1255
|
-
planDigest !== approval.subject.planDigest
|
|
1256
|
-
) {
|
|
1257
|
-
changed.push("plan");
|
|
1258
|
-
}
|
|
1259
|
-
if (changed.length > 0) {
|
|
1260
|
-
ledger.append({
|
|
1261
|
-
type: "APPROVAL_STALE",
|
|
1262
|
-
actor: KERNEL,
|
|
1263
|
-
ts: now(),
|
|
1264
|
-
data: { approvalRef: approval.decisionRef, changed },
|
|
1265
|
-
});
|
|
1266
|
-
return { runId, ledgerPath, state: ledger.state, resumed: true, stale: true, reason: null };
|
|
1666
|
+
return withRunLocks(
|
|
1667
|
+
repoRoot,
|
|
1668
|
+
runId,
|
|
1669
|
+
() => {
|
|
1670
|
+
const { ledger, ledgerPath } = openLedgerFor(repoRoot, runId);
|
|
1671
|
+
const target = resumeTarget(options, { ledger, ledgerPath });
|
|
1672
|
+
if (target.early) {
|
|
1673
|
+
return target.early;
|
|
1267
1674
|
}
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
const
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1675
|
+
const { workspace } = target;
|
|
1676
|
+
const state = ledger.state;
|
|
1677
|
+
const context = makeContext(options, ledger, workspace);
|
|
1678
|
+
const now = context.now;
|
|
1679
|
+
|
|
1680
|
+
if (state.run.status === "WAITING_HUMAN") {
|
|
1681
|
+
// Pending request already resolved: continue only if the approval's
|
|
1682
|
+
// subject is still exactly what the human saw (F6 machine closure).
|
|
1683
|
+
const approval = [...state.approvals]
|
|
1684
|
+
.reverse()
|
|
1685
|
+
.find((entry) => !entry.stale);
|
|
1686
|
+
if (!approval) {
|
|
1687
|
+
return {
|
|
1688
|
+
runId,
|
|
1689
|
+
ledgerPath,
|
|
1690
|
+
state,
|
|
1691
|
+
resumed: false,
|
|
1692
|
+
reason: "no active approval to act on",
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
const tree = readback(workspace.worktreePath);
|
|
1696
|
+
const changed = [];
|
|
1697
|
+
if (tree.head !== approval.subject.candidate) {
|
|
1698
|
+
changed.push("candidate");
|
|
1699
|
+
}
|
|
1700
|
+
if (
|
|
1701
|
+
planDigest &&
|
|
1702
|
+
approval.subject.planDigest !== "UNVERIFIED" &&
|
|
1703
|
+
planDigest !== approval.subject.planDigest
|
|
1704
|
+
) {
|
|
1705
|
+
changed.push("plan");
|
|
1706
|
+
}
|
|
1707
|
+
if (changed.length > 0) {
|
|
1708
|
+
ledger.append({
|
|
1709
|
+
type: "APPROVAL_STALE",
|
|
1710
|
+
actor: KERNEL,
|
|
1711
|
+
ts: now(),
|
|
1712
|
+
data: { approvalRef: approval.decisionRef, changed },
|
|
1713
|
+
});
|
|
1714
|
+
return {
|
|
1715
|
+
runId,
|
|
1716
|
+
ledgerPath,
|
|
1717
|
+
state: ledger.state,
|
|
1718
|
+
resumed: true,
|
|
1719
|
+
stale: true,
|
|
1720
|
+
reason: null,
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1723
|
+
// An adopted candidate names where to resume (verify, by convention):
|
|
1724
|
+
// the fixer step the request was waiting on has nothing left to do.
|
|
1725
|
+
const step =
|
|
1726
|
+
approval.resumeAt ?? resumeStepFromTransition(approval.transition);
|
|
1727
|
+
if (!step || !context.workflow.stepIds.has(step)) {
|
|
1728
|
+
throw new OrchestratorError(
|
|
1729
|
+
`cannot derive a resume step from approved transition ${approval.transition}`,
|
|
1730
|
+
);
|
|
1731
|
+
}
|
|
1732
|
+
// An approved resume-<step> on an exhausted budget is the human saying
|
|
1733
|
+
// "one more"; record the grant before driving or the same request
|
|
1734
|
+
// comes straight back (the pilot's app-login runs ended CANCELLED
|
|
1735
|
+
// with their candidates in production because of exactly that).
|
|
1736
|
+
const decisionIndex = ledger.events.findIndex(
|
|
1737
|
+
(event) =>
|
|
1738
|
+
event.type === "DECISION_RECORDED" &&
|
|
1739
|
+
event.data.decisionRef === approval.decisionRef,
|
|
1274
1740
|
);
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
(
|
|
1306
|
-
approval.subject
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1741
|
+
// APPROVAL_STALE is itself a new request, with no inherited grants.
|
|
1742
|
+
// Restrict lookup to this decision, rather than an older request for
|
|
1743
|
+
// the same transition (or one whose subject was refreshed).
|
|
1744
|
+
const request = ledger.events
|
|
1745
|
+
.slice(0, decisionIndex)
|
|
1746
|
+
.reverse()
|
|
1747
|
+
.find(
|
|
1748
|
+
(event) =>
|
|
1749
|
+
event.type === "HUMAN_REQUESTED" ||
|
|
1750
|
+
event.type === "APPROVAL_STALE",
|
|
1751
|
+
);
|
|
1752
|
+
const requestData =
|
|
1753
|
+
request?.type === "HUMAN_REQUESTED" &&
|
|
1754
|
+
request.data.transition === approval.transition
|
|
1755
|
+
? request.data
|
|
1756
|
+
: null;
|
|
1757
|
+
// The grant plan is fixed once, then pinned on the first
|
|
1758
|
+
// BUDGET_EXTENDED it produces; a resume after a crash between two
|
|
1759
|
+
// grants replays that plan instead of re-deriving it from a state the
|
|
1760
|
+
// first grant already raised.
|
|
1761
|
+
const applied = ledger.events.filter(
|
|
1762
|
+
(event) =>
|
|
1763
|
+
event.type === "BUDGET_EXTENDED" &&
|
|
1764
|
+
event.data.approvalRef === approval.decisionRef,
|
|
1765
|
+
);
|
|
1766
|
+
let grants;
|
|
1767
|
+
if (Array.isArray(applied[0]?.data.grants)) {
|
|
1768
|
+
grants = applied[0].data.grants;
|
|
1769
|
+
} else if (
|
|
1770
|
+
Array.isArray(requestData?.grants) &&
|
|
1771
|
+
(canonicalJson(requestData.subject) ===
|
|
1772
|
+
canonicalJson(approval.subject) ||
|
|
1773
|
+
// resume --adopt answers this very request with a new candidate by
|
|
1774
|
+
// design; the grant belongs to the round, not to a candidate. Real
|
|
1775
|
+
// incident: the session's hand fix was adopted and the run stopped
|
|
1776
|
+
// again at resume-review for the round the human had just granted.
|
|
1777
|
+
(ledger.events[decisionIndex]?.data.adopted &&
|
|
1778
|
+
approval.subject.planDigest === requestData.subject.planDigest))
|
|
1779
|
+
) {
|
|
1780
|
+
grants = requestData.grants;
|
|
1781
|
+
} else {
|
|
1782
|
+
// Requests without grants: ledgers written before grants existed,
|
|
1783
|
+
// or a request refreshed by APPROVAL_STALE.
|
|
1784
|
+
grants = [];
|
|
1785
|
+
const attempts = state.steps[step]?.attempts ?? 0;
|
|
1786
|
+
const runCapApproval =
|
|
1787
|
+
approval.transition.startsWith("resume-") &&
|
|
1788
|
+
(attempts >= context.maxAttemptsFor(step) ||
|
|
1789
|
+
attempts >= context.totalAttemptsFor(step));
|
|
1790
|
+
if (requestData?.kind === "work-review-cap")
|
|
1791
|
+
grants.push({ step, scope: "work" });
|
|
1792
|
+
if (runCapApproval) grants.push({ step, scope: "run" });
|
|
1793
|
+
if (grants.length > 0) grants.push(...budgetGrants(context, step));
|
|
1794
|
+
}
|
|
1795
|
+
const plan = [];
|
|
1796
|
+
const planned = new Set();
|
|
1797
|
+
for (const grant of grants) {
|
|
1798
|
+
const key = `${grant.step}:${grant.scope}`;
|
|
1799
|
+
if (!planned.has(key)) {
|
|
1800
|
+
planned.add(key);
|
|
1801
|
+
plan.push({ step: grant.step, scope: grant.scope });
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
const extended = new Set(
|
|
1805
|
+
applied.map(
|
|
1806
|
+
(event) => `${event.data.step}:${event.data.scope ?? "run"}`,
|
|
1807
|
+
),
|
|
1808
|
+
);
|
|
1809
|
+
for (const grant of plan) {
|
|
1810
|
+
const key = `${grant.step}:${grant.scope}`;
|
|
1811
|
+
if (extended.has(key)) continue;
|
|
1812
|
+
extended.add(key);
|
|
1813
|
+
ledger.append({
|
|
1814
|
+
type: "BUDGET_EXTENDED",
|
|
1815
|
+
actor: KERNEL,
|
|
1816
|
+
ts: now(),
|
|
1817
|
+
data: {
|
|
1818
|
+
step: grant.step,
|
|
1819
|
+
amount: 1,
|
|
1820
|
+
...(grant.scope === "work" ? { scope: "work" } : {}),
|
|
1821
|
+
maxAttempts:
|
|
1822
|
+
grant.scope === "work"
|
|
1823
|
+
? context.reviewRoundsPerWork +
|
|
1824
|
+
(ledger.state.workReviewGrants ?? 0) +
|
|
1825
|
+
1
|
|
1826
|
+
: context.maxAttemptsFor(grant.step) + 1,
|
|
1827
|
+
approvalRef: approval.decisionRef,
|
|
1828
|
+
grants: plan,
|
|
1829
|
+
},
|
|
1830
|
+
});
|
|
1327
1831
|
}
|
|
1328
|
-
}
|
|
1329
|
-
const extended = new Set(applied.map((event) => `${event.data.step}:${event.data.scope ?? "run"}`));
|
|
1330
|
-
for (const grant of plan) {
|
|
1331
|
-
const key = `${grant.step}:${grant.scope}`;
|
|
1332
|
-
if (extended.has(key)) continue;
|
|
1333
|
-
extended.add(key);
|
|
1334
1832
|
ledger.append({
|
|
1335
|
-
type: "
|
|
1833
|
+
type: "RUN_STARTED",
|
|
1336
1834
|
actor: KERNEL,
|
|
1337
1835
|
ts: now(),
|
|
1338
|
-
data: {
|
|
1339
|
-
step: grant.step,
|
|
1340
|
-
amount: 1,
|
|
1341
|
-
...(grant.scope === "work" ? { scope: "work" } : {}),
|
|
1342
|
-
maxAttempts: grant.scope === "work"
|
|
1343
|
-
? context.reviewRoundsPerWork + (ledger.state.workReviewGrants ?? 0) + 1
|
|
1344
|
-
: context.maxAttemptsFor(grant.step) + 1,
|
|
1345
|
-
approvalRef: approval.decisionRef,
|
|
1346
|
-
grants: plan,
|
|
1347
|
-
},
|
|
1836
|
+
data: {},
|
|
1348
1837
|
});
|
|
1838
|
+
drive(context, step, { skipBoundaryOnce: true });
|
|
1839
|
+
return {
|
|
1840
|
+
runId,
|
|
1841
|
+
ledgerPath,
|
|
1842
|
+
state: ledger.state,
|
|
1843
|
+
resumed: true,
|
|
1844
|
+
reason: null,
|
|
1845
|
+
};
|
|
1349
1846
|
}
|
|
1350
|
-
ledger.append({ type: "RUN_STARTED", actor: KERNEL, ts: now(), data: {} });
|
|
1351
|
-
drive(context, step, { skipBoundaryOnce: true });
|
|
1352
|
-
return { runId, ledgerPath, state: ledger.state, resumed: true, reason: null };
|
|
1353
|
-
}
|
|
1354
1847
|
|
|
1355
|
-
|
|
1356
|
-
ledger.append({
|
|
1357
|
-
type: "RUN_INTERRUPTED",
|
|
1358
|
-
actor: KERNEL,
|
|
1359
|
-
ts: now(),
|
|
1360
|
-
data: { cause: "resume after process loss" },
|
|
1361
|
-
});
|
|
1362
|
-
const tree = readback(workspace.worktreePath);
|
|
1363
|
-
let startStep = null;
|
|
1364
|
-
if (state.currentStep) {
|
|
1365
|
-
const step = state.currentStep;
|
|
1366
|
-
const attempt = state.steps[step].attempts;
|
|
1848
|
+
// Crash recovery: the process died while RUNNING.
|
|
1367
1849
|
ledger.append({
|
|
1368
|
-
type: "
|
|
1850
|
+
type: "RUN_INTERRUPTED",
|
|
1369
1851
|
actor: KERNEL,
|
|
1370
1852
|
ts: now(),
|
|
1371
|
-
data: {
|
|
1853
|
+
data: { cause: "resume after process loss" },
|
|
1372
1854
|
});
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1855
|
+
const tree = readback(workspace.worktreePath);
|
|
1856
|
+
let startStep = null;
|
|
1857
|
+
if (state.currentStep) {
|
|
1858
|
+
const step = state.currentStep;
|
|
1859
|
+
const attempt = state.steps[step].attempts;
|
|
1860
|
+
ledger.append({
|
|
1861
|
+
type: "STEP_FINISHED",
|
|
1862
|
+
actor: KERNEL,
|
|
1863
|
+
ts: now(),
|
|
1864
|
+
data: { step, attempt, status: "crashed" },
|
|
1865
|
+
});
|
|
1866
|
+
if (tree.dirty) {
|
|
1867
|
+
context.waitHuman(`resume-${step}`, [
|
|
1868
|
+
`interrupted step ${step} left a dirty worktree; decide whether to keep or discard before rerunning`,
|
|
1869
|
+
]);
|
|
1870
|
+
return {
|
|
1871
|
+
runId,
|
|
1872
|
+
ledgerPath,
|
|
1873
|
+
state: ledger.state,
|
|
1874
|
+
resumed: true,
|
|
1875
|
+
reason: null,
|
|
1876
|
+
};
|
|
1877
|
+
}
|
|
1878
|
+
// An interrupted attempt says nothing about the candidate, so the step
|
|
1879
|
+
// itself reruns (the lost attempt still counts against its budget)
|
|
1880
|
+
// instead of settling as a step failure — routing a crash through the
|
|
1881
|
+
// failure edge dispatched a fixer with no verifier evidence (real
|
|
1882
|
+
// incident: deploy-18's verify worker was killed by a host timeout).
|
|
1883
|
+
startStep = step;
|
|
1884
|
+
} else if (tree.dirty) {
|
|
1885
|
+
context.waitHuman("resume-run", [
|
|
1886
|
+
"worktree is dirty at resume with no step in flight; human triage required",
|
|
1376
1887
|
]);
|
|
1377
|
-
return {
|
|
1888
|
+
return {
|
|
1889
|
+
runId,
|
|
1890
|
+
ledgerPath,
|
|
1891
|
+
state: ledger.state,
|
|
1892
|
+
resumed: true,
|
|
1893
|
+
reason: null,
|
|
1894
|
+
};
|
|
1895
|
+
} else {
|
|
1896
|
+
startStep =
|
|
1897
|
+
state.lastCheckpoint?.resumePoint?.step ?? state.run.entry ?? null;
|
|
1898
|
+
if (!startStep) {
|
|
1899
|
+
throw new OrchestratorError(
|
|
1900
|
+
"no checkpoint and no recorded entry; cannot derive a safe resume point",
|
|
1901
|
+
);
|
|
1902
|
+
}
|
|
1378
1903
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
// instead of settling as a step failure — routing a crash through the
|
|
1382
|
-
// failure edge dispatched a fixer with no verifier evidence (real
|
|
1383
|
-
// incident: deploy-18's verify worker was killed by a host timeout).
|
|
1384
|
-
startStep = step;
|
|
1385
|
-
} else if (tree.dirty) {
|
|
1386
|
-
context.waitHuman("resume-run", [
|
|
1387
|
-
"worktree is dirty at resume with no step in flight; human triage required",
|
|
1388
|
-
]);
|
|
1389
|
-
return { runId, ledgerPath, state: ledger.state, resumed: true, reason: null };
|
|
1390
|
-
} else {
|
|
1391
|
-
startStep = state.lastCheckpoint?.resumePoint?.step ?? state.run.entry ?? null;
|
|
1392
|
-
if (!startStep) {
|
|
1393
|
-
throw new OrchestratorError(
|
|
1394
|
-
"no checkpoint and no recorded entry; cannot derive a safe resume point",
|
|
1395
|
-
);
|
|
1904
|
+
if (startStep) {
|
|
1905
|
+
drive(context, startStep);
|
|
1396
1906
|
}
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1907
|
+
return {
|
|
1908
|
+
runId,
|
|
1909
|
+
ledgerPath,
|
|
1910
|
+
state: ledger.state,
|
|
1911
|
+
resumed: true,
|
|
1912
|
+
reason: null,
|
|
1913
|
+
};
|
|
1914
|
+
},
|
|
1915
|
+
{ workId: outerLedger.state.run.work, parallel: options.parallel === true },
|
|
1916
|
+
);
|
|
1403
1917
|
}
|