forgedock 1.40.0 → 1.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/bin/engine/outcome-codes.mjs +6 -3
  2. package/bin/engine-cli.mjs +40 -4
  3. package/bin/forgedock.mjs +22 -1
  4. package/bin/lean/batch.test.mjs +33 -1
  5. package/bin/lean/capwait.mjs +36 -0
  6. package/bin/lean/capwait.test.mjs +48 -0
  7. package/bin/lean/conformance.mjs +150 -2
  8. package/bin/lean/conformance.test.mjs +126 -0
  9. package/bin/lean/depcache.mjs +10 -2
  10. package/bin/lean/depcache.test.mjs +22 -17
  11. package/bin/lean/execution.mjs +21 -5
  12. package/bin/lean/execution.test.mjs +7 -1
  13. package/bin/lean/heavy.mjs +22 -1
  14. package/bin/lean/lanecheck.mjs +129 -0
  15. package/bin/lean/lanecheck.test.mjs +118 -0
  16. package/bin/lean/lanelock.mjs +31 -0
  17. package/bin/lean/lean.mjs +102 -30
  18. package/bin/lean/lean.test.mjs +173 -3
  19. package/bin/lean/orchestrate.mjs +42 -7
  20. package/bin/lean/phaseledger.mjs +116 -0
  21. package/bin/lean/phaseledger.test.mjs +72 -0
  22. package/bin/lean/prodwatch.mjs +12 -1
  23. package/bin/lean/prodwatch.test.mjs +32 -3
  24. package/bin/lean/prompts.mjs +19 -0
  25. package/bin/lean/render.mjs +10 -2
  26. package/bin/lean/sandbox-provider.contract.test.mjs +48 -0
  27. package/bin/lean/sandbox-remote.mjs +8 -1
  28. package/bin/lean/schedd.mjs +5 -1
  29. package/bin/lean/selftest-gate.mjs +46 -0
  30. package/bin/lean/selftest-gate.test.mjs +40 -0
  31. package/bin/lean/selftest-live.mjs +248 -0
  32. package/bin/lean/selftest-live.test.mjs +154 -0
  33. package/bin/lean/selftest-nightly.mjs +81 -0
  34. package/bin/lean/selftest-nightly.test.mjs +84 -0
  35. package/bin/lean/selftest-static.mjs +81 -0
  36. package/bin/lean/selftest-static.test.mjs +32 -0
  37. package/bin/pool/agent.mjs +16 -5
  38. package/bin/pool/router.mjs +17 -1
  39. package/bin/pool/warm.mjs +80 -0
  40. package/bin/review-staging.mjs +28 -6
  41. package/bin/stats/aggregate.mjs +2 -1
  42. package/bin/stats/efficiency.mjs +3 -1
  43. package/bin/stats/efficiency.test.mjs +4 -1
  44. package/bin/stats/github-source.mjs +2 -1
  45. package/bin/stats/local-source.mjs +1 -0
  46. package/bin/stats/render.mjs +2 -0
  47. package/commands/deploy-pr.md +3 -3
  48. package/commands/review-pr-staging.md +2 -2
  49. package/commands/work-on/build.md +2 -2
  50. package/docs/CONFIG.md +33 -0
  51. package/docs/OUTCOME-CODES.md +1 -1
  52. package/docs/PROD-WATCH.md +1 -1
  53. package/docs/RELEASE-NOTES.md +10 -0
  54. package/docs/SELFTEST.md +34 -0
  55. package/docs/eval/conformance-labels-v1.json +13 -0
  56. package/package.json +1 -1
  57. package/pool/image/Dockerfile +2 -1
  58. package/pool/image/entrypoint.sh +44 -11
  59. package/pool/image/warm-refresh.sh +14 -0
  60. package/scripts/conformance-calibrate.mjs +132 -0
  61. package/scripts/file-deferred-followups.sh +21 -8
  62. package/scripts/file-deferred-followups.test.sh +27 -6
@@ -0,0 +1,118 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ // #4866: lane preflight checks, the orchestrate admission hold, sandbox.clone typing and the heavy shim-loop finder.
4
+ import { describe, it } from "node:test";
5
+ import assert from "node:assert/strict";
6
+ import { mkdirSync, writeFileSync, chmodSync } from "node:fs";
7
+ import { join } from "node:path";
8
+ import { tmpDir } from "../tests/_tmp.mjs";
9
+ import { checkPoolAuth, checkPoolVersion, checkPathLoop, checkGhFields, availableFields, runLaneChecks, formatLaneChecks, UPGRADE_COMMAND } from "./lanecheck.mjs";
10
+ import { findShimLoops } from "./heavy.mjs";
11
+ import { orchestrate, deferPolicy } from "./orchestrate.mjs";
12
+ import { providerLanes, isCloneFailure } from "./execution.mjs";
13
+
14
+ const pool = { mode: "remote", provider: "pool", tokenEnv: null, endpoint: null };
15
+ const until = async (f, ms = 3000) => { const t = Date.now(); while (!f()) { if (Date.now() - t > ms) throw new Error("timeout"); await new Promise((r) => setTimeout(r, 5)); } };
16
+
17
+ describe("pool_auth", () => {
18
+ it("fails naming the sources tried when no token resolves, passes once it appears (no restart)", async () => {
19
+ let tok = null;
20
+ const run = () => checkPoolAuth({ exec: pool, router: { endpoint: "http://r", token: tok }, credential: () => null });
21
+ const a = await run();
22
+ assert.equal(a.ok, false); assert.equal(a.code, "preflight.pool_auth"); assert.match(a.detail, /COMPUTESHARE_TOKEN/); assert.match(a.detail, /forgedock_token_file/);
23
+ tok = "t";
24
+ assert.equal((await run()).ok, true);
25
+ });
26
+ it("fails on a 401 ping and passes when the router is unreachable or no remote execution", async () => {
27
+ const r = { endpoint: "http://r", token: "t" };
28
+ assert.equal((await checkPoolAuth({ exec: pool, router: r, ping: async () => ({ status: 401 }) })).ok, false);
29
+ assert.equal((await checkPoolAuth({ exec: pool, router: r, ping: async () => { throw new Error("down"); } })).ok, true);
30
+ assert.equal((await checkPoolAuth({ exec: { mode: "local" }, router: null })).ok, true);
31
+ });
32
+ });
33
+
34
+ describe("pool_version", () => {
35
+ it("reports the agent, image version, CLI version and upgrade command", () => {
36
+ const c = checkPoolVersion({ exec: pool, cliVersion: "1.39.0", agents: [{ name: "a1", ok: true, version: "1.37.0" }, { name: "a2", ok: true, version: "1.39.0" }] });
37
+ assert.equal(c.ok, false); assert.equal(c.code, "preflight.pool_version");
38
+ assert.match(c.detail, /a1/); assert.match(c.detail, /1\.37\.0/); assert.match(c.detail, /1\.39\.0/); assert.ok(c.detail.includes(UPGRADE_COMMAND)); assert.ok(!/a2/.test(c.detail));
39
+ assert.equal(checkPoolVersion({ exec: pool, cliVersion: "1.39.0", agents: [{ name: "a", ok: true, version: "1.39.0" }] }).ok, true);
40
+ });
41
+ });
42
+
43
+ describe("path_loop", () => {
44
+ const wrapper = "#!/bin/sh\nexec \"$(command -v docker)\" \"$@\"\n";
45
+ const mk = () => { const d = tmpDir("lc-"); const w = join(d, "wrap"); mkdirSync(w); writeFileSync(join(w, "docker"), wrapper); chmodSync(join(w, "docker"), 0o755); return { d, w }; };
46
+ it("names the wrapper dir when no real binary sits behind it", () => {
47
+ const { w } = mk();
48
+ const c = checkPathLoop({ env: { PATH: w } });
49
+ assert.equal(c.ok, false); assert.equal(c.code, "preflight.path_loop"); assert.ok(c.detail.includes(w));
50
+ });
51
+ it("passes when a real binary is behind the wrapper", () => {
52
+ const { d, w } = mk(); const real = join(d, "real"); mkdirSync(real); writeFileSync(join(real, "docker"), "#!/bin/sh\necho hi\n"); chmodSync(join(real, "docker"), 0o755);
53
+ assert.equal(checkPathLoop({ env: { PATH: `${w}:${real}` } }).ok, true);
54
+ assert.deepEqual(findShimLoops(`${w}:${real}`), []);
55
+ });
56
+ });
57
+
58
+ describe("gh_fields", () => {
59
+ const out = (fields) => ({ stdout: "", stderr: `Unknown JSON field: "x"\nAvailable fields:\n${fields.join("\n")}\n` });
60
+ const ghWith = (fields) => async (args) => (args[0] === "--version" ? { stdout: "gh version 2.40.0 (x)\n" } : out(fields[args[0] + " " + args[1]] ?? []));
61
+ const all = { "issue view": ["number", "title", "body", "state", "stateReason", "labels", "comments", "milestone", "author", "createdAt", "updatedAt"], "pr view": ["number", "state", "mergeable", "mergeStateStatus"], "repo view": ["nameWithOwner", "defaultBranchRef"] };
62
+ it("names the missing field and gh version, and passes when complete", async () => {
63
+ assert.equal((await checkGhFields({ exec: ghWith(all) })).ok, true);
64
+ const c = await checkGhFields({ exec: ghWith({ ...all, "issue view": all["issue view"].filter((f) => f !== "stateReason") }) });
65
+ assert.equal(c.ok, false); assert.equal(c.code, "preflight.gh_fields"); assert.match(c.detail, /stateReason/); assert.match(c.detail, /2\.40\.0/);
66
+ });
67
+ it("parses the available list and passes on unparseable output", async () => {
68
+ assert.deepEqual(availableFields("Available fields:\n a\n b\n"), ["a", "b"]);
69
+ assert.equal((await checkGhFields({ exec: async () => ({ stdout: "gh version 2.1.0\n", stderr: "boom" }) })).ok, true);
70
+ });
71
+ });
72
+
73
+ describe("runLaneChecks / formatLaneChecks", () => {
74
+ it("returns every check and prints PASS or FAIL per line", async () => {
75
+ const checks = await runLaneChecks({ exec: pool, env: { PATH: "" }, cliVersion: "1.39.0", io: { router: () => ({ endpoint: "http://r", token: null }), credential: () => null } });
76
+ assert.deepEqual(checks.map((c) => c.code), ["preflight.pool_auth", "preflight.pool_version", "preflight.path_loop", "preflight.gh_fields"]);
77
+ const lines = formatLaneChecks(checks).split("\n");
78
+ assert.equal(lines.length, 4); assert.match(lines[0], /^FAIL preflight\.pool_auth/); assert.match(lines[2], /^PASS preflight\.path_loop/);
79
+ });
80
+ });
81
+
82
+ describe("orchestrate preflight hold", () => {
83
+ it("admits no remote lane while pool_auth fails, says why once, never trips the breaker, and resumes when it clears", async () => {
84
+ let bad = true; const started = [], rows = [];
85
+ const done = orchestrate({ issues: [1], parallel: 2, pollMs: 5, readBody: async () => "", onUpdate: (r) => rows.push(r), breaker: { errors: 1, window_min: 30 },
86
+ remote: { max: 1, isRemote: () => true, place: () => ({ where: "remote", reason: "t" }), admit: () => ({ ok: true }) },
87
+ preflight: async () => (bad ? [{ code: "preflight.pool_auth", detail: "no token", scope: "remote" }] : []),
88
+ runOne: async (n) => { started.push(n); return { terminalReason: "merged" }; } });
89
+ await new Promise((r) => setTimeout(r, 60));
90
+ assert.deepEqual(started, []);
91
+ assert.equal(rows.filter((r) => r.code === "preflight.pool_auth").length, 1);
92
+ assert.ok(!rows.some((r) => r.outcome === "breaker"));
93
+ bad = false;
94
+ await until(() => started.length === 1);
95
+ await done;
96
+ assert.ok(rows.some((r) => /preflight cleared/.test(r.detail ?? "")));
97
+ });
98
+ });
99
+
100
+ describe("sandbox.clone", () => {
101
+ const prov = (over) => ({ name: "pool", idFor: (n) => `sb-${n}`, start: async () => ({}), status: async () => "exited", fetchResult: async () => null, teardown: async () => true, ...over });
102
+ const lanes = (p, extra = {}) => providerLanes({ provider: p, laneArgs: () => [], ledgerFor: () => null, pollMs: 1, sleep: async () => {}, ...extra });
103
+ it("a lane that exited without a result after a clone timeout ends deferred sandbox.clone with retry-in", async () => {
104
+ const r = await lanes(prov(), { logTail: () => "fatal: git clone timed out after 300s" }).runOne(7, {}, {});
105
+ assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "sandbox.clone"); assert.ok(r.retryInSeconds > 0);
106
+ });
107
+ it("an engine-error result naming a clone failure is retyped; other failures stay engine-error", async () => {
108
+ const r = await lanes(prov({ fetchResult: async () => ({ terminalReason: "engine-error", detail: "git clone failed: early EOF" }) })).runOne(7, {}, {});
109
+ assert.equal(r.code, "sandbox.clone");
110
+ const o = await lanes(prov()).runOne(7, {}, {});
111
+ assert.equal(o.terminalReason, "engine-error");
112
+ assert.equal(isCloneFailure("clone of repo ok"), false);
113
+ });
114
+ it("is requeued with its retry-in and is not an engine-error (breaker input)", () => {
115
+ const p = deferPolicy({ terminalReason: "deferred", code: "sandbox.clone", retryInSeconds: 120 }, { now: 0 });
116
+ assert.equal(p.delayMs, 120000);
117
+ });
118
+ });
@@ -0,0 +1,31 @@
1
+ /**
2
+ * #4920: who else owns an issue's lane, read from GitHub (never host-local state), so a second batch, a remote
3
+ * sandbox or a restarted daemon sees the same answer. A live FORGE:STATE lease held by another lane, or an open
4
+ * `feat/issue-N-lean` PR that is not ours, means another lane owns the issue. Generic: no repo or pool logic.
5
+ */
6
+ import { parseState } from "../engine/state.mjs";
7
+ import { MAX_LEASE_TTL_MS } from "../engine/projector.mjs";
8
+
9
+ /** The live lease on an issue body held by someone other than `selfId` (null: none, expired, ours or implausibly far out). */
10
+ export function foreignLease(body, { selfId = null, now = Date.now() } = {}) {
11
+ const l = parseState(body)?.lease;
12
+ if (!l || !Number.isFinite(l.until) || l.until <= now || l.until > now + MAX_LEASE_TTL_MS) return null;
13
+ return l.by && l.by !== selfId ? l : null;
14
+ }
15
+
16
+ /**
17
+ * @param {{gh: Function, repo: string, issue: number, branch?: string, selfId?: string|null, ownPr?: number|null, now?: number, checkLease?: boolean, checkPr?: boolean}} a
18
+ * @returns {Promise<{kind: "lease"|"pr", detail: string}|null>} null when nothing else owns it or GitHub is unreadable (fail-open)
19
+ */
20
+ export async function otherLaneOwner({ gh, repo, issue, branch = `feat/issue-${issue}-lean`, selfId = null, ownPr = null, now = Date.now(), checkLease = true, checkPr = true }) {
21
+ const json = async (args) => { try { return JSON.parse(String(await gh(args))); } catch { return null; } };
22
+ if (checkLease) {
23
+ const iss = await json(["issue", "view", String(issue), "-R", repo, "--json", "body"]);
24
+ const l = foreignLease(iss?.body ?? "", { selfId, now });
25
+ if (l) return { kind: "lease", detail: `issue ${issue} leased by ${l.by} until ${new Date(l.until).toISOString()}` };
26
+ }
27
+ if (!checkPr) return null;
28
+ const prs = await json(["pr", "list", "-R", repo, "--head", branch, "--state", "open", "--json", "number", "--limit", "5"]);
29
+ const other = Array.isArray(prs) ? prs.find((p) => Number.isInteger(p?.number) && p.number !== ownPr) : null;
30
+ return other ? { kind: "pr", detail: `PR #${other.number} is open on ${branch} from another lane` } : null;
31
+ }
package/bin/lean/lean.mjs CHANGED
@@ -9,11 +9,12 @@ import { join, dirname, basename, delimiter } from "node:path";
9
9
  import { hostname as osHostname } from "node:os";
10
10
  import { appendEvent } from "../engine/runlog.mjs";
11
11
  import { readSelfReview, compareSelfReview } from "./selfreview.mjs";
12
- import { codeForTerminalReason, OUTCOME_CODES } from "../engine/outcome-codes.mjs";
12
+ import { codeForTerminalReason, OUTCOME_CODES, ENV_VALIDATE_TIMEOUT } from "../engine/outcome-codes.mjs";
13
13
  import { ciPollDelayMs, ENGINE_VERSION, DEFAULT_MAX_SESSION_LIMIT_PAUSES, DEFAULT_SESSION_LIMIT_WAIT_MS, parseLeaseHolder, sanitizeLeaseHost } from "../engine.mjs";
14
14
  import { openDependency, laneMergedPrs, dependencyOnLane, isTrustedComment, decomposedSubissues, isDecomposedRecord, reopenRevokedDecomposition } from "../engine/phases.mjs";
15
- import { createHash } from "node:crypto";
16
- import { TRACKED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
15
+ import { createHash, randomBytes } from "node:crypto";
16
+ import { otherLaneOwner } from "./lanelock.mjs";
17
+ import { TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
17
18
  import { checkAdmission, hostParallelism, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
18
19
  import { namedPaths, isTrustedIssueComment } from "./prefilter.mjs";
19
20
  import { checksAt } from "./postdeploy-check.mjs";
@@ -28,7 +29,7 @@ import { toolStats } from "./toolstats.mjs";
28
29
  import { readTestMap, readTestMapAtRef, mergeChecks, factsPath, TEST_MAP_FILE } from "./testmap.mjs";
29
30
  import { runChecks, runWithTimeoutRetry, planPrecheck, worktreeFingerprint, selfCheckState, quietFailure, CHECK_PLAN_FILE, SELF_CHECK_FILE, CHECK_LOG_DIR } from "./precheck.mjs";
30
31
  import { retrying, ghCacheRetry, refreshAuthOnce, runRefreshScript, GH_AUTH_RE, GH_AUTH_RETRY_SECONDS, PUSH_TRANSIENT_RE, PUSH_PERMANENT_RE, GH_TRANSIENT_RE, MERGE_TRANSIENT_RE, MERGE_CONFLICT_RE } from "./retry.mjs";
31
- import { renderInvestigation, renderImplementation, renderReview, renderPrBody, renderTrajectory, renderContract, renderAmendments, renderShadowReview, parseContractComment, contractJson, capComment, sanitizeMd, MAX_COMMENT_CHARS, recoveryEvidence } from "./render.mjs";
32
+ import { renderInvestigation, renderImplementation, renderReview, renderPrBody, renderTrajectory, renderContract, renderAmendments, renderShadowReview, parseContractComment, parseAmendmentsComment, contractJson, capComment, sanitizeMd, MAX_COMMENT_CHARS, recoveryEvidence } from "./render.mjs";
32
33
  import { parseVerifierConfig, pickVerifierModel, changedSymbols, callersOf, verifierBudget, verifierPrompt, parseVerifierFindings, findingsToFailures, renderVerifierNote, VERIFIER_SYSTEM } from "./verifier.mjs";
33
34
  import { computeTrigger, riskPathHits, TRIGGER_MODES } from "./review-trigger.mjs";
34
35
  import { hostCapabilityBlock, contractHostNeeds } from "./execution.mjs";
@@ -36,7 +37,7 @@ import { hostCapabilityBlock, contractHostNeeds } from "./execution.mjs";
36
37
  const MISSING_TOOL_RE = /command not found|not found in PATH|no such file or directory.*\b(?:docker|promtool)\b|\b(?:docker|promtool)\b[^\n]{0,80}(?:not (?:installed|found|available)|no such file|cannot connect|daemon)|cannot connect to the docker daemon|no docker daemon|no module named|ModuleNotFoundError|\bENOENT\b[^\n]{0,60}spawn/i;
37
38
  /** #4685: true when a builder's block summary names a missing tool or module (environment), not a credential. */
38
39
  export function missingToolBlock(summary) { return MISSING_TOOL_RE.test(String(summary ?? "")); }
39
- import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, sequenceSyncPrompt, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt, i18nCopiesPrompt } from "./prompts.mjs";
40
+ import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, decisionNote, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, sequenceSyncPrompt, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt, i18nCopiesPrompt } from "./prompts.mjs";
40
41
  import { postMergeCommands } from "./postmerge.mjs";
41
42
  import { guardAfterMerge, withHistoryRows } from "./guards.mjs";
42
43
  import { loadGuards, selectGuards, runGuards, supersededIds, supersessionComment } from "./guardgate.mjs";
@@ -63,6 +64,7 @@ import { parsePredicate, evalPredicate, premiseAnnotation, registerAsk, registry
63
64
  import { lightEligibility } from "./light.mjs";
64
65
  import { untrackToolState, unstageToolState, isToolState, clearLaneScratch } from "./toolstate.mjs";
65
66
  import { ensureSignedOff } from "./signoff.mjs";
67
+ import { createLedger, flattenPhases } from "./phaseledger.mjs";
66
68
  import { runLightStep, runLightTests, lightBuildPrompt, upgradeNote, renderLiteReceipt } from "./lightlane.mjs";
67
69
 
68
70
  /**
@@ -77,11 +79,12 @@ export const RECOVER_HOW = Object.freeze({
77
79
  [OUTCOME_CODES.BUDGET_EXCEEDED]: "an earlier run spent the implement turn budget for this issue's size; its digest is in the earlier run's detail and the branch holds its checkpointed work. Continue from the checkpoint against the frozen contract without starting over; if the remaining work cannot fit the budget, stop with a typed split-required block naming the sub-issues instead of grinding",
78
80
  [OUTCOME_CODES.CONTRACT_INVALID_COMMAND]: "a frozen validate command was rejected by its runner's CLI (vitest 3 rejects -q). The change may already be correct: rerun the validate with the runner's own flags (vitest: --reporter=dot) and amend the command, do not rewrite the deliverables",
79
81
  [OUTCOME_CODES.ENV_TOOL_MISSING]: "a validate command could not run because the host lacks the tool or module named in the earlier run's detail; the change may already be correct. Do not edit the deliverables for it and never install anything globally: run the same check through the repo's own toolchain (its virtualenv, lockfile-pinned runner or package scripts), record that exit under the same validate id and name the command you used in your summary",
82
+ [ENV_VALIDATE_TIMEOUT]: "a validate command timed out (exit 124) on the engine's rerun, even after one longer retry: host load, not a failed check; the change may already be correct. Do not edit the deliverables for it: run the same command narrowed to the changed files (or through the repo's own cached toolchain) with a longer timeout, record that exit under the same validate id and name the command in your summary",
80
83
  [OUTCOME_CODES.ENV_CAPABILITY_REMOTE]: "the earlier run ran in a remote sandbox that lacks a host-only capability (SOPS/age key, ops credentials, read-only prod monitor SSH); this host has it. Do the work that needed it now, with the host's own tools and credentials, and never print or commit a secret in the clear",
81
84
  [OUTCOME_CODES.ENV_REPRO_UNRUNNABLE]: "the repro harness could not run the contract's repro_test on head (timeout, missing runner or dependencies); the change may already be correct. Do not edit the deliverables or stub dependencies for it and never install anything globally: run the repro command through the repo's own toolchain and installed dependencies, and when it cannot run in this checkout, fix the repro_test command (not the code) so it does",
82
85
  });
83
86
  /** #4323/#4334: precheck failure codes that are environment errors (state "environment"), never a builder resume. */
84
- const ENV_CODES = Object.freeze([OUTCOME_CODES.ENV_TOOL_MISSING, OUTCOME_CODES.ENV_REPRO_UNRUNNABLE, OUTCOME_CODES.CONTRACT_INVALID_COMMAND]);
87
+ const ENV_CODES = Object.freeze([OUTCOME_CODES.ENV_TOOL_MISSING, OUTCOME_CODES.ENV_REPRO_UNRUNNABLE, ENV_VALIDATE_TIMEOUT, OUTCOME_CODES.CONTRACT_INVALID_COMMAND]);
85
88
  export const DEFAULT_REVIEWER_MODEL = "claude-sonnet-5-5";
86
89
  /** forge.yaml engine.conformance: "shadow" (default) posts and logs the gate's verdict without enforcing it; "enforce" blocks on it. */
87
90
  export const CONFORMANCE_MODES = Object.freeze(["shadow", "enforce"]);
@@ -187,6 +190,8 @@ export function isProtectedLaneBranch(b, { issue, lane = null, branches = {}, de
187
190
  * @param {object} [o.baseMerge] forge.yaml lean.base_merge_* (#4588): {unionGlobs, jsonGlobs, generatedGlobs, generatedCommand} for deterministic base-merge conflict resolution
188
191
  * @param {number} [o.ciMaxMinutes] remote-CI wait budget (default 360); exhaustion is NOT needs-human
189
192
  * @param {"shadow"|"enforce"} [o.conformance] conformance gate mode (default shadow)
193
+ * @param {"off"|"report"} [o.conformanceMutation] forge.yaml engine.conformance_mutation (#4913): rerun each evidence test with the change reverted; a survivor is conformance.mutation_survived (shadow unless conformance_kinds enforces it)
194
+ * @param {Record<string,"enforce"|"shadow">} [o.conformanceKinds] forge.yaml engine.conformance_kinds: a failure code's own mode (enforce blocks, shadow is recorded only); others follow `conformance`
190
195
  * @param {"needs-human"|"degraded"} [o.reviewIncomplete] shadow only: what an incomplete review after the re-ask does (default needs-human)
191
196
  * @param {object} [o.reviewTrigger] forge.yaml review.trigger/risk_paths (#4273): {mode: shadow (default)|enforce|off, riskPaths: globs}
192
197
  * @param {object} [o.verifier] forge.yaml review.verifier/verifier_model (#4397): {mode: off (default)|shadow|enforce, model}
@@ -212,7 +217,7 @@ export function isProtectedLaneBranch(b, { issue, lane = null, branches = {}, de
212
217
  */
213
218
  export async function runCore(o) {
214
219
  const { issue, repo, lane: requestedLane = null, laneOverride = false, branches = {}, cwd, dir, io, builderModel = null, reviewerModel = DEFAULT_REVIEWER_MODEL, fixModel = null,
215
- toolchainOverride = null, envSetup = null, ciMaxMinutes = 360, conformance = "shadow", reviewIncomplete = "needs-human", reproTest = "prefer", closeParent = false, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, cliIdleMinutes = 15, cliMaxMinutes = 180, cliBusyMinutes = 45, contextCeilingTokens = 120000, turnBudgets = null, laneLimits = null, maxMergeRetries = 5, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
220
+ toolchainOverride = null, envSetup = null, ciMaxMinutes = 360, conformance = "shadow", conformanceKinds = null, conformanceMutation = "off", reviewIncomplete = "needs-human", reproTest = "prefer", closeParent = false, mergeWait = "required", maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES, cliIdleMinutes = 15, cliMaxMinutes = 180, cliBusyMinutes = 45, contextCeilingTokens = 120000, turnBudgets = null, laneLimits = null, maxMergeRetries = 5, sessionLimitMaxWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS, onProgress = () => {}, emitTelemetry = null, random = Math.random } = o;
216
221
  const now = io.now ?? (() => Date.now());
217
222
  const vcfg = { ...parseVerify({}), ...(o.verify ?? {}) };
218
223
  const learnedCommands = (Array.isArray(o.precheck?.commands) ? o.precheck.commands : []).map((c) => String(c ?? "").trim()).filter(Boolean);
@@ -232,6 +237,7 @@ export async function runCore(o) {
232
237
  const branch = `feat/issue-${issue}-lean`;
233
238
  const wt = join(o.worktreeRoot ?? join(cwd, ".claude", "worktrees"), `lean-${issue}`);
234
239
  const t0 = now();
240
+ const phaseLedger = createLedger(now, "sandbox_start", t0); // #4908: where the lane wall time goes
235
241
  const usage = { cost_usd: 0, input: 0, output: 0, cache: 0 };
236
242
  const sessionCosts = new Map(); // #4407: session id -> last reported cumulative cost
237
243
  let sessionId = null;
@@ -260,6 +266,7 @@ export async function runCore(o) {
260
266
  if (phase === "implement") { builderTriggers.push({ trigger, restart_of: restartOf }); impl.cost += c.delta ?? 0; if (r?.model && r.model !== "default") impl.models.add(r.model); tb.note(num(r?.turns), stage === "implement"); }
261
267
  ev(rec); await emit(rec);
262
268
  modelMs += rec.duration_ms; // #4451: model time next to the lane's wall time
269
+ phaseLedger.addModel(phase === "review" ? "review" : stage === "plan" ? "plan" : "implement", rec.duration_ms, c.delta);
263
270
  const u = r?.usage ?? {};
264
271
  usage.cost_usd += c.delta ?? 0;
265
272
  usage.input += u.input_tokens ?? 0; usage.output += u.output_tokens ?? 0;
@@ -275,8 +282,9 @@ export async function runCore(o) {
275
282
  const usageLine = () => `cost $${usage.cost_usd.toFixed(4)}, tokens ${usage.input} in / ${usage.output} out / ${usage.cache} cache, wall ${Math.round((now() - t0) / 60000 * 10) / 10} min`;
276
283
 
277
284
  // ---- lease (same FORGE:STATE lease the full path uses) ----
278
- const agentId = o.agentId ?? `core_${process.pid}`;
279
285
  const myHost = o.hostName ?? osHostname();
286
+ // #4920: unique per lane. A sandbox's pid is 1, so `core_<pid>` made every remote lane the same holder and the lease never refused one.
287
+ const agentId = o.agentId ?? `core_${sanitizeLeaseHost(myHost)}_${o.pid ?? process.pid}_${randomBytes(3).toString("hex")}`;
280
288
  const leaseObj = () => ({ by: agentId, until: now() + leaseTtlMs, host: myHost, pid: o.pid ?? process.pid });
281
289
  /** Provably dead: same host and `kill(pid, 0)` throws ESRCH. A different host keeps the TTL behaviour. */
282
290
  const holderDead = (l) => {
@@ -313,6 +321,7 @@ export async function runCore(o) {
313
321
  let factsSeen = []; // #4401: trusted comment bodies on the issue, so an identical FORGE:FACTS is never posted twice
314
322
  let guardAddedMs = 0; // #4705 (R12): wall time the guard gate added, summed over the lane, carried on RUN_TERMINAL
315
323
  let shadowWould = null; // forge#4165: the shadow gate's verdict, carried on RUN_TERMINAL (no telemetry write of its own)
324
+ const kinds = normalizeKindModes(conformanceKinds); // engine.conformance_kinds: a failure code's own mode
316
325
  let riskEnforced = null, gateRan = false; // #4398: declared risk paths that made a shadow lane enforce (sticky); the gate ran
317
326
  let done = null, finishing = null, wtCreated = false, salvagedRef = null;
318
327
  let light = false, modelMs = 0; // #4451: the light lane finished without an upgrade (the same structure/hotspot gates as the full lane, #4502); summed session time
@@ -438,8 +447,9 @@ export async function runCore(o) {
438
447
  conformance_mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", ...(shadowWould ? { shadow_would: shadowWould } : {}),
439
448
  context_pack: ctxCfg.mode, ...ctxAb, ...expTag, explore: exploreCfg.mode, wall_ms: now() - t0, ...(guardAddedMs ? { guard_added_ms: guardAddedMs } : {}), model_ms: modelMs, light, ...(ctxLane ? { lane_type: ctxLane } : {}), ...(pitMatched != null ? { pitfalls_matched: pitMatched } : {}), ...(tb.record() ? { turns: tb.record() } : {}),
440
449
  ...(route ? { builder_tier: route.tier, builder_class: route.class, builder_escalated: escalated, builder_model: route.model ?? "default" } : {}) }; // forge#4275: the on/off A/B groups runs by these; #4394: per-tier stats
441
- ev(term);
442
- await emit(term);
450
+ const phases = phaseLedger.snapshot(); // #4908: run log gets the nested ledger, FORGE:TELEMETRY the flattened scalars
451
+ ev({ ...term, phases });
452
+ await emit({ ...term, ...flattenPhases(phases) });
443
453
  // #4297: only a final failure the runner never requeues puts needs-human back (needsHuman() relabels on its own path). invalid,
444
454
  // decomposed and dependency-* are legitimate non-failure endings; deferred/engine-error may be requeued, so the batch runner
445
455
  // (orchestrate settle, relabelNeedsHuman) relabels those once they are final. #4302: a non-final attempt never relabels; the runner
@@ -462,12 +472,12 @@ export async function runCore(o) {
462
472
  try { await post(["FORGE:TRAJECTORY", REGISTER_MARKER], renderTrajectory({ ...T, usage: usageLine() })); } catch { /* best effort */ }
463
473
  }
464
474
  await releaseLease(reason, code, extra.mergedTo ?? null);
465
- done = { terminalReason: reason, detail, branch, worktree: wt, cost_usd: usage.cost_usd, tokens: { input: usage.input, output: usage.output, cache: usage.cache }, wall_ms: now() - t0, model_ms: modelMs,
475
+ done = { terminalReason: reason, detail, branch, worktree: wt, cost_usd: usage.cost_usd, tokens: { input: usage.input, output: usage.output, cache: usage.cache }, wall_ms: now() - t0, model_ms: modelMs, phases: phaseLedger.snapshot(),
466
476
  light, ...(lightVerdict ? { light_eligibility: lightVerdict } : {}), ...(lightUpgrade ? { light_upgrade: lightUpgrade } : {}), // #4453
467
477
  ...(Object.keys(waits).length ? { waits: { ...waits } } : {}), ...(heavyQueueMax ? { heavyQueueMax } : {}), ...(precheckRan ? { precheckMs, precheckDeferred } : {}),
468
478
  ...(firstPassOf(builderTriggers) != null ? { firstPass: firstPassOf(builderTriggers) } : {}), ...(tb.record() ? { turns: tb.record() } : {}), // #4392
469
479
  ...(route ? { builderRoute: { tier: route.tier, class: route.class, escalated, model: route.model ?? "default" } } : {}),
470
- ...(gateRan ? { conformance: { mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", would: shadowWould, risk: Boolean(riskEnforced) } } : {}), ...extra };
480
+ ...(gateRan ? { conformance: { mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", would: shadowWould, risk: Boolean(riskEnforced), ...(Object.keys(kinds).length ? { kinds: [...failureClasses] } : {}) } } : {}), ...extra };
471
481
  return done;
472
482
  }
473
483
  /** #4401: post the lane's FORGE:FACTS (result.json facts and validation, changed files, contract decisions); never fails a run. */
@@ -686,6 +696,7 @@ export async function runCore(o) {
686
696
  const tcOpts = () => ({ toolchain: toolchain?.summary ?? null, sparse, ...(facts ? { facts } : {}) });
687
697
  let hotFacts = null; // forge#4195: scripts/hotspots.sh facts for the files the issue names, read once before the plan call
688
698
  let enumList = []; // #4322: tests enumerating the members of the paths the issue names, read once before the plan call
699
+ let decisionsNote = ""; // #4901: trusted FORGE:DECISION comments newer than the frozen contract, "" otherwise
689
700
  let resumeNote = ""; // #4400: the resume digest of the last run (resume.mjs), "" on a first run or with context_pack off
690
701
  const implementRestart = () => builderImplementPrompt(vcfg.mode, tcOpts()) + hotspotBuildRule(hotFacts) + enumeratingBuildRule(enumList) + invariantsBuildRule(vcfg.invariants) + exploreBuildRule(exploreCfg) + CONTINUE_NOTE.replace("BASE", lane) + resumeNote;
691
702
  /** #4390: builder calls stopped at the context ceiling restart fresh; planning (and its corrections) run uncapped. */
@@ -976,6 +987,12 @@ export async function runCore(o) {
976
987
  leaseState = { ...(st ?? { v: 1, run: `core_${issue}`, issue, lane, committed: [], phase: null, branch: null, pr: null, terminalReason: null }), terminal: false,
977
988
  ...(o.placement ? { placement: o.placement } : {}) }; // #4256: the latest placement replaces an earlier run's
978
989
  try { await writeLease(leaseObj()); } catch { /* best effort */ }
990
+ try { // #4920: read it back; a lane that wrote at the same moment and won leaves its id here, so this one defers
991
+ const back = await io.projector.readState(issue);
992
+ if (back?.lease && back.lease.until > now() && back.lease.by !== agentId && back.lease.by !== st?.lease?.by) { // a holder this lane just took over (dead) is not a racer
993
+ return { terminalReason: "deferred", code: "lease-held", detail: `issue ${issue} leased by ${back.lease.by} until ${new Date(back.lease.until).toISOString()}`, branch, worktree: wt };
994
+ }
995
+ } catch { /* unreadable: proceed */ }
979
996
  leaseTimer = setInterval(() => { writeLease(leaseObj()).catch(() => {}); }, o.leaseRenewMs ?? 120000);
980
997
  leaseTimer.unref?.();
981
998
  }
@@ -1156,6 +1173,7 @@ export async function runCore(o) {
1156
1173
  // #4220: link node_modules/.venv from the shared cache keyed by lock hash (or install once and publish). Never fails the run.
1157
1174
  // After 2b (#4347): installs run with the toolchain env, so a pm only its shim provides resolves; no env is the host PATH.
1158
1175
  if (io.deps?.prepare) {
1176
+ phaseLedger.enter("deps_install");
1159
1177
  let res;
1160
1178
  try { res = await io.deps.prepare({ wt, repoKey: repoKey(cwd), env: toolchain?.env ?? undefined }); } catch (e) { res = [{ status: "failed", eco: null, key: null, ms: 0, reason: String(e?.message ?? e).slice(0, 300) }]; }
1161
1179
  for (const r of Array.isArray(res) ? res : []) {
@@ -1226,6 +1244,7 @@ export async function runCore(o) {
1226
1244
  };
1227
1245
 
1228
1246
  // ---- 3. builder: plan call -> frozen contract -> implement call in the same session ----
1247
+ phaseLedger.enter("plan");
1229
1248
  let result = null, attemptN = 0, reviewN = 0;
1230
1249
  const lastSubject = resumed ? String(await git(["log", "-1", "--format=%s"]).catch(() => "")).trim() : "";
1231
1250
  const fromCheckpoint = lastSubject.startsWith("wip(forge): checkpoint");
@@ -1361,14 +1380,21 @@ export async function runCore(o) {
1361
1380
  let out = "";
1362
1381
  try { out = String(await ghc(["api", "--paginate", `repos/${repo}/issues/${issue}/comments`, "--jq", ".[] | {body, author_association, user: {type: .user.type, login: .user.login}}"])); } catch { return null; }
1363
1382
  let found = null;
1383
+ const amendLog = []; // #4899: trusted FORGE:AMENDMENT payloads in order, bound to a contract hash
1364
1384
  for (const l of out.split("\n")) {
1365
1385
  let c; try { c = JSON.parse(l); } catch { continue; }
1366
1386
  if (!isTrustedComment(c)) continue;
1367
1387
  const b = String(c?.body ?? "");
1388
+ const am = parseAmendmentsComment(b);
1389
+ if (am) amendLog.push(am);
1368
1390
  if (!b.startsWith("<!-- FORGE:CONTRACT -->")) { if (found && b.trim() && !b.trimStart().startsWith("<!-- FORGE:")) found.humanAfter = true; continue; }
1369
1391
  const fc = parseContractComment(b);
1370
1392
  if (fc && contractHash(fc.text) === fc.hash) found = { ...fc, humanAfter: false };
1371
1393
  }
1394
+ if (found) {
1395
+ const seen = new Set(); found.amendments = [];
1396
+ for (const st of amendLog) if (st.contract === found.hash) for (const a of st.amendments) { const k = JSON.stringify(a); if (!seen.has(k)) { seen.add(k); found.amendments.push(a); } }
1397
+ }
1372
1398
  return found;
1373
1399
  }
1374
1400
 
@@ -1436,6 +1462,7 @@ export async function runCore(o) {
1436
1462
  const nowOk = (await io.docsFreshness({ cwd: wt, base }).catch(() => null))?.ok === true;
1437
1463
  for (const a of impliedDocDeliverables({ before, after: await changedFiles(), contract: frozen, wasFailing, nowOk })) if (!impliedDocs.some((x) => x.path === a.path)) impliedDocs.push(a);
1438
1464
  }
1465
+ let priorAmendments = []; // #4899: accepted amendments recovered with the frozen contract
1439
1466
  const pendingPits = []; // #4375: rework rounds whose gate has not passed again yet
1440
1467
  async function notePitfall(trigger, text) {
1441
1468
  const pit = io.pitfalls ? pendingPitfall(trigger, text) : null;
@@ -1493,7 +1520,7 @@ export async function runCore(o) {
1493
1520
  if (stale) {
1494
1521
  ev({ event: "CONTRACT_STALE", hash: fc.hash, scope: fc.scope, now: scopeHash, reason: stale });
1495
1522
  T.decisions.push(`Rerun: frozen contract ${fc.hash} is stale (${stale}); re-planning`);
1496
- } else { frozen = fc.contract; hash = fc.hash; T.decisions.push(`Rerun: frozen contract ${hash} recovered from the issue`); }
1523
+ } else { frozen = fc.contract; hash = fc.hash; priorAmendments = fc.amendments ?? []; T.decisions.push(`Rerun: frozen contract ${hash} recovered from the issue`); }
1497
1524
  }
1498
1525
  }
1499
1526
  if ((resumed || salvagedRef || recover) && ctxCfg.mode !== "off") {
@@ -1505,6 +1532,8 @@ export async function runCore(o) {
1505
1532
  T.decisions.push(`Resume digest given to the builder: ${d.files} file(s), ${d.validates} validate result(s)${d.reason ? `, last run ${d.reason}` : ""}`);
1506
1533
  }
1507
1534
  }
1535
+ decisionsNote = frozen && (resumed || salvagedRef || recover) ? decisionNote(trustedComments) : ""; // #4901
1536
+ if (decisionsNote) T.decisions.push("Trusted FORGE:DECISION comment(s) posted after the contract froze are binding notes for the builder");
1508
1537
  await label("building");
1509
1538
  // #4450: the light lane (lean.light mode on, eligible, fresh lane): one builder session, derived contract, or an upgrade
1510
1539
  const elig = lightEligibility({ title: iss.title, body: iss.body ?? "", labels: iss.labels ?? [], predictedFiles: namedPaths(iss.body ?? ""), riskPaths, cfg: o.light ?? {} });
@@ -1520,7 +1549,7 @@ export async function runCore(o) {
1520
1549
  const planPrompt = () => builderPlanPrompt(issue, tcOpts()) + hotspotPlanNote(hotFacts) + enumeratingPlanNote(enumList) + invariantsPlanNote(vcfg.invariants) + reproPlanNote(reproOn()) + explorePlanRule(exploreCfg) + (ctxCfg.mode === "on" ? contextPackNote(ctxPack) : "");
1521
1550
  let lightDone = false, lightCommits = false;
1522
1551
  if (lightTry) {
1523
- liveKeys = ["implementation"]; stage = "implement";
1552
+ liveKeys = ["implementation"]; stage = "implement"; phaseLedger.enter("implement");
1524
1553
  tb.setSize(recoverySize(laneSize({ labels: (iss?.labels ?? []).map((l) => l?.name ?? l) }), recover)); // #4505
1525
1554
  const lr = await runLightStep({ title: iss.title, issueText, riskPaths, predictedFiles: namedPaths(iss.body ?? ""), cfg: o.light ?? {}, validate: (c) => validateContract(c, issueText, { repro: reproOn() }),
1526
1555
  build: () => builder(++attemptN, lightBuildPrompt(issue, tcOpts()) + hotspotBuildRule(hotFacts) + enumeratingBuildRule(enumList) + invariantsBuildRule(vcfg.invariants), null, true, undefined, { trigger: "build" }),
@@ -1555,7 +1584,7 @@ export async function runCore(o) {
1555
1584
  }
1556
1585
  }
1557
1586
  if (!frozen) {
1558
- await builder(++attemptN, planPrompt() + (hasCommits || lightCommits ? PLAN_RERUN_NOTE.replace("BASE", lane) : "") + recoverNote() + resumeNote, null, true, planPrompt, { trigger: "plan" });
1587
+ await builder(++attemptN, planPrompt() + (hasCommits || lightCommits ? PLAN_RERUN_NOTE.replace("BASE", lane) : "") + recoverNote() + resumeNote + decisionsNote, null, true, planPrompt, { trigger: "plan" });
1559
1588
  planned = true;
1560
1589
  result = await readResult();
1561
1590
  const evidence = (Array.isArray(result?.evidence) ? result.evidence : []).filter((x) => typeof x === "string" && x.trim());
@@ -1645,9 +1674,9 @@ export async function runCore(o) {
1645
1674
  ev(rec); await emit(rec);
1646
1675
  T.decisions.push(`Builder routed to the ${route.tier} tier (${route.class}; ${route.model ?? "CLI default"})`);
1647
1676
  }
1648
- const implementPrompt = builderImplementPrompt(vcfg.mode, tcOpts()) + hotspotBuildRule(hotFacts) + enumeratingBuildRule(enumList) + invariantsBuildRule(vcfg.invariants) + exploreBuildRule(exploreCfg) + (planned ? "" : (fromCheckpoint ? CONTINUE_NOTE : RERUN_NOTE).replace("BASE", lane)) + (salvagedRef ? SALVAGE_NOTE.replaceAll("REF", salvagedRef) : "") + recoverNote() + (planned ? "" : resumeNote);
1677
+ const implementPrompt = builderImplementPrompt(vcfg.mode, tcOpts()) + hotspotBuildRule(hotFacts) + enumeratingBuildRule(enumList) + invariantsBuildRule(vcfg.invariants) + exploreBuildRule(exploreCfg) + (planned ? "" : (fromCheckpoint ? CONTINUE_NOTE : RERUN_NOTE).replace("BASE", lane)) + (salvagedRef ? SALVAGE_NOTE.replaceAll("REF", salvagedRef) : "") + recoverNote() + (planned ? "" : resumeNote + decisionsNote);
1649
1678
  liveKeys = ["implementation"];
1650
- stage = "implement";
1679
+ stage = "implement"; phaseLedger.enter("implement");
1651
1680
  if (!lightDone) await builder(++attemptN, implementPrompt, planned ? sessionId : null, true, undefined, { trigger: "build" });
1652
1681
  stage = "fix"; // later builder rounds fix precheck, CI, structure, docs or conformance findings
1653
1682
  result = await readResult();
@@ -1668,6 +1697,7 @@ export async function runCore(o) {
1668
1697
  T.decisions.push(`Builder verdict: ${result?.status ?? "none"}`);
1669
1698
  }
1670
1699
 
1700
+ phaseLedger.enter("precheck");
1671
1701
  // ---- 3a. declared check commands (#4240): the repo's own cheap gates (forge.yaml learned.test_commands), run locally in
1672
1702
  // the worktree before review and PR; no Actions minutes. A failure goes back to the builder in-session (#4149), up to
1673
1703
  // verify.max_rounds; a round with no commit or exhausted rounds is an anomaly, not a stop: CI stays the gate.
@@ -1933,6 +1963,7 @@ export async function runCore(o) {
1933
1963
  }
1934
1964
  const structureLine = structureNote(structure);
1935
1965
 
1966
+ phaseLedger.enter("ci_wait");
1936
1967
  // ---- 3c. verify by CI (#4167): ci|hybrid push the branch and run the repo's CI on GitHub runners, before review ----
1937
1968
  // A red run resumes the builder with the failed jobs' logs (verify.max_rounds), slow CI ends ci-pending, and CI that
1938
1969
  // cannot run falls back once to local validation. A head verified here lets the precheck accept validate ids left to CI.
@@ -2028,6 +2059,7 @@ export async function runCore(o) {
2028
2059
  }
2029
2060
  }
2030
2061
 
2062
+ phaseLedger.enter("precheck");
2031
2063
  // ---- 4. conformance: typed amendments, deterministic precheck, structured review; the engine computes the verdict ----
2032
2064
  // enforce: every failure class blocks (one builder resume, then needs-human). shadow: the verdict is computed and posted,
2033
2065
  // but only what the previous reviewer enforced blocks (a review that finds a requirement not implemented, or a regression),
@@ -2035,6 +2067,10 @@ export async function runCore(o) {
2035
2067
  // A review without usable rows ends needs-human (or, with engine.review_incomplete: degraded, merges flagged).
2036
2068
  await label("in-review");
2037
2069
  let shadow = conformance !== "enforce";
2070
+ // per-kind modes (engine.conformance_kinds) apply until a risk path forces the whole lane to enforce (#4398)
2071
+ const kindsNow = () => (riskEnforced ? null : kinds);
2072
+ const recording = () => shadow || (!riskEnforced && Object.keys(kinds).length > 0); // the gate records what it would have blocked
2073
+ const blockingOf = (fs, allow) => blockingFailures(fs, { shadow, kinds: kindsNow(), allow });
2038
2074
  let effective = frozen, review = null, lastAmend = "", resumes = 0, verdict = null, degraded = null, triggerSkipped = false, selfReviewCompared = false;
2039
2075
  const TRIGGER_SKIP = "skipped (review.trigger enforce: no risk trigger)";
2040
2076
  const triggerSkipNote = "## Conformance Review\n**Skipped by `review.trigger: enforce`:** no risk path, package edge or hotspot in the diff, and every deterministic gate passed; no reviewer ran (#4273).";
@@ -2045,11 +2081,14 @@ export async function runCore(o) {
2045
2081
  genPaths ??= (hotFacts ? hotFacts.generated_paths : light ? [] : (await hotspotsOf({ paths: [] }))?.generated_paths) ?? [];
2046
2082
  const wantsDecision = (Array.isArray(result?.amendments) ? result.amendments : []).some((x) => x?.kind === "decision");
2047
2083
  baseTree ??= wantsDecision ? String(await git(["ls-tree", "-r", "--name-only", base]).catch(() => "")).split("\n").filter(Boolean) : null;
2048
- const a = applyAmendments(frozen, [...(Array.isArray(result?.amendments) ? result.amendments : []), ...impliedDocs], issueText, { generatedPaths: genPaths, baseFiles: baseTree?.length ? baseTree : null });
2084
+ const own = [...(Array.isArray(result?.amendments) ? result.amendments : []), ...impliedDocs];
2085
+ const ownKeys = new Set(own.map((x) => JSON.stringify(x)));
2086
+ const all = [...priorAmendments.filter((x) => !ownKeys.has(JSON.stringify(x))), ...own];
2087
+ const a = applyAmendments(frozen, all, issueText, { generatedPaths: genPaths, baseFiles: baseTree?.length ? baseTree : null });
2049
2088
  const sig = JSON.stringify(a.applied);
2050
2089
  if (a.applied.length && sig !== lastAmend) {
2051
2090
  lastAmend = sig;
2052
- await post("FORGE:AMENDMENT", renderAmendments(a.applied));
2091
+ await post("FORGE:AMENDMENT", renderAmendments(a.applied, {}, { contract: hash, amendments: all }));
2053
2092
  T.decisions.push(`Contract amended: ${a.applied.map((x) => `${x.kind} ${x.target}`).join(", ")}`);
2054
2093
  }
2055
2094
  pendingDecisions = a.decisions; // #4737: persisted only once the PR merges
@@ -2058,7 +2097,8 @@ export async function runCore(o) {
2058
2097
  }
2059
2098
  const IGNORE = [`${LEAN_DIR}/`];
2060
2099
  /** Up to two reviewer runs; a reply without usable rows (unparseable, incomplete, inconsistent) is re-asked, never charged to the builder. */
2061
- async function reviewOnce() {
2100
+ function reviewOnce() { return phaseLedger.during("review", reviewOnceRaw); }
2101
+ async function reviewOnceRaw() {
2062
2102
  const diff = String(await git(["diff", `${base}...HEAD`]));
2063
2103
  const stat = Buffer.byteLength(diff, "utf-8") > REVIEW_DIFF_MAX_BYTES ? String(await git(["diff", "--stat=200", `${base}...HEAD`]).catch(() => "")) : "";
2064
2104
  const listed = new Set(String(await git(["ls-files"]).catch(() => "")).split("\n").map((x) => x.trim()).filter(Boolean));
@@ -2133,7 +2173,8 @@ export async function runCore(o) {
2133
2173
  }
2134
2174
  const verifierSection = () => { const n = renderVerifierNote(verifierRun); return n ? `\n\n${n}` : ""; };
2135
2175
  /** #4323: rerun once each validate command the builder recorded as failed, with the lane env; {[id]: {rc, output}} or null. */
2136
- async function rerunFailedValidates(contract) {
2176
+ function rerunFailedValidates(contract) { return phaseLedger.during("validate", () => rerunFailedValidatesRaw(contract)); }
2177
+ async function rerunFailedValidatesRaw(contract) {
2137
2178
  if (!io.runCommand) return null;
2138
2179
  const runs = Array.isArray(result?.validation) ? result.validation : [], out = {};
2139
2180
  const env = laneEnv();
@@ -2215,15 +2256,33 @@ export async function runCore(o) {
2215
2256
  const note = `Validate ${pc.ciCovered.join(", ")} verified by CI at ${gateHead.slice(0, 12)} (no local result)`;
2216
2257
  if (!T.decisions.includes(note)) T.decisions.push(note);
2217
2258
  }
2259
+ // report-only: a deliver requirement whose test never ran green is named, not judged on prose
2260
+ const unverified = unverifiedRequirements(effective, { runs: result?.validation, reruns: validateRuns, ciCovered: pc.ciCovered });
2261
+ if (unverified.length) {
2262
+ ev({ event: "REQUIREMENT_UNVERIFIED", ids: unverified.map((u) => u.id) });
2263
+ const note = `Unverified (no executed evidence test): ${unverified.map((u) => u.id).join(", ")}`;
2264
+ if (!T.decisions.includes(note)) T.decisions.push(note);
2265
+ }
2266
+ let mutBlock = [];
2267
+ if (conformanceMutation === "report" && io.runCommand) { // #4913: report-only unless conformance_kinds enforces the kind
2268
+ const m = await phaseLedger.during("validate", () => runMutationCheck({ targets: mutationTargets(effective, { runs: result?.validation, reruns: validateRuns, ciCovered: pc.ciCovered }), files, base, git: (a) => git(a), runCommand: (a) => io.runCommand(a), cwd: wt, env: laneEnv(), tmp: runTmp ?? undefined }));
2269
+ const mf = mutationFailures(m.results);
2270
+ if (m.results.length || m.skipped) ev({ event: "REQUIREMENT_MUTATION_SURVIVED", ids: mf.map((f) => f.detail.split(" ")[0]), checked: m.results.length, skipped: m.skipped || undefined });
2271
+ if (mf.length) {
2272
+ mutBlock = blockingFailures(mf, { shadow: true, kinds: { [MUTATION_SURVIVED]: "shadow", ...kinds } }).block;
2273
+ const note = `Mutation check: evidence test still passes with the change reverted for ${mf.map((f) => f.detail.split(" ")[0]).join(", ")}${mutBlock.length ? "" : " (shadow: not blocking)"}`;
2274
+ if (!T.decisions.includes(note)) T.decisions.push(note);
2275
+ }
2276
+ }
2218
2277
  Object.assign(verifyFacts, { precheck: pc.failures, amendments: amendProblems, hotspots: Boolean(hotspots && hotspots.enabled !== false && Array.isArray(hotspots.files)) });
2219
2278
  // #4323/#4334: a missing host tool or module, or a repro test the harness could not run, is an environment error:
2220
2279
  // never a conformance failure, never a builder resume
2221
2280
  const envFails = pc.failures.filter((f) => ENV_CODES.includes(f.code));
2222
- const pre = [...pc.failures.filter((f) => !ENV_CODES.includes(f.code)), ...amendProblems];
2281
+ const pre = [...pc.failures.filter((f) => !ENV_CODES.includes(f.code)), ...amendProblems, ...mutBlock];
2223
2282
  if (pre.length) ev({ event: "PRECHECK", failures: pre.map((f) => f.code) });
2224
- if (pre.length && !shadow) return { state: "nonconforming", by: "precheck", failures: pre };
2283
+ if (pre.length && !recording()) return { state: "nonconforming", by: "precheck", failures: pre };
2225
2284
  // shadow: the builder's own failed checks block before any reviewer tokens are spent (#4160)
2226
- const hard = shadow ? pre.filter((f) => SHADOW_PRE_BLOCKS.includes(f.code)) : [];
2285
+ const hard = recording() ? blockingOf(pre, SHADOW_PRE_BLOCKS).block : [];
2227
2286
  if (hard.length) return { state: "nonconforming", by: "precheck", failures: hard, would: "nonconforming", wouldFail: pre, reviewState: null };
2228
2287
  if (envFails.length) return { state: "environment", by: "precheck", failures: envFails };
2229
2288
  // #4273: deterministic review trigger; off computes nothing, shadow logs and always reviews, enforce skips when untriggered
@@ -2234,7 +2293,7 @@ export async function runCore(o) {
2234
2293
  ev({ event: "REVIEW_TRIGGER", mode: triggerMode, would_skip: t ? t.would_skip : false, reasons: t ? t.reasons.map((r) => r.code) : ["trigger.error"], skipped: skip });
2235
2294
  if (skip) {
2236
2295
  triggerSkipped = true;
2237
- return { state: "conforms", by: "trigger", failures: [], ...(shadow ? { would: "conforms", wouldFail: [], reviewState: null } : {}) };
2296
+ return { state: "conforms", by: "trigger", failures: [], ...(recording() ? { would: "conforms", wouldFail: [], reviewState: null } : {}) };
2238
2297
  }
2239
2298
  triggerSkipped = false;
2240
2299
  }
@@ -2244,12 +2303,12 @@ export async function runCore(o) {
2244
2303
  const vf = await runVerifier();
2245
2304
  if (vf.length) d = { state: "nonconforming", failures: vf, by: "verifier" };
2246
2305
  }
2247
- if (!shadow) return { ...d, by: d.by ?? "review" };
2306
+ if (!recording()) return { ...d, by: d.by ?? "review" };
2248
2307
  // shadow: record what enforce would have done; block only on what the previous reviewer blocked on
2249
2308
  const would = d.state === "conforms" && !pre.length ? "conforms" : d.state.startsWith("review.") ? d.state : "nonconforming";
2250
2309
  const wouldFail = [...pre, ...(d.state === "conforms" ? [] : d.failures)];
2251
2310
  if (d.state.startsWith("review.")) return { ...d, by: "review", would, wouldFail, reviewState: d.state };
2252
- const blocking = d.state === "nonconforming" ? d.failures.filter((f) => SHADOW_BLOCKS.includes(f.code)) : [];
2311
+ const blocking = d.state === "nonconforming" ? blockingOf(d.failures, SHADOW_BLOCKS).block : [];
2253
2312
  return { state: blocking.length ? "nonconforming" : "conforms", by: d.by ?? "review", failures: blocking, would, wouldFail, reviewState: d.state };
2254
2313
  }
2255
2314
  const verifyReported = new Set();
@@ -2282,6 +2341,10 @@ export async function runCore(o) {
2282
2341
  T.anomalies.push("Environment error: the repro harness could not run the repro test on head");
2283
2342
  return await needsHuman(`Environment error (not a conformance failure): the repro harness could not run the repro test on head; no builder resume was spent and nothing merged:\n${failures.map((f) => `- ${f.code}: ${f.detail}`).join("\n")}\n\nRun the repro command through the repo's own toolchain and installed dependencies, then rerun.`, OUTCOME_CODES.ENV_REPRO_UNRUNNABLE);
2284
2343
  }
2344
+ if (failures.every((f) => f.code === ENV_VALIDATE_TIMEOUT)) { // #4906: the teardown below pushes the lane's commits, so CI and recovery still see them
2345
+ T.anomalies.push("Environment error: a validate command timed out on the engine's rerun (infra, exit 124)");
2346
+ return await needsHuman(`Environment error (not a conformance failure): a validate command timed out (exit 124) even after one longer retry, which is host load rather than a failed check; no builder resume was spent, the branch is pushed and nothing merged:\n${failures.map((f) => `- ${f.code}: ${f.detail}`).join("\n")}\n\nRecovery reruns the command narrowed to the changed files; CI decides the head.`, ENV_VALIDATE_TIMEOUT);
2347
+ }
2285
2348
  if (failures.every((f) => f.code === OUTCOME_CODES.CONTRACT_INVALID_COMMAND)) {
2286
2349
  T.anomalies.push("Contract defect: a frozen validate command is a runner CLI usage error");
2287
2350
  return await needsHuman(`Contract defect (not a conformance failure): a frozen validate command is rejected by its runner's CLI, so no builder resume was spent and nothing merged:\n${failures.map((f) => `- ${f.code}: ${f.detail}`).join("\n")}\n\nRecovery re-plans the command (vitest: --reporter=dot, not -q).`, OUTCOME_CODES.CONTRACT_INVALID_COMMAND);
@@ -2335,14 +2398,14 @@ export async function runCore(o) {
2335
2398
  if (degradedReq) await track([degradedReq], "the conformance review stayed incomplete");
2336
2399
  if (triggerSkipped) {
2337
2400
  T.decisions.push("Conformance reviewer skipped by review.trigger enforce: no trigger reason (risk path, package edge, hotspot, failed or ambiguous gate)");
2338
- if (shadow) {
2401
+ if (recording()) {
2339
2402
  shadowWould = verdict.would;
2340
2403
  ev({ event: "CONFORMANCE_SHADOW", would: verdict.would, failures: [] });
2341
2404
  await emit({ event: "CONFORMANCE_SHADOW", would: verdict.would, failures: [] });
2342
2405
  }
2343
2406
  T.review = [TRIGGER_SKIP, `${reviewN} reviewer run(s), ${resumes + 1} round(s)`];
2344
2407
  await post("FORGE:REVIEW", triggerSkipNote);
2345
- } else if (shadow) {
2408
+ } else if (recording()) {
2346
2409
  const wf = verdict.wouldFail ?? [];
2347
2410
  shadowWould = verdict.would;
2348
2411
  ev({ event: "CONFORMANCE_SHADOW", would: verdict.would, failures: wf.map((f) => f.code) });
@@ -2368,6 +2431,7 @@ export async function runCore(o) {
2368
2431
  }
2369
2432
  if (degradedReq?.tracked && !closing.tracked.includes(degradedReq.tracked)) closing.tracked.push(degradedReq.tracked);
2370
2433
 
2434
+ phaseLedger.enter("ci_wait");
2371
2435
  // ---- 5. push + PR ----
2372
2436
  files = await changedFiles();
2373
2437
  await gitPush(["push", "-u", "origin", branch]);
@@ -2442,7 +2506,8 @@ export async function runCore(o) {
2442
2506
  || (await gitOut(["diff", "--check"])) !== "" || (await gitOut(["rev-parse", "-q", "--verify", "MERGE_HEAD"])) !== "";
2443
2507
  const syncRow = [];
2444
2508
  /** Returns null to continue with the merge flow, or a terminal result. */
2445
- async function baseSync(trigger, soft = false) {
2509
+ function baseSync(trigger, soft = false) { return phaseLedger.during("base_sync", () => baseSyncRaw(trigger, soft)); }
2510
+ async function baseSyncRaw(trigger, soft = false) {
2446
2511
  await renew();
2447
2512
  try { await git(["fetch", "origin", lane]); } catch { /* offline: use the local ref */ }
2448
2513
  const baseFull = await gitOut(["rev-parse", base]), baseSha = baseFull.slice(0, 12);
@@ -2756,6 +2821,7 @@ export async function runCore(o) {
2756
2821
  T.merge = [policyText, `merged at ${localHead.slice(0, 12)}`];
2757
2822
  laneGuard(resumed?.base ?? lane, "merge");
2758
2823
  try {
2824
+ phaseLedger.enter("merge");
2759
2825
  await ghc(["pr", "merge", String(pr), "-R", repo, "--merge", "--match-head-commit", localHead]);
2760
2826
  try { seqLease?.release(); } catch { /* fail-open */ }
2761
2827
  seqLease = null;
@@ -2840,6 +2906,12 @@ export async function runCore(o) {
2840
2906
  if (e?.code === "CLI_SPAWN_FAILED" || e?.spawnDeterministic === true) { // #4129: certain to fail again, so not engine-error (orchestrate would relaunch it)
2841
2907
  return await needsHuman(`The claude CLI could not be started (${e.spawnErrno ?? "spawn error"}); a rerun fails the same way: ${String(e.message).slice(0, 400)}`, OUTCOME_CODES.CLI_SPAWN_FAILED);
2842
2908
  }
2909
+ if (e?.pushRejected && /non-fast-forward|fetch first|\[rejected\]/i.test(String(e?.stderr ?? e?.message ?? e))) {
2910
+ // #4920: another live lane owns this branch (its lease, or its open PR): this lane is superseded, nothing for a human to do
2911
+ let owner = null;
2912
+ try { owner = await otherLaneOwner({ gh: (a) => ghc(a), repo, issue, branch, selfId: agentId, ownPr: Number(/#(\d+)/.exec(String(T.pr?.[0] ?? ""))?.[1]) || null }); } catch { /* fail-open */ }
2913
+ if (owner) return finish("deferred", `git push was rejected (non-fast-forward): ${owner.detail}; another lane owns ${branch}`, { code: OUTCOME_CODES.LEASE_HELD, branch }, OUTCOME_CODES.LEASE_HELD);
2914
+ }
2843
2915
  if (e?.pushRejected) return await needsHuman(`git push was rejected (no retry): ${String(e?.stderr ?? e?.message ?? e).slice(0, 400)}`);
2844
2916
  return finish("engine-error", e?.message ?? String(e));
2845
2917
  }