@intentius/chant 0.124.89 → 0.124.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci-comment-commands.d.ts +18 -2
- package/dist/ci-comment-commands.d.ts.map +1 -1
- package/dist/ci-comment-commands.js +30 -2
- package/dist/ci-job-guard.d.ts +42 -45
- package/dist/ci-job-guard.d.ts.map +1 -1
- package/dist/ci-job-guard.js +206 -3
- package/dist/ci-note-commands.d.ts +41 -1
- package/dist/ci-note-commands.d.ts.map +1 -1
- package/dist/ci-note-commands.js +64 -5
- package/dist/ci-product.d.ts +41 -1
- package/dist/ci-product.d.ts.map +1 -1
- package/dist/ci-product.js +44 -7
- package/dist/cli/handlers/pr-comment.d.ts +13 -1
- package/dist/cli/handlers/pr-comment.d.ts.map +1 -1
- package/dist/cli/handlers/pr-comment.js +90 -1
- package/dist/cli/handlers/pr-notes.d.ts.map +1 -1
- package/dist/cli/handlers/pr-notes.js +23 -1
- package/dist/cli/handlers/pr.js +16 -14
- package/dist/cli/handlers/run-wave.d.ts.map +1 -1
- package/dist/cli/handlers/run-wave.js +154 -4
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +17 -0
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +15 -0
- package/dist/cli/registry.d.ts +8 -1
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/components/pr-checks.d.ts +28 -1
- package/dist/components/pr-checks.d.ts.map +1 -1
- package/dist/components/pr-checks.js +35 -2
- package/dist/components/pr-pipeline.d.ts +3 -2
- package/dist/components/pr-pipeline.d.ts.map +1 -1
- package/dist/components/pr-pipeline.js +4 -2
- package/dist/components/product-pipeline.d.ts +54 -2
- package/dist/components/product-pipeline.d.ts.map +1 -1
- package/dist/components/product-pipeline.js +59 -5
- package/dist/components/product-plan.d.ts +13 -1
- package/dist/components/product-plan.d.ts.map +1 -1
- package/dist/components/product-plan.js +36 -10
- package/dist/floor-table.data.d.ts.map +1 -1
- package/dist/floor-table.data.js +32 -23
- package/dist/lexicon.d.ts +82 -4
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/lifecycle/gate-applied.d.ts +9 -0
- package/dist/lifecycle/gate-applied.d.ts.map +1 -1
- package/dist/op/apply-status.d.ts +84 -0
- package/dist/op/apply-status.d.ts.map +1 -0
- package/dist/op/apply-status.js +106 -0
- package/dist/op/before-merge-decide.d.ts +102 -0
- package/dist/op/before-merge-decide.d.ts.map +1 -0
- package/dist/op/before-merge-decide.js +142 -0
- package/dist/op/before-merge-done.d.ts +56 -0
- package/dist/op/before-merge-done.d.ts.map +1 -0
- package/dist/op/before-merge-done.js +69 -0
- package/dist/op/before-merge-store.d.ts +20 -0
- package/dist/op/before-merge-store.d.ts.map +1 -0
- package/dist/op/before-merge-store.js +48 -0
- package/dist/op/before-merge.d.ts +150 -0
- package/dist/op/before-merge.d.ts.map +1 -0
- package/dist/op/before-merge.js +207 -0
- package/dist/op/follow-on-run.d.ts +117 -0
- package/dist/op/follow-on-run.d.ts.map +1 -0
- package/dist/op/follow-on-run.js +154 -0
- package/dist/op/follow-on-store.d.ts +17 -0
- package/dist/op/follow-on-store.d.ts.map +1 -0
- package/dist/op/follow-on-store.js +54 -0
- package/dist/op/gate.d.ts +6 -0
- package/dist/op/gate.d.ts.map +1 -1
- package/dist/op/gate.js +10 -1
- package/dist/op/index.d.ts +3 -1
- package/dist/op/index.d.ts.map +1 -1
- package/dist/op/index.js +1 -0
- package/dist/op/op-waves-after-plan.d.ts +13 -0
- package/dist/op/op-waves-after-plan.d.ts.map +1 -1
- package/dist/op/op-waves-evidence-jobs.d.ts +4 -1
- package/dist/op/op-waves-evidence-jobs.d.ts.map +1 -1
- package/dist/op/op-waves-evidence-jobs.js +10 -2
- package/dist/op/op-waves-follow-on.d.ts +162 -0
- package/dist/op/op-waves-follow-on.d.ts.map +1 -0
- package/dist/op/op-waves-follow-on.js +355 -0
- package/dist/op/op-waves-run.d.ts +146 -6
- package/dist/op/op-waves-run.d.ts.map +1 -1
- package/dist/op/op-waves-run.js +871 -99
- package/dist/op/op-waves-split.d.ts +122 -0
- package/dist/op/op-waves-split.d.ts.map +1 -0
- package/dist/op/op-waves-split.js +166 -0
- package/dist/op/op-waves.d.ts +20 -0
- package/dist/op/op-waves.d.ts.map +1 -1
- package/dist/op/op-waves.js +15 -1
- package/dist/op/schedule-overdue.d.ts +15 -1
- package/dist/op/schedule-overdue.d.ts.map +1 -1
- package/dist/op/schedule-overdue.js +18 -2
- package/dist/op/stage-hooks.d.ts +36 -1
- package/dist/op/stage-hooks.d.ts.map +1 -1
- package/dist/op/stage-hooks.js +17 -3
- package/dist/op/wave-stop.d.ts +4 -1
- package/dist/op/wave-stop.d.ts.map +1 -1
- package/dist/op/wave-stop.js +4 -2
- package/dist/pr-post.d.ts.map +1 -1
- package/dist/pr-post.js +2 -1
- package/dist/release-floors.d.ts +3 -1
- package/dist/release-floors.d.ts.map +1 -1
- package/dist/release-floors.js +21 -19
- package/dist/wave-blocked.d.ts +7 -1
- package/dist/wave-blocked.d.ts.map +1 -1
- package/dist/wave-blocked.js +7 -2
- package/dist/wave-runtime.mjs +364 -351
- package/package.json +1 -1
- package/src/ci-comment-commands.test.ts +63 -0
- package/src/ci-comment-commands.ts +43 -4
- package/src/ci-image.test.ts +1 -2
- package/src/ci-job-guard.before-merge.test.ts +148 -0
- package/src/ci-job-guard.follow-on-record.test.ts +137 -0
- package/src/ci-job-guard.follow-on.test.ts +144 -0
- package/src/ci-job-guard.ts +221 -3
- package/src/ci-note-commands.test.ts +76 -1
- package/src/ci-note-commands.ts +96 -7
- package/src/ci-product.ts +55 -7
- package/src/cli/handlers/pr-comment.ts +85 -2
- package/src/cli/handlers/pr-notes.ts +30 -6
- package/src/cli/handlers/pr.ts +14 -11
- package/src/cli/handlers/run-wave.ts +144 -4
- package/src/cli/main.ts +17 -0
- package/src/cli/parse-args.ts +11 -0
- package/src/cli/registry.ts +8 -1
- package/src/components/pr-checks.ts +55 -2
- package/src/components/pr-pipeline.ts +5 -3
- package/src/components/product-pipeline.apply-status.test.ts +50 -0
- package/src/components/product-pipeline.ts +113 -8
- package/src/components/product-plan.test.ts +23 -5
- package/src/components/product-plan.ts +43 -12
- package/src/floor-table.data.ts +32 -23
- package/src/lexicon.ts +78 -1
- package/src/lifecycle/gate-applied.ts +7 -0
- package/src/op/apply-status.test.ts +83 -0
- package/src/op/apply-status.ts +132 -0
- package/src/op/before-merge-decide.ts +190 -0
- package/src/op/before-merge-done.ts +99 -0
- package/src/op/before-merge-store.ts +44 -0
- package/src/op/before-merge.test.ts +201 -0
- package/src/op/before-merge.ts +266 -0
- package/src/op/follow-on-run.ts +206 -0
- package/src/op/follow-on-store.ts +54 -0
- package/src/op/gate.ts +16 -1
- package/src/op/index.ts +5 -0
- package/src/op/op-waves-after-plan.ts +13 -0
- package/src/op/op-waves-evidence-jobs.ts +11 -3
- package/src/op/op-waves-follow-on-record.test.ts +152 -0
- package/src/op/op-waves-follow-on.test.ts +247 -0
- package/src/op/op-waves-follow-on.ts +385 -0
- package/src/op/op-waves-render-spec.test.ts +1 -2
- package/src/op/op-waves-run.ts +971 -93
- package/src/op/op-waves-split-product.test.ts +288 -0
- package/src/op/op-waves-split-step.test.ts +298 -0
- package/src/op/op-waves-split.test.ts +196 -0
- package/src/op/op-waves-split.ts +189 -0
- package/src/op/op-waves.ts +28 -3
- package/src/op/schedule-overdue.test.ts +29 -1
- package/src/op/schedule-overdue.ts +27 -3
- package/src/op/stage-hooks.ts +52 -4
- package/src/op/wave-stop.ts +5 -3
- package/src/pr-post.ts +2 -1
- package/src/release-floors.ts +22 -19
- package/src/wave-blocked.ts +9 -4
package/dist/op/op-waves-run.js
CHANGED
|
@@ -50,13 +50,14 @@
|
|
|
50
50
|
*/
|
|
51
51
|
import { execFileSync } from "node:child_process";
|
|
52
52
|
import { createHash } from "node:crypto";
|
|
53
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
53
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
54
54
|
import { dirname, isAbsolute, join, posix } from "node:path";
|
|
55
|
+
import { OP_WAVE_PHASES, OP_WAVE_RUN_RECORD_FILE, opWaveRunRecordFileName, opWaveSavedPlanPath } from "./op-waves-split.js";
|
|
55
56
|
import { CHANT_EXIT } from "../exit-codes.js";
|
|
56
57
|
import { decideWaveGate, runWaveLoop, waveDigestMembers, waveSetDigest, writeWaveRecordFile, } from "../wave-engine.js";
|
|
57
58
|
import { blockedMembers, blockedWords, earlierWaveOutcomes, passesThrough, readWaveOutcomeFile, waveOutcomeFileName, writeWaveOutcomeFile, } from "../wave-blocked.js";
|
|
58
59
|
import { approveCommand, evaluateGate, gitGateLedgerPort } from "./gate.js";
|
|
59
|
-
import { isApprovalSpent } from "../lifecycle/gate-applied.js";
|
|
60
|
+
import { finishedApply, isApprovalSpent } from "../lifecycle/gate-applied.js";
|
|
60
61
|
import { samePlanDigest } from "../lifecycle/plan-digest.js";
|
|
61
62
|
import { movedSincePreview } from "./op-waves-preview.js";
|
|
62
63
|
import { isLifecycleLocalNote } from "../lifecycle/local-note.js";
|
|
@@ -78,8 +79,8 @@ import { gitOpWaveNewerCommit, opWaveStandDownReason, tipAppliesTargets } from "
|
|
|
78
79
|
import { keyedSerial, mapLimited } from "../parallelism.js";
|
|
79
80
|
import { applyOpWaveStep, opWaveCosts, opWaveCostVerdict, opWaveHoldsAt, opWaveHoldsProblems, opWavePolicyVerdicts, opWaveRegistry, opWaveStepChangeSet, planJsonOf, planOpWaveStep, planOpWaveStepGroup, discoverOpWaveStepRuns, applyOpWaveStepGroup, OP_WAVE_TOGETHER_MEMBER, } from "./op-waves-capability.js";
|
|
80
81
|
import { changeInvolvement, quorumProblems, quorumRuleLine, quorumRulesFor, quorumWaiting, recordedReviewApprovals, recordedReviewOf, ruleNeedsProof, samePrincipal, trustQuorumJudge, } from "./gate-quorum.js";
|
|
81
|
-
import { OP_WAVE_DEFAULT_APPLY, OP_WAVE_GATE_POLICIES, OP_WAVE_HOOK_MOMENTS, GATE_APPROVAL_SOURCES, assertOpWaveHooks, assertOpWavesSpec, opWaveApplyHooksRunBy, opWaveGateName, opWaveHooksAt, opWaveChecksDir, opWaveOutcomeDir, opWavePointAt, opWavePointProblems, opWaveRecordPath, opWaveGroupRuns, opWaveGroupsProblem, opWaveShare, OP_LEDGER_NAME, opWavesInvolvedProblems, waveApproveTarget, } from "./op-waves.js";
|
|
82
|
-
import { OP_WAVE_PENDING_RESERVED, opWaveArgv, pendingFields, holdFinishedPath, readHoldFinished, readHoldRequest, runHookList, runHookListAsync, runStageHooks, runStageHooksAsync, spawnExec, spawnExecAsync, } from "./stage-hooks.js";
|
|
82
|
+
import { OP_WAVE_DEFAULT_APPLY, OP_WAVE_GATE_POLICIES, OP_WAVE_HOOK_MOMENTS, GATE_APPROVAL_SOURCES, assertOpWaveHooks, assertOpWavesSpec, opWaveApplyHooksRunBy, opWaveAt, opWaveGateName, opWaveHooksAt, opWaveChecksDir, opWaveOutcomeDir, opWavePointAt, opWavePointProblems, opWaveRecordPath, opWaveGroupRuns, opWaveGroupsProblem, opWaveShare, OP_LEDGER_NAME, opWavesInvolvedProblems, waveApproveTarget, } from "./op-waves.js";
|
|
83
|
+
import { OP_WAVE_PENDING_RESERVED, opWaveArgv, pendingFields, holdFinishedPath, readHoldFinished, readHoldRequest, inProcessHookOf, runHookList, runHookListAsync, runStageHooks, runStageHooksAsync, spawnExec, spawnExecAsync, } from "./stage-hooks.js";
|
|
83
84
|
export { OP_WAVE_PENDING_RESERVED, opWaveArgv, runStageHooks, runStageHooksAsync, spawnExecAsync };
|
|
84
85
|
export const OP_WAVE_APPLY_OUTCOMES = ["stood-down", "held", "replan"];
|
|
85
86
|
/** {@link OpWaveShowAtBase} through git: `git show <base>:<path>`. A run-time spec (#4982, `./op-waves-render-spec.ts`) reads every other file through it. */
|
|
@@ -586,7 +587,7 @@ async function declaredLockGrain(cwd, base) {
|
|
|
586
587
|
* run that holds it. A run whose plan changes nothing takes no row, and
|
|
587
588
|
* outside a git work tree the waiting grains take none.
|
|
588
589
|
*/
|
|
589
|
-
async function leaseWaveRows(options, ctx, runs) {
|
|
590
|
+
async function leaseWaveRows(options, ctx, runs, opts = {}) {
|
|
590
591
|
const locks = runs.flatMap((r) => {
|
|
591
592
|
const plan = ctx.steps?.get(r.target);
|
|
592
593
|
return plan?.lock && !plan.empty ? [plan.lock] : [];
|
|
@@ -624,7 +625,9 @@ async function leaseWaveRows(options, ctx, runs) {
|
|
|
624
625
|
},
|
|
625
626
|
}
|
|
626
627
|
: {};
|
|
627
|
-
|
|
628
|
+
// #4992: a split wave's gate job ends before its rows are applied, so its hold carries no heartbeat of its process:
|
|
629
|
+
// the forge's run, the same in its applying and record jobs, keeps it, as a hold from before #4660 is kept.
|
|
630
|
+
leased = await leaseLockRows(gate, wanted, { cwd: ctx.cwd, ...alive, ...waitFor, ...(opts.noBeat ? { beat: () => () => { } } : {}) });
|
|
628
631
|
}
|
|
629
632
|
catch (err) {
|
|
630
633
|
return { failed: `${label}: could not hold the rows its runs change in ${gate}, so nothing was applied: ${err instanceof Error ? err.message : String(err)}` };
|
|
@@ -649,7 +652,43 @@ async function leaseWaveRows(options, ctx, runs) {
|
|
|
649
652
|
return stop;
|
|
650
653
|
}
|
|
651
654
|
}
|
|
652
|
-
return { release: releaseAll };
|
|
655
|
+
return { release: releaseAll, held: held.map((l) => ({ gate: l.gate, run: l.holder.run, keys: [...l.keys] })) };
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* The lease of a split wave with a `step` (#4992), in its gate job: the
|
|
659
|
+
* rows its runs wait for and change, taken as {@link leaseWaveRows} takes
|
|
660
|
+
* them before an apply, with no heartbeat, since this job ends before the
|
|
661
|
+
* applying jobs start. The decision names the lease, and the record job
|
|
662
|
+
* lets go of it. A held row ends the wave here with the `held` outcome,
|
|
663
|
+
* exit 5, before the approval is spent.
|
|
664
|
+
*/
|
|
665
|
+
async function leaseSplitRows(options, ctx, decision, recordPath) {
|
|
666
|
+
const lease = await leaseWaveRows(options, ctx, ctx.wave.runs, { noBeat: true });
|
|
667
|
+
if (!lease)
|
|
668
|
+
return undefined;
|
|
669
|
+
if ("failed" in lease)
|
|
670
|
+
return { end: failed(ctx, lease.failed) };
|
|
671
|
+
if ("outcome" in lease) {
|
|
672
|
+
ctx.log(`wave ${ctx.k} (${ctx.wave.name}): applied nothing: ${lease.reason}`);
|
|
673
|
+
ctx.result.outcome = lease;
|
|
674
|
+
ctx.result.exitCode = CHANT_EXIT.held;
|
|
675
|
+
writeRecord(recordPath, decision, ctx.result);
|
|
676
|
+
return { end: ctx.result };
|
|
677
|
+
}
|
|
678
|
+
const first = lease.held[0];
|
|
679
|
+
if (first) {
|
|
680
|
+
decision.lease = { gate: first.gate, run: first.run, keys: [...new Set(lease.held.flatMap((h) => h.keys))].sort() };
|
|
681
|
+
ctx.log(`wave ${ctx.k} (${ctx.wave.name}): holds ${decision.lease.keys.length} row${decision.lease.keys.length === 1 ? "" : "s"} of ${first.gate} for its applying jobs, until its record job lets go of them`);
|
|
682
|
+
}
|
|
683
|
+
return lease;
|
|
684
|
+
}
|
|
685
|
+
/** Let go of the rows a split wave's gate job leased (#4992), as the holder the decision names. */
|
|
686
|
+
async function releaseSplitLease(cwd, lease, k, log) {
|
|
687
|
+
const { releaseLockRows } = await import("../lifecycle/resource-lock.js");
|
|
688
|
+
if (await releaseLockRows(lease.gate, lease.keys, lease.run, { cwd }))
|
|
689
|
+
log(`wave ${k}: let go of the ${lease.keys.length} row${lease.keys.length === 1 ? "" : "s"} of ${lease.gate} its gate job held`);
|
|
690
|
+
else
|
|
691
|
+
log(`wave ${k}: could not let go of the rows its gate job held on ${lease.gate}; the next apply takes them over once this run is gone`);
|
|
653
692
|
}
|
|
654
693
|
function inGitWorkTree(cwd) {
|
|
655
694
|
try {
|
|
@@ -867,9 +906,11 @@ async function runHooks(ctx, moment, hooks, runs, members, digest, extraEnv = {}
|
|
|
867
906
|
if (!hooks || hooks.length === 0)
|
|
868
907
|
return { holds: [] };
|
|
869
908
|
const runsPath = writeRunsFile(ctx, runs, members, digest);
|
|
870
|
-
const
|
|
909
|
+
const own = ctx.options.inProcessHooks;
|
|
910
|
+
// #4994: a hook the job's process serves runs in it; the other hooks run as before, through hookExec or exec.
|
|
911
|
+
const hookExec = ctx.options.hookExec ?? (own ? (argv, cwd, env) => Promise.resolve(ctx.exec(argv, cwd, env)) : undefined);
|
|
871
912
|
const out = hookExec
|
|
872
|
-
? await runHookListAsync({ ...placeOf(ctx), exec: hookExec }, moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv)
|
|
913
|
+
? await runHookListAsync({ ...placeOf(ctx), exec: hookExec, ...(own ? { inProcess: own } : {}) }, moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv)
|
|
873
914
|
: runHookList(placeOf(ctx), moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv);
|
|
874
915
|
ctx.result.hooks.push(...out.outcomes);
|
|
875
916
|
const back = { holds: out.holds, ...(out.failed ? { failed: out.failed } : {}) };
|
|
@@ -901,7 +942,24 @@ async function runHooks(ctx, moment, hooks, runs, members, digest, extraEnv = {}
|
|
|
901
942
|
// #4623: a hold that asks chant to record its gate is spent here, before the re-run acts under the approval.
|
|
902
943
|
let spent;
|
|
903
944
|
let finishedAt;
|
|
904
|
-
|
|
945
|
+
// #4994: the gate job of an earlier run of this head spent the approval for this re-run; the gate job after this
|
|
946
|
+
// plan job writes the finished line, from what the re-run leaves at CHANT_HOLD_FINISHED.
|
|
947
|
+
const spentEarlier = hold.request.record && ctx.options.phase === "plan" ? held.spentForRerun : undefined;
|
|
948
|
+
if (spentEarlier) {
|
|
949
|
+
spent = spentEarlier;
|
|
950
|
+
const finishedRel = holdFinishedPath(hold.env.CHANT_HOLD);
|
|
951
|
+
finishedAt = join(ctx.cwd, finishedRel);
|
|
952
|
+
rmSync(finishedAt, { force: true });
|
|
953
|
+
approvedEnv.CHANT_HOLD_FINISHED = finishedRel;
|
|
954
|
+
}
|
|
955
|
+
else if (hold.request.record && noTokenPhase(ctx.options)) {
|
|
956
|
+
// #4992: the approval is spent before the re-run acts under it, and a job with no write token cannot spend it.
|
|
957
|
+
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: 1, outcome: "failed" });
|
|
958
|
+
back.failed = `${hold.label}: ${held.approvedBy} approved ${held.op} / ${held.gate}, and the hold asks chant to record the gate, which spends the approval before the hook runs again; this ${ctx.options.phase} job of a split wave holds no write token, so it cannot spend it and the hook did not run again`;
|
|
959
|
+
ctx.log(back.failed);
|
|
960
|
+
return back;
|
|
961
|
+
}
|
|
962
|
+
else if (hold.request.record) {
|
|
905
963
|
const got = await spendHeldGate(ctx, hold, now);
|
|
906
964
|
if ("failed" in got) {
|
|
907
965
|
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: 1, outcome: "failed" });
|
|
@@ -917,13 +975,19 @@ async function runHooks(ctx, moment, hooks, runs, members, digest, extraEnv = {}
|
|
|
917
975
|
}
|
|
918
976
|
let code;
|
|
919
977
|
try {
|
|
920
|
-
|
|
978
|
+
// #4994: an in-process hook runs again in this process, as it ran the first time.
|
|
979
|
+
const again = inProcessHookOf(own, hold.name);
|
|
980
|
+
code = again
|
|
981
|
+
? await again({ name: hold.name, moment: hold.moment, ...(hold.target ? { target: hold.target } : {}), argv: hold.argv, cwd: ctx.cwd, env: approvedEnv })
|
|
982
|
+
: hookExec ? await hookExec(hold.argv, ctx.cwd, approvedEnv) : ctx.exec(hold.argv, ctx.cwd, approvedEnv);
|
|
921
983
|
}
|
|
922
984
|
catch (err) {
|
|
923
985
|
ctx.log(`${hold.label} did not run: ${err instanceof Error ? err.message : String(err)}`);
|
|
924
986
|
code = 127;
|
|
925
987
|
}
|
|
926
|
-
if (spent && finishedAt)
|
|
988
|
+
if (spent && finishedAt && spentEarlier)
|
|
989
|
+
handHoldFinished(ctx, hold, spent, readHoldFinished(finishedAt, ctx.log, hold.label), code);
|
|
990
|
+
else if (spent && finishedAt)
|
|
927
991
|
await finishHeldGate(ctx, hold, spent, readHoldFinished(finishedAt, ctx.log, hold.label), code);
|
|
928
992
|
if (code === 0) {
|
|
929
993
|
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: code, outcome: "passed" });
|
|
@@ -1080,20 +1144,15 @@ async function settleOtherGate(ctx, hold, runs) {
|
|
|
1080
1144
|
const { port } = await gatePort(ctx.options, ctx.cwd, hold.request.gate, mode, rules.base);
|
|
1081
1145
|
const neverOverMcp = rules.neverOverMcp;
|
|
1082
1146
|
const now = ctx.options.now ?? new Date().toISOString();
|
|
1147
|
+
// #4992: a split wave's plan and apply jobs hold no write token. They read the other gate and, while it waits, leave its
|
|
1148
|
+
// pending fact and the file the request carries to the gate or record job, which records them (recordDeferredHold).
|
|
1149
|
+
const deferring = noTokenPhase(ctx.options);
|
|
1083
1150
|
// #4623: the file the request carries, stored once before the pending fact, which names its SHA-256.
|
|
1084
1151
|
const attachment = hold.request.attachment;
|
|
1085
|
-
if (attachment !== undefined)
|
|
1086
|
-
|
|
1087
|
-
if (!port.storeRequestAttachment)
|
|
1088
|
-
throw new Error("this gate ledger keeps no request files");
|
|
1089
|
-
await port.storeRequestAttachment(op, hold.request.digest, attachment);
|
|
1090
|
-
}
|
|
1091
|
-
catch (err) {
|
|
1092
|
-
log(`${op} / ${hold.request.gate}: the file ${hold.label} attached was not stored beside the pending fact: ${err instanceof Error ? err.message : String(err)}`);
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1152
|
+
if (attachment !== undefined && !deferring)
|
|
1153
|
+
await storeHoldAttachment(port, op, hold.request.digest, attachment, `${op} / ${hold.request.gate}: the file ${hold.label} attached`, log);
|
|
1095
1154
|
const proof = attachment !== undefined ? { proof: `sha256:${createHash("sha256").update(attachment).digest("hex")}` } : {};
|
|
1096
|
-
const
|
|
1155
|
+
const input = {
|
|
1097
1156
|
op,
|
|
1098
1157
|
gate: hold.request.gate,
|
|
1099
1158
|
planDigest: hold.request.digest,
|
|
@@ -1102,15 +1161,41 @@ async function settleOtherGate(ctx, hold, runs) {
|
|
|
1102
1161
|
// #3839: the wave a hold belongs to, as a field, whatever the hook's own description says.
|
|
1103
1162
|
fields: { ...hold.request.pending, ...proof, approvalMode: rules.approval, wave: { number: k, name: wave.name, op: spec.name }, heldBy: { hook: hold.name, moment: hold.moment, ...(hold.target ? { target: hold.target } : {}) } },
|
|
1104
1163
|
...(neverOverMcp ? { neverOverMcp: true } : {}),
|
|
1105
|
-
}
|
|
1164
|
+
};
|
|
1165
|
+
const check = await evaluateGate(deferring ? readOnlyGatePort(port) : port, input);
|
|
1106
1166
|
const base = { op, gate: hold.request.gate, digest: hold.request.digest, hook: hold.label };
|
|
1107
1167
|
logRefused(log, `${op} / ${hold.request.gate}`, check);
|
|
1108
|
-
if (check.satisfied)
|
|
1109
|
-
return { ...base, status: "approved", approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp };
|
|
1110
1168
|
// #4574: a per-run hook's hold names its run; else the wave's target, as the wave's own gate does.
|
|
1111
1169
|
const target = hold.target ?? waveApproveTarget(wave.name, runs.map((r) => r.target));
|
|
1112
1170
|
const approve = approveCommand(op, hold.request.gate, undefined, hold.request.digest, mode === "sealed", target);
|
|
1171
|
+
// #4994: a split wave's plan job cannot spend the approval a `record` hold needs spent before its hook runs again.
|
|
1172
|
+
const spendLater = deferring && hold.request.record === true && ctx.options.phase === "plan";
|
|
1173
|
+
if (spendLater) {
|
|
1174
|
+
// The gate job of an earlier run of this commit spent the approval for this very re-run, and no finished line names it yet.
|
|
1175
|
+
const spent = await spentForRerun(port, op, hold.request.gate, hold.request.digest, waveHeadOf(ctx.options, ctx.cwd));
|
|
1176
|
+
if (spent) {
|
|
1177
|
+
log(`${hold.label}: the approval by ${spent.approvedBy} of ${op} / ${hold.request.gate} at ${hold.request.digest} was spent at ${spent.timestamp} for this re-run of the hook, so it runs again under it`);
|
|
1178
|
+
return { ...base, status: "approved", approvedBy: spent.approvedBy, approvedAt: spent.approvedAt, spentForRerun: spent };
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
if (check.satisfied && spendLater) {
|
|
1182
|
+
const head = waveHeadOf(ctx.options, ctx.cwd);
|
|
1183
|
+
log(`wave ${k} (${wave.name}): ${check.resolution.resolvedBy} approved ${op} / ${hold.request.gate} at ${hold.request.digest}, and ${hold.label} asks chant to record the gate, which spends the approval before the hook runs again; this plan job holds no write token, so the gate job spends it and the wave waits until it runs again from its plan job, where the hook runs again under it`);
|
|
1184
|
+
return {
|
|
1185
|
+
...base,
|
|
1186
|
+
status: "waiting",
|
|
1187
|
+
approve,
|
|
1188
|
+
recorded: false,
|
|
1189
|
+
deferred: { input, mode, base: rules.base, ...(attachment !== undefined ? { attachment } : {}), spend: { approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp, ...(head ? { head } : {}) } },
|
|
1190
|
+
};
|
|
1191
|
+
}
|
|
1192
|
+
if (check.satisfied)
|
|
1193
|
+
return { ...base, status: "approved", approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp };
|
|
1113
1194
|
log(`wave ${k} (${wave.name}) waits on ${op} / ${hold.request.gate}, which ${hold.label} holds it on: nobody approved ${hold.request.digest}. Approve: ${approve}`);
|
|
1195
|
+
if (deferring) {
|
|
1196
|
+
log(`wave ${k} (${wave.name}): this job holds no write token, so the ${ctx.options.phase === "plan" ? "gate" : "record"} job records the pending fact of ${op} / ${hold.request.gate}`);
|
|
1197
|
+
return { ...base, status: "waiting", approve, recorded: false, deferred: { input, mode, base: rules.base, ...(attachment !== undefined ? { attachment } : {}) }, ...(check.mismatch ? { mismatch: check.mismatch } : {}) };
|
|
1198
|
+
}
|
|
1114
1199
|
return {
|
|
1115
1200
|
...base,
|
|
1116
1201
|
status: "waiting",
|
|
@@ -1120,6 +1205,209 @@ async function settleOtherGate(ctx, hold, runs) {
|
|
|
1120
1205
|
...(check.mismatch ? { mismatch: check.mismatch } : {}),
|
|
1121
1206
|
};
|
|
1122
1207
|
}
|
|
1208
|
+
/** The commit a wave job runs (#4994): `options.head`, a split wave's decided head, else the checkout's HEAD; undefined outside a git checkout. */
|
|
1209
|
+
function waveHeadOf(options, cwd) {
|
|
1210
|
+
if (options.head)
|
|
1211
|
+
return options.head;
|
|
1212
|
+
try {
|
|
1213
|
+
return execFileSync("git", ["rev-parse", "HEAD"], { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] }).trim() || undefined;
|
|
1214
|
+
}
|
|
1215
|
+
catch {
|
|
1216
|
+
return undefined;
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
/**
|
|
1220
|
+
* The applied line a split wave's gate job wrote for a re-run of a held
|
|
1221
|
+
* hook at `head` (#4994), when no finished line names it yet; else
|
|
1222
|
+
* undefined. A port that cannot read finished lines gives none, so no
|
|
1223
|
+
* approval is used twice.
|
|
1224
|
+
*/
|
|
1225
|
+
async function spentForRerun(port, op, gate, digest, head) {
|
|
1226
|
+
if (!head || !port.readApplied || !port.readFinished)
|
|
1227
|
+
return undefined;
|
|
1228
|
+
const line = (await port.readApplied(op)).filter((a) => a.gate === gate && samePlanDigest(a.planDigest, digest)).at(-1);
|
|
1229
|
+
if (!line || line.rerun?.head !== head)
|
|
1230
|
+
return undefined;
|
|
1231
|
+
return finishedApply(await port.readFinished(op), line) ? undefined : line;
|
|
1232
|
+
}
|
|
1233
|
+
/** The finished lines a split wave's plan job leaves its gate job to write (#4994), by the job's options. */
|
|
1234
|
+
const handedFinished = new WeakMap();
|
|
1235
|
+
/** Keep a re-run's end for the gate job (#4994), in the saved plan. */
|
|
1236
|
+
function handHoldFinished(ctx, hold, spent, wrote, code) {
|
|
1237
|
+
const list = handedFinished.get(ctx.options) ?? [];
|
|
1238
|
+
list.push({
|
|
1239
|
+
hook: hold.label,
|
|
1240
|
+
applied: { op: spent.op, gate: spent.gate, planDigest: spent.planDigest, timestamp: spent.timestamp, ...(spent.rerun ? { rerun: spent.rerun } : {}) },
|
|
1241
|
+
result: wrote?.result ?? (code === 0 ? "applied" : "failed"),
|
|
1242
|
+
...(wrote?.detail ? { detail: wrote.detail } : {}),
|
|
1243
|
+
});
|
|
1244
|
+
handedFinished.set(ctx.options, list);
|
|
1245
|
+
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Write the finished lines a split wave's plan job handed on (#4994), with
|
|
1248
|
+
* the token: each for an applied line the ledger holds, spent for a re-run
|
|
1249
|
+
* at this job's head and not finished yet. The plan job ran the change's
|
|
1250
|
+
* code, so a line it names that is not one of those is left out.
|
|
1251
|
+
*/
|
|
1252
|
+
async function writeHandedFinished(options, cwd, handed, log) {
|
|
1253
|
+
if (!handed?.length)
|
|
1254
|
+
return;
|
|
1255
|
+
const port = options.gates ?? gitGateLedgerPort({ cwd });
|
|
1256
|
+
const head = waveHeadOf(options, cwd);
|
|
1257
|
+
for (const h of handed) {
|
|
1258
|
+
const where = `${h.applied.op} / ${h.applied.gate}`;
|
|
1259
|
+
try {
|
|
1260
|
+
if (!port.appendFinished || !port.readApplied || !port.readFinished)
|
|
1261
|
+
throw new Error("this gate ledger keeps no finished lines");
|
|
1262
|
+
const line = (await port.readApplied(h.applied.op)).find((a) => a.gate === h.applied.gate && samePlanDigest(a.planDigest, h.applied.planDigest) && a.timestamp === h.applied.timestamp);
|
|
1263
|
+
if (!line || !head || line.rerun?.head !== head) {
|
|
1264
|
+
log(`${h.hook}: the plan job says it ran again under ${where} applied at ${h.applied.timestamp}, which the ledger holds no re-run of this commit for, so no finished line is written`);
|
|
1265
|
+
continue;
|
|
1266
|
+
}
|
|
1267
|
+
if (finishedApply(await port.readFinished(h.applied.op), line))
|
|
1268
|
+
continue;
|
|
1269
|
+
const record = {
|
|
1270
|
+
version: 1,
|
|
1271
|
+
kind: "finished",
|
|
1272
|
+
op: line.op,
|
|
1273
|
+
gate: line.gate,
|
|
1274
|
+
planDigest: line.planDigest,
|
|
1275
|
+
applied: line.timestamp,
|
|
1276
|
+
result: h.result,
|
|
1277
|
+
timestamp: options.now ?? new Date().toISOString(),
|
|
1278
|
+
...(h.detail ? { detail: h.detail } : {}),
|
|
1279
|
+
};
|
|
1280
|
+
const pushed = await port.appendFinished(record);
|
|
1281
|
+
if (!pushed.pushed && pushed.pushWarning && !isLifecycleLocalNote(pushed.pushWarning))
|
|
1282
|
+
log(`${h.hook}: the finished line of ${where} was written and not pushed: ${pushed.pushWarning}`);
|
|
1283
|
+
log(`${h.hook}: wrote the finished line of ${where} (${h.result}) for the plan job's re-run`);
|
|
1284
|
+
}
|
|
1285
|
+
catch (err) {
|
|
1286
|
+
log(`${h.hook}: the finished line of ${where} was not written: ${err instanceof Error ? err.message : String(err)}`);
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
/**
|
|
1291
|
+
* Spend the approval a split wave's plan job found for a `record` hold
|
|
1292
|
+
* (#4994), with the token, under this job's rules at base: the gate is read
|
|
1293
|
+
* again, and still approved, an applied line bound to this job's head is
|
|
1294
|
+
* written for the hook's re-run in the next run's plan job. Not approved
|
|
1295
|
+
* any more, the hold is recorded as waiting (`recordDeferredHold`). A digest
|
|
1296
|
+
* already applied under the gate is not spent again.
|
|
1297
|
+
*/
|
|
1298
|
+
async function spendDeferredHold(options, ctx, heldOn, log) {
|
|
1299
|
+
const deferred = heldOn.deferred;
|
|
1300
|
+
const rule = ctx.gateRule;
|
|
1301
|
+
const mode = rule.approval === "sealed" ? "sealed" : "ledger";
|
|
1302
|
+
const { port } = await gatePort(options, ctx.cwd, deferred.input.gate, mode, rule.base);
|
|
1303
|
+
const where = `${deferred.input.op} / ${deferred.input.gate}`;
|
|
1304
|
+
const digest = deferred.input.planDigest;
|
|
1305
|
+
// A re-run this commit's earlier gate job spent the approval for, which the plan job did not take up: the wave waits for it.
|
|
1306
|
+
const pending = await spentForRerun(port, deferred.input.op, deferred.input.gate, digest, waveHeadOf(options, ctx.cwd));
|
|
1307
|
+
if (pending) {
|
|
1308
|
+
log(`${heldOn.hook}: the approval of ${where} at ${digest} is already spent for a re-run of the hook; the wave waits until it runs again from its plan job`);
|
|
1309
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
1310
|
+
return { heldOn: { ...rest, approvedBy: pending.approvedBy, approvedAt: pending.approvedAt, spentForRerun: pending } };
|
|
1311
|
+
}
|
|
1312
|
+
const check = await evaluateGate(readOnlyGatePort(port), { ...deferred.input, now: options.now ?? new Date().toISOString(), ...(rule.neverOverMcp || deferred.input.neverOverMcp ? { neverOverMcp: true } : {}) });
|
|
1313
|
+
if (!check.satisfied) {
|
|
1314
|
+
log(`${heldOn.hook}: ${where} is not approved under this job's rules, so its approval is not spent`);
|
|
1315
|
+
const { spend: _s, ...waiting } = deferred;
|
|
1316
|
+
return { heldOn: await recordDeferredHold(options, ctx.cwd, { ...heldOn, deferred: waiting }, rule, log) };
|
|
1317
|
+
}
|
|
1318
|
+
const head = waveHeadOf(options, ctx.cwd);
|
|
1319
|
+
if (!head)
|
|
1320
|
+
return { failed: `${heldOn.hook}: this job cannot tell which commit the wave runs, so it does not spend the approval of ${where} for the hook's re-run` };
|
|
1321
|
+
if (deferred.spend?.head && deferred.spend.head !== head)
|
|
1322
|
+
return { failed: `${heldOn.hook}: the plan job ran at ${deferred.spend.head.slice(0, 12)} and this job at ${head.slice(0, 12)}, so the approval of ${where} is not spent` };
|
|
1323
|
+
try {
|
|
1324
|
+
if (!port.appendApplied || !port.readApplied)
|
|
1325
|
+
throw new Error("this gate ledger cannot record an applied approval");
|
|
1326
|
+
const prior = (await port.readApplied(deferred.input.op)).filter((a) => a.gate === deferred.input.gate && samePlanDigest(a.planDigest, digest)).at(-1);
|
|
1327
|
+
if (prior)
|
|
1328
|
+
return { failed: `${heldOn.hook}: ${where} at ${digest} applied already under the approval by ${prior.approvedBy} (${prior.timestamp}); a held request applies once, so it is not spent again` };
|
|
1329
|
+
const commit = typeof deferred.input.fields?.commit === "string" ? deferred.input.fields.commit : undefined;
|
|
1330
|
+
const runId = ciRunIdFromEnv();
|
|
1331
|
+
const record = {
|
|
1332
|
+
version: 1,
|
|
1333
|
+
kind: "applied",
|
|
1334
|
+
op: deferred.input.op,
|
|
1335
|
+
gate: deferred.input.gate,
|
|
1336
|
+
planDigest: digest,
|
|
1337
|
+
approvedAt: check.resolution.timestamp,
|
|
1338
|
+
approvedBy: check.resolution.resolvedBy,
|
|
1339
|
+
timestamp: options.now ?? new Date().toISOString(),
|
|
1340
|
+
wave: { number: ctx.k, name: ctx.wave.name },
|
|
1341
|
+
...(runId ? { runId } : {}),
|
|
1342
|
+
...(commit ? { commit } : {}),
|
|
1343
|
+
rerun: { head },
|
|
1344
|
+
};
|
|
1345
|
+
requireSpendPushed(await port.appendApplied(record));
|
|
1346
|
+
log(`${heldOn.hook}: ${where} applies under the approval by ${record.approvedBy} at ${record.approvedAt}, which is now spent for the hook's re-run at ${head.slice(0, 12)}; run the wave again, and its plan job runs the hook again under it`);
|
|
1347
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
1348
|
+
return { heldOn: { ...rest, approvedBy: record.approvedBy, approvedAt: record.approvedAt, spentForRerun: record } };
|
|
1349
|
+
}
|
|
1350
|
+
catch (err) {
|
|
1351
|
+
return { failed: `${heldOn.hook}: could not record that ${where} applies under the approval by ${check.resolution.resolvedBy}: ${err instanceof Error ? err.message : String(err)}` };
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
/** Whether a split wave's job holds no write token (#4992): its plan and applying jobs run the change's code. */
|
|
1355
|
+
function noTokenPhase(options) {
|
|
1356
|
+
return options.phase === "plan" || options.phase === "apply";
|
|
1357
|
+
}
|
|
1358
|
+
/** Store the file a held request carries (#4623). A file not stored is logged; the pending fact still names its SHA-256. */
|
|
1359
|
+
async function storeHoldAttachment(port, op, digest, attachment, what, log) {
|
|
1360
|
+
try {
|
|
1361
|
+
if (!port.storeRequestAttachment)
|
|
1362
|
+
throw new Error("this gate ledger keeps no request files");
|
|
1363
|
+
await port.storeRequestAttachment(op, digest, attachment);
|
|
1364
|
+
}
|
|
1365
|
+
catch (err) {
|
|
1366
|
+
log(`${what} was not stored beside the pending fact: ${err instanceof Error ? err.message : String(err)}`);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
/**
|
|
1370
|
+
* The gate ledger as a job with no write token reads it (#4992): every read
|
|
1371
|
+
* the port has, and a pending fact that is never written. `evaluateGate`
|
|
1372
|
+
* then says whether the gate is approved, and the fact it would have
|
|
1373
|
+
* recorded goes to a job that can write it.
|
|
1374
|
+
*/
|
|
1375
|
+
function readOnlyGatePort(port) {
|
|
1376
|
+
const { appendResolution: _r, appendApplied: _a, appendFinished: _f, storeRequestAttachment: _s, writeCheckpoint: _c, ...reads } = port;
|
|
1377
|
+
return {
|
|
1378
|
+
...reads,
|
|
1379
|
+
async appendPending(input) {
|
|
1380
|
+
return { record: { version: 1, kind: "pending", ...input }, pushed: false, pushWarning: "not recorded: this job holds no write token" };
|
|
1381
|
+
},
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
/**
|
|
1385
|
+
* Record a hold on another gate that a split wave's plan or apply job left
|
|
1386
|
+
* (#4992): the file its request carries and the gate's pending fact, with
|
|
1387
|
+
* the token, as that job would have without the split. Returns the hold as
|
|
1388
|
+
* the decision records it. A gate approved since the plan or apply job read
|
|
1389
|
+
* it is said so: the wave waits all the same, and running it again from its
|
|
1390
|
+
* plan job goes on under the approval.
|
|
1391
|
+
*/
|
|
1392
|
+
async function recordDeferredHold(options, cwd, heldOn, rule, log) {
|
|
1393
|
+
const deferred = heldOn.deferred;
|
|
1394
|
+
if (!deferred)
|
|
1395
|
+
return heldOn;
|
|
1396
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
1397
|
+
// The approval mode, the base and neverOverMcp are this job's own, read at base, whatever the job that left the hold said.
|
|
1398
|
+
const mode = rule.approval === "sealed" ? "sealed" : "ledger";
|
|
1399
|
+
const { port } = await gatePort(options, cwd, deferred.input.gate, mode, rule.base);
|
|
1400
|
+
const where = `${deferred.input.op} / ${deferred.input.gate}`;
|
|
1401
|
+
if (deferred.attachment !== undefined)
|
|
1402
|
+
await storeHoldAttachment(port, deferred.input.op, deferred.input.planDigest, deferred.attachment, `${where}: the file ${heldOn.hook} attached`, log);
|
|
1403
|
+
const check = await evaluateGate(port, { ...deferred.input, now: options.now ?? new Date().toISOString(), ...(rule.neverOverMcp || deferred.input.neverOverMcp ? { neverOverMcp: true } : {}) });
|
|
1404
|
+
if (check.satisfied) {
|
|
1405
|
+
log(`${where} was approved by ${check.resolution.resolvedBy} after the job that ${heldOn.hook} ran in read it; the wave waits all the same, and goes on when it runs again from its plan job`);
|
|
1406
|
+
return rest;
|
|
1407
|
+
}
|
|
1408
|
+
log(`${where}: ${check.recorded ? "recorded the pending fact" : "the pending fact stands"} that ${heldOn.hook} holds wave ${options.wave} on`);
|
|
1409
|
+
return { ...rest, waitingSince: check.pending.timestamp, recorded: check.recorded };
|
|
1410
|
+
}
|
|
1123
1411
|
/**
|
|
1124
1412
|
* Read the hooks file chant hands an apply command (#3804): the one at
|
|
1125
1413
|
* `path`, by default `CHANT_HOOKS`. Null when there is none, as for an apply
|
|
@@ -1263,6 +1551,16 @@ function noteHooksAtBase(ctx, rules) {
|
|
|
1263
1551
|
*/
|
|
1264
1552
|
export async function runOpWave(options) {
|
|
1265
1553
|
assertOpWavesSpec(options.spec);
|
|
1554
|
+
if (options.phase !== undefined) {
|
|
1555
|
+
// #4987: a split wave's plan and gate jobs are its deciding job, in two; the record job writes the applying jobs' records.
|
|
1556
|
+
const wrong = opWavePhaseProblem(options);
|
|
1557
|
+
if (wrong)
|
|
1558
|
+
throw new Error(`Op waves "${options.spec.name}": ${wrong}`);
|
|
1559
|
+
if (options.phase === "record")
|
|
1560
|
+
return runOpWaveRecordPhase(options);
|
|
1561
|
+
if (options.phase === "plan" || options.phase === "gate")
|
|
1562
|
+
options = { ...options, decide: true };
|
|
1563
|
+
}
|
|
1266
1564
|
const afterPlanWrong = opWaveAfterPlanProblems(options.afterPlan);
|
|
1267
1565
|
if (afterPlanWrong.length > 0)
|
|
1268
1566
|
throw new Error(`Op waves "${options.spec.name}": ${afterPlanWrong.join("; ")}`);
|
|
@@ -1416,7 +1714,7 @@ export function opWaveRunRecord(options, times, outcome) {
|
|
|
1416
1714
|
const wave = {
|
|
1417
1715
|
spec: spec.name,
|
|
1418
1716
|
number: options.wave,
|
|
1419
|
-
name: spec
|
|
1717
|
+
name: opWaveAt(spec, options.wave)?.name ?? `wave-${options.wave}`,
|
|
1420
1718
|
job: options.decide ? "decide" : options.group !== undefined ? "group" : options.share !== undefined ? "share" : "wave",
|
|
1421
1719
|
...(options.share !== undefined ? { share: options.share } : {}),
|
|
1422
1720
|
...(options.group !== undefined ? { group: options.group } : {}),
|
|
@@ -1457,7 +1755,8 @@ export function opWaveRunRecord(options, times, outcome) {
|
|
|
1457
1755
|
*/
|
|
1458
1756
|
async function recordOpWaveRun(options, started, outcome) {
|
|
1459
1757
|
const cwd = options.cwd ?? process.cwd();
|
|
1460
|
-
|
|
1758
|
+
// #4987: a split wave's plan job records nothing (its gate job records the decision), and an applying job leaves its record for the record job.
|
|
1759
|
+
const port = options.phase === "plan" ? undefined : options.phase === "apply" ? fileOpWaveRunPort(options, "result" in outcome ? outcome.result.handoff : undefined) : options.runs ?? (options.gates ? undefined : gitOpWaveRunPort({ cwd }));
|
|
1461
1760
|
const log = options.log ?? ((line) => console.error(line));
|
|
1462
1761
|
let gitSha = options.head;
|
|
1463
1762
|
if (gitSha === undefined) {
|
|
@@ -1512,6 +1811,9 @@ async function runOpWaveJob(options) {
|
|
|
1512
1811
|
const cwd = options.cwd ?? process.cwd();
|
|
1513
1812
|
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(spec);
|
|
1514
1813
|
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
1814
|
+
// #4987: the gate job of a split wave reads what the plan job saved, not the outcome files again.
|
|
1815
|
+
if (options.phase === "gate")
|
|
1816
|
+
return runOpWaveGatePhase(options, dir);
|
|
1515
1817
|
// #4322: what the earlier jobs' outcome files say: an earlier wave that stopped stops this one, and a failure blocks its dependents.
|
|
1516
1818
|
const before = opWaveBefore(options, dir);
|
|
1517
1819
|
const blocked = before.blocked;
|
|
@@ -1519,6 +1821,8 @@ async function runOpWaveJob(options) {
|
|
|
1519
1821
|
log(before.stop.line);
|
|
1520
1822
|
const stopped = { exitCode: before.stop.exitCode, applied: [], failed: [], moved: [], hooks: [], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) };
|
|
1521
1823
|
writeOpWaveOutcome(options, dir, stopped, before.stop.members);
|
|
1824
|
+
if (options.phase === "plan")
|
|
1825
|
+
saveOpWavePlan(options, { end: stopped });
|
|
1522
1826
|
return stopped;
|
|
1523
1827
|
}
|
|
1524
1828
|
if (blocked.size > 0)
|
|
@@ -1528,8 +1832,12 @@ async function runOpWaveJob(options) {
|
|
|
1528
1832
|
// Every run of this job is blocked: nothing to plan, decide or apply.
|
|
1529
1833
|
const none = { exitCode: options.decide ? 0 : 1, applied: [], failed: [], moved: [], hooks: [], blocked: Object.fromEntries(blocked) };
|
|
1530
1834
|
writeOpWaveOutcome(options, dir, none);
|
|
1835
|
+
if (options.phase === "plan")
|
|
1836
|
+
saveOpWavePlan(options, { end: none });
|
|
1531
1837
|
return none;
|
|
1532
1838
|
}
|
|
1839
|
+
if (options.phase === "plan")
|
|
1840
|
+
return runOpWavePlanPhase(options, dir, blocked);
|
|
1533
1841
|
let result;
|
|
1534
1842
|
const applied = [];
|
|
1535
1843
|
try {
|
|
@@ -1539,6 +1847,12 @@ async function runOpWaveJob(options) {
|
|
|
1539
1847
|
writeOpWaveOutcome(options, dir, { exitCode: 1, applied, failed: [], moved: [], hooks: [], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) });
|
|
1540
1848
|
throw err;
|
|
1541
1849
|
}
|
|
1850
|
+
if (options.phase === "apply" && result.replan) {
|
|
1851
|
+
// #4987: the plan job plans; an applying job that would plan again stops, and the wave runs again from its plan job.
|
|
1852
|
+
log(`wave ${options.wave}: the apply applied nothing and asks to plan again${result.replan.reason ? ` (${result.replan.reason})` : ""}; before merge the plan job plans, so run the wave again`);
|
|
1853
|
+
delete result.replan;
|
|
1854
|
+
result.exitCode = CHANT_EXIT.failed;
|
|
1855
|
+
}
|
|
1542
1856
|
const all = { ...Object.fromEntries(blocked), ...(result.blocked ?? {}) };
|
|
1543
1857
|
if (Object.keys(all).length > 0) {
|
|
1544
1858
|
result.blocked = all;
|
|
@@ -1591,6 +1905,375 @@ async function runOpWavePolicyOnly(options) {
|
|
|
1591
1905
|
return end(CHANT_EXIT.denied, rows, `wave ${options.wave} (${wave.name}): ci.policy denied ${denied.length} of its ${rows.length} plans, so ${check} did not pass`);
|
|
1592
1906
|
return end(CHANT_EXIT.done, rows, `wave ${options.wave} (${wave.name}): ci.policy ran over its ${rows.length} plans and denied none`);
|
|
1593
1907
|
}
|
|
1908
|
+
/** Why `options` cannot run its phase (#4987), or undefined. */
|
|
1909
|
+
function opWavePhaseProblem(options) {
|
|
1910
|
+
const { phase } = options;
|
|
1911
|
+
if (!OP_WAVE_PHASES.includes(phase))
|
|
1912
|
+
return `the phase is one of ${OP_WAVE_PHASES.join(", ")}, not ${JSON.stringify(phase)}`;
|
|
1913
|
+
if (options.decide)
|
|
1914
|
+
return "a phase takes the deciding job's place; pass --decide or --phase, not both";
|
|
1915
|
+
if (options.policyOnly && phase !== "plan")
|
|
1916
|
+
return "--policy-only runs in a split wave's plan job alone";
|
|
1917
|
+
if ((options.share !== undefined || options.group !== undefined) && phase !== "apply")
|
|
1918
|
+
return `--${options.share !== undefined ? "share" : "group"} names an applying job; the ${phase} job covers the whole wave`;
|
|
1919
|
+
return undefined;
|
|
1920
|
+
}
|
|
1921
|
+
/** Write the plan job's saved plan (#4987). */
|
|
1922
|
+
function saveOpWavePlan(options, plan) {
|
|
1923
|
+
const path = join(options.cwd ?? process.cwd(), opWaveSavedPlanPath(options.spec, options.wave));
|
|
1924
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1925
|
+
const name = opWaveAt(options.spec, options.wave)?.name ?? `wave-${options.wave}`;
|
|
1926
|
+
// #4994: the re-runs under an approval an earlier gate job spent, whose finished lines the gate job writes.
|
|
1927
|
+
const finished = handedFinished.get(options);
|
|
1928
|
+
const saved = { version: 1, op: options.spec.name, wave: options.wave, name, ...plan, ...(finished?.length ? { holdsFinished: finished } : {}) };
|
|
1929
|
+
writeFileSync(path, JSON.stringify(saved, null, 2) + "\n");
|
|
1930
|
+
}
|
|
1931
|
+
/** Read the saved plan the gate job decides on (#4987); throws when the plan job left none. */
|
|
1932
|
+
function readSavedOpWavePlan(options) {
|
|
1933
|
+
const { spec, wave: k } = options;
|
|
1934
|
+
const rel = opWaveSavedPlanPath(spec, k);
|
|
1935
|
+
let doc;
|
|
1936
|
+
try {
|
|
1937
|
+
doc = JSON.parse(readFileSync(join(options.cwd ?? process.cwd(), rel), "utf-8"));
|
|
1938
|
+
}
|
|
1939
|
+
catch {
|
|
1940
|
+
throw new Error(`wave ${k} found no saved plan at ${rel}: its plan job did not finish, so nothing is decided`);
|
|
1941
|
+
}
|
|
1942
|
+
if (doc?.version !== 1 || doc.op !== spec.name || doc.wave !== k)
|
|
1943
|
+
throw new Error(`${rel} is not the saved plan of wave ${k} of ${spec.name}`);
|
|
1944
|
+
if (doc.end === undefined && (!Array.isArray(doc.members) || typeof doc.digest !== "string"))
|
|
1945
|
+
throw new Error(`${rel} holds neither the plans of wave ${k} nor how its plan job ended`);
|
|
1946
|
+
return doc;
|
|
1947
|
+
}
|
|
1948
|
+
/**
|
|
1949
|
+
* The plan job of a split wave (#4987): plan the wave as its deciding job
|
|
1950
|
+
* would, then save the plan for the gate job and stop. It writes nothing to
|
|
1951
|
+
* `chant/lifecycle` and keeps no run record. A wave its plan ends (a hook
|
|
1952
|
+
* failed, a plan failed, the rules at base refuse it) writes its outcome
|
|
1953
|
+
* here, and the saved plan says how it ended.
|
|
1954
|
+
*/
|
|
1955
|
+
async function runOpWavePlanPhase(options, dir, blocked) {
|
|
1956
|
+
const { spec, wave: k } = options;
|
|
1957
|
+
const log = options.log ?? ((line) => console.error(line));
|
|
1958
|
+
const ended = (r) => {
|
|
1959
|
+
const all = { ...Object.fromEntries(blocked), ...(r.blocked ?? {}) };
|
|
1960
|
+
if (Object.keys(all).length > 0)
|
|
1961
|
+
r.blocked = all;
|
|
1962
|
+
writeOpWaveOutcome(options, dir, r);
|
|
1963
|
+
saveOpWavePlan(options, { end: r });
|
|
1964
|
+
return r;
|
|
1965
|
+
};
|
|
1966
|
+
let step;
|
|
1967
|
+
try {
|
|
1968
|
+
step = await planOpWave(options, { k, pass: 0 }, blocked);
|
|
1969
|
+
}
|
|
1970
|
+
catch (err) {
|
|
1971
|
+
ended({ exitCode: CHANT_EXIT.failed, applied: [], failed: [], moved: [], hooks: [] });
|
|
1972
|
+
throw err;
|
|
1973
|
+
}
|
|
1974
|
+
if ("end" in step)
|
|
1975
|
+
return ended(step.end);
|
|
1976
|
+
const p = step.go;
|
|
1977
|
+
if (p.kind !== "decide")
|
|
1978
|
+
throw new Error(`wave ${k}'s plan job planned as a share; it plans the whole wave`);
|
|
1979
|
+
const digest = opWaveSetDigest(spec, p.members, p.extras);
|
|
1980
|
+
saveOpWavePlan(options, {
|
|
1981
|
+
digest,
|
|
1982
|
+
members: p.members,
|
|
1983
|
+
...(p.extras ? { extras: p.extras } : {}),
|
|
1984
|
+
...(p.pending ? { pending: p.pending } : {}),
|
|
1985
|
+
...(p.overriders ? { overriders: p.overriders } : {}),
|
|
1986
|
+
...(p.overridersSource ? { overridersSource: p.overridersSource } : {}),
|
|
1987
|
+
...(p.changeSet ? { changeSet: p.changeSet } : {}),
|
|
1988
|
+
holds: p.holds,
|
|
1989
|
+
before: p.before,
|
|
1990
|
+
...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}),
|
|
1991
|
+
...(p.ctx.report !== undefined ? { report: p.ctx.report } : {}),
|
|
1992
|
+
...(p.ctx.answerApprovals?.length ? { answerApprovals: p.ctx.answerApprovals } : {}),
|
|
1993
|
+
...(p.decision.ciPolicySource ? { ciPolicySource: p.decision.ciPolicySource } : {}),
|
|
1994
|
+
...(p.decision.discovered ? { discovered: p.decision.discovered } : {}),
|
|
1995
|
+
...(p.ctx.result.hooks.length > 0 ? { hooks: p.ctx.result.hooks } : {}),
|
|
1996
|
+
...(p.ctx.result.plans ? { plans: p.ctx.result.plans } : {}),
|
|
1997
|
+
// #4992: the step plans, for the gate job's lease.
|
|
1998
|
+
...(p.ctx.steps?.size ? { steps: [...p.ctx.steps.values()].map(({ artifact: _a, part: _p, group: _g, ...s }) => s) } : {}),
|
|
1999
|
+
});
|
|
2000
|
+
log(`wave ${k} (${p.ctx.wave.name}): planned ${p.members.length} run${p.members.length === 1 ? "" : "s"} to digest ${digest}; the plan is saved at ${opWaveSavedPlanPath(spec, k)} for the gate job`);
|
|
2001
|
+
return { exitCode: CHANT_EXIT.done, applied: [], failed: [], moved: [], hooks: [...p.ctx.result.hooks], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) };
|
|
2002
|
+
}
|
|
2003
|
+
/** The runs whose plan file no longer gives the digest the saved plan names (#4987). */
|
|
2004
|
+
function savedPlansMoved(ctx, members) {
|
|
2005
|
+
return members.filter((m) => readPlanFileField(join(ctx.cwd, opWavePlanPath(ctx.spec, ctx.k, m.member)), "planDigest") !== m.planDigest).map((m) => m.member);
|
|
2006
|
+
}
|
|
2007
|
+
/** An applying job's plans of `runs` (#4987), read from the plan files its plan job saved. */
|
|
2008
|
+
function savedRunPlans(ctx, runs) {
|
|
2009
|
+
const planned = new Map();
|
|
2010
|
+
for (const run of runs) {
|
|
2011
|
+
const path = opWavePlanPath(ctx.spec, ctx.k, run.target);
|
|
2012
|
+
try {
|
|
2013
|
+
const file = JSON.parse(readFileSync(join(ctx.cwd, path), "utf-8"));
|
|
2014
|
+
if (typeof file.planDigest !== "string" || file.planDigest === "")
|
|
2015
|
+
throw new Error("it has no planDigest");
|
|
2016
|
+
planned.set(run.target, member(run.target, run.op ?? ctx.spec.op, file));
|
|
2017
|
+
}
|
|
2018
|
+
catch (err) {
|
|
2019
|
+
return { failed: `${shareLabel(ctx.options)} of wave ${ctx.k} (${ctx.wave.name}) found no saved plan of ${run.target} at ${path} (${err instanceof Error ? err.message : String(err)}), so nothing in it was applied` };
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
return { planned };
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* The gate job of a split wave (#4987), at the default branch's code with
|
|
2026
|
+
* the token: read the rules at base, take the plan its plan job saved,
|
|
2027
|
+
* refuse it (exit 4) when its set digest is not the one its plan files
|
|
2028
|
+
* give, then decide the gate on it as a wide wave's deciding job does:
|
|
2029
|
+
* write the pending fact, spend the approval and the overrides, keep the
|
|
2030
|
+
* decision record and the outcome. A wave its plan job ended is not
|
|
2031
|
+
* decided again; that end is this job's result, which its run record keeps.
|
|
2032
|
+
*/
|
|
2033
|
+
async function runOpWaveGatePhase(options, dir) {
|
|
2034
|
+
const { spec, wave: k } = options;
|
|
2035
|
+
const cwd = options.cwd ?? process.cwd();
|
|
2036
|
+
const log = options.log ?? ((line) => console.error(line));
|
|
2037
|
+
const saved = readSavedOpWavePlan(options);
|
|
2038
|
+
// #4994: the plan job ran a held hook again under an approval an earlier gate job spent; this job writes how it ended.
|
|
2039
|
+
await writeHandedFinished(options, cwd, saved.holdsFinished, log);
|
|
2040
|
+
// #4992: a follow-on run's wave past the spec's is the rest wave's.
|
|
2041
|
+
const at = opWaveAt(spec, k);
|
|
2042
|
+
// #4992: a hook in the plan job held the wave on another gate it could not record; this job records it.
|
|
2043
|
+
const heldOn = saved.end?.exitCode === CHANT_EXIT.waiting ? saved.end.decision?.heldOn : undefined;
|
|
2044
|
+
if (saved.end && !(heldOn?.deferred && at)) {
|
|
2045
|
+
log(`wave ${k}: its plan job ended the wave with exit ${saved.end.exitCode}, so there is nothing to decide`);
|
|
2046
|
+
return saved.end;
|
|
2047
|
+
}
|
|
2048
|
+
if (!at)
|
|
2049
|
+
throw new Error(`Op waves "${spec.name}" has ${spec.waves.length} waves; there is no wave ${k}.`);
|
|
2050
|
+
const members = saved.members ?? [];
|
|
2051
|
+
// The runs the plan job planned, which a discovering wave found and the spec at the default branch may not list.
|
|
2052
|
+
const wave = saved.end ? at : { ...at, runs: members.map((m) => ({ target: m.member, ...(m.op !== spec.op ? { op: m.op } : {}) })) };
|
|
2053
|
+
const blocked = saved.blocked ?? {};
|
|
2054
|
+
const result = { exitCode: 0, applied: [], failed: [], moved: [], hooks: [...(saved.end?.hooks ?? saved.hooks ?? [])], ...(saved.plans ? { plans: saved.plans } : {}) };
|
|
2055
|
+
const ctx = { spec, k, wave, cwd, exec: options.exec ?? spawnExec, log, result, options, gateRule: { approval: "ledger", base: "HEAD", neverOverMcp: spec.neverOverMcp === true } };
|
|
2056
|
+
// #4992: the rows a step's runs lease, as the plan job planned them.
|
|
2057
|
+
if (saved.steps?.length)
|
|
2058
|
+
ctx.steps = new Map(saved.steps.map((s) => [s.target, s]));
|
|
2059
|
+
const end = (r) => {
|
|
2060
|
+
if (Object.keys(blocked).length > 0)
|
|
2061
|
+
r.blocked = { ...blocked, ...(r.blocked ?? {}) };
|
|
2062
|
+
writeOpWaveOutcome(options, dir, r);
|
|
2063
|
+
return r;
|
|
2064
|
+
};
|
|
2065
|
+
if (heldOn?.deferred) {
|
|
2066
|
+
// The decision is this job's own, from the rules at base: of the plan job's end it takes only the hold.
|
|
2067
|
+
const opened = await openWaveDecision(options, ctx, { evidence: false });
|
|
2068
|
+
if ("end" in opened)
|
|
2069
|
+
return end(opened.end);
|
|
2070
|
+
// #4994: an approved `record` hold, whose approval this job spends for the hook's re-run in the next run's plan job.
|
|
2071
|
+
if (heldOn.deferred.spend) {
|
|
2072
|
+
const spent = await spendDeferredHold(options, ctx, heldOn, log);
|
|
2073
|
+
if ("failed" in spent)
|
|
2074
|
+
return end(failed(ctx, `wave ${k} (${at.name}): ${spent.failed}`));
|
|
2075
|
+
return end(await decided(ctx, opened.go.decision, join(cwd, opWaveRecordPath(spec, k)), spent.heldOn));
|
|
2076
|
+
}
|
|
2077
|
+
const recorded = await recordDeferredHold(options, cwd, heldOn, ctx.gateRule, log);
|
|
2078
|
+
return end(await decided(ctx, opened.go.decision, join(cwd, opWaveRecordPath(spec, k)), recorded));
|
|
2079
|
+
}
|
|
2080
|
+
const opened = await openWaveDecision(options, ctx, { evidence: true });
|
|
2081
|
+
if ("end" in opened)
|
|
2082
|
+
return end(opened.end);
|
|
2083
|
+
const { rules, holds, decision } = opened.go;
|
|
2084
|
+
const digest = opWaveSetDigest(spec, members, saved.extras);
|
|
2085
|
+
const moved = savedPlansMoved(ctx, members);
|
|
2086
|
+
if (digest !== saved.digest || moved.length > 0) {
|
|
2087
|
+
log(`wave ${k} (${wave.name}): ` +
|
|
2088
|
+
(moved.length > 0 ? `the saved plan names plans its plan files do not hold (${moved.join(", ")})` : `the saved plan says digest ${saved.digest}, and its plans give ${digest}`) +
|
|
2089
|
+
", so nothing is decided or applied; run the wave again");
|
|
2090
|
+
return end({ ...result, exitCode: CHANT_EXIT.moved, moved: moved.length > 0 ? moved : members.map((m) => m.member) });
|
|
2091
|
+
}
|
|
2092
|
+
if (saved.discovered)
|
|
2093
|
+
decision.discovered = saved.discovered;
|
|
2094
|
+
if (saved.ciPolicySource)
|
|
2095
|
+
decision.ciPolicySource = saved.ciPolicySource;
|
|
2096
|
+
if (saved.report !== undefined)
|
|
2097
|
+
ctx.report = saved.report;
|
|
2098
|
+
if (saved.answerApprovals?.length)
|
|
2099
|
+
ctx.answerApprovals = saved.answerApprovals;
|
|
2100
|
+
const planned = {
|
|
2101
|
+
kind: "decide",
|
|
2102
|
+
ctx,
|
|
2103
|
+
recordPath: join(cwd, opWaveRecordPath(spec, k)),
|
|
2104
|
+
rules,
|
|
2105
|
+
decision,
|
|
2106
|
+
holds: [...new Set([...holds, ...(saved.holds ?? [])])],
|
|
2107
|
+
before: saved.before ?? [],
|
|
2108
|
+
members,
|
|
2109
|
+
planned: new Map(members.map((m) => [m.member, m])),
|
|
2110
|
+
...(saved.pending ? { pending: saved.pending } : {}),
|
|
2111
|
+
...(saved.overriders ? { overriders: saved.overriders } : {}),
|
|
2112
|
+
...(saved.overridersSource ? { overridersSource: saved.overridersSource } : {}),
|
|
2113
|
+
...(saved.extras ? { extras: saved.extras } : {}),
|
|
2114
|
+
...(saved.changeSet ? { changeSet: saved.changeSet } : {}),
|
|
2115
|
+
};
|
|
2116
|
+
const step = await decideOpWave(options, { k, pass: 0 }, planned, digest);
|
|
2117
|
+
return end("end" in step ? step.end : result);
|
|
2118
|
+
}
|
|
2119
|
+
/**
|
|
2120
|
+
* An applying job's run record port (#4987): a file in the outcome
|
|
2121
|
+
* directory, for the record job, in place of the ledger. #4992: with what
|
|
2122
|
+
* the job hands its record job under `handoff`, which the record job takes
|
|
2123
|
+
* off before it writes the record.
|
|
2124
|
+
*/
|
|
2125
|
+
function fileOpWaveRunPort(options, handoff) {
|
|
2126
|
+
const cwd = options.cwd ?? process.cwd();
|
|
2127
|
+
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(options.spec);
|
|
2128
|
+
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
2129
|
+
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}) };
|
|
2130
|
+
return {
|
|
2131
|
+
async append(record) {
|
|
2132
|
+
mkdirSync(dir, { recursive: true });
|
|
2133
|
+
const file = { ...record, ...(handoff && (handoff.releases?.length || handoff.heldOn) ? { handoff } : {}) };
|
|
2134
|
+
writeFileSync(join(dir, opWaveRunRecordFileName(options.wave, part)), JSON.stringify(file, null, 2) + "\n");
|
|
2135
|
+
},
|
|
2136
|
+
};
|
|
2137
|
+
}
|
|
2138
|
+
/** The run records wave `k`'s applying jobs left under `dir` (#4987), at any depth up to four, the newest of each name, by name. */
|
|
2139
|
+
function waveRunRecordFiles(dir, k, log) {
|
|
2140
|
+
const newest = new Map();
|
|
2141
|
+
const walk = (at, depth) => {
|
|
2142
|
+
let names;
|
|
2143
|
+
try {
|
|
2144
|
+
names = readdirSync(at);
|
|
2145
|
+
}
|
|
2146
|
+
catch {
|
|
2147
|
+
return;
|
|
2148
|
+
}
|
|
2149
|
+
for (const name of names) {
|
|
2150
|
+
const abs = join(at, name);
|
|
2151
|
+
let isDir = false;
|
|
2152
|
+
try {
|
|
2153
|
+
isDir = statSync(abs).isDirectory();
|
|
2154
|
+
}
|
|
2155
|
+
catch {
|
|
2156
|
+
continue;
|
|
2157
|
+
}
|
|
2158
|
+
if (isDir) {
|
|
2159
|
+
if (depth < 4)
|
|
2160
|
+
walk(abs, depth + 1);
|
|
2161
|
+
continue;
|
|
2162
|
+
}
|
|
2163
|
+
if (OP_WAVE_RUN_RECORD_FILE.exec(name)?.[1] !== String(k))
|
|
2164
|
+
continue;
|
|
2165
|
+
let record;
|
|
2166
|
+
try {
|
|
2167
|
+
record = JSON.parse(readFileSync(abs, "utf-8"));
|
|
2168
|
+
}
|
|
2169
|
+
catch (err) {
|
|
2170
|
+
log(`wave ${k}: the run record ${abs} does not parse, so it is not written: ${err instanceof Error ? err.message : String(err)}`);
|
|
2171
|
+
continue;
|
|
2172
|
+
}
|
|
2173
|
+
const had = newest.get(name);
|
|
2174
|
+
if (!had || had.ended <= record.ended)
|
|
2175
|
+
newest.set(name, record);
|
|
2176
|
+
}
|
|
2177
|
+
};
|
|
2178
|
+
walk(dir, 0);
|
|
2179
|
+
return [...newest].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([name, record]) => ({ name, record }));
|
|
2180
|
+
}
|
|
2181
|
+
/**
|
|
2182
|
+
* The record job of a split wave (#4987), at the default branch's code with
|
|
2183
|
+
* the token: write each run record the wave's applying jobs left to the run
|
|
2184
|
+
* ledger, and end as they did: 0 when each applied, else the first exit that
|
|
2185
|
+
* was not a failed run (a moved plan, a held or stood-down apply), else 1.
|
|
2186
|
+
* A record whose push never landed fails the job, as it fails a wave job.
|
|
2187
|
+
*
|
|
2188
|
+
* #4992: it also writes what the applying jobs handed it: the release
|
|
2189
|
+
* records of the runs they applied through the step, and the pending fact
|
|
2190
|
+
* of another gate a hook held the wave on (the wave then waits, exit 3).
|
|
2191
|
+
* Last it lets go of the rows the gate job leased, which the decision names.
|
|
2192
|
+
*/
|
|
2193
|
+
async function runOpWaveRecordPhase(options) {
|
|
2194
|
+
const { spec, wave: k } = options;
|
|
2195
|
+
const cwd = options.cwd ?? process.cwd();
|
|
2196
|
+
const log = options.log ?? ((line) => console.error(line));
|
|
2197
|
+
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(spec);
|
|
2198
|
+
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
2199
|
+
const result = { exitCode: CHANT_EXIT.done, applied: [], failed: [], moved: [], hooks: [] };
|
|
2200
|
+
// The gate job's decision: the lease it took, and the rules a hold on another gate is recorded under.
|
|
2201
|
+
let decision;
|
|
2202
|
+
try {
|
|
2203
|
+
decision = JSON.parse(readFileSync(join(cwd, opWaveRecordPath(spec, k)), "utf-8"));
|
|
2204
|
+
}
|
|
2205
|
+
catch {
|
|
2206
|
+
decision = undefined;
|
|
2207
|
+
}
|
|
2208
|
+
try {
|
|
2209
|
+
return await writeOpWaveRecords(options, { cwd, dir, log, result, decision });
|
|
2210
|
+
}
|
|
2211
|
+
finally {
|
|
2212
|
+
if (decision?.lease?.keys.length)
|
|
2213
|
+
await releaseSplitLease(cwd, decision.lease, k, log);
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
/** {@link runOpWaveRecordPhase}'s writes, before it lets go of the lease. */
|
|
2217
|
+
async function writeOpWaveRecords(options, at) {
|
|
2218
|
+
const { wave: k } = options;
|
|
2219
|
+
const { cwd, dir, log, result, decision } = at;
|
|
2220
|
+
const records = waveRunRecordFiles(dir, k, log);
|
|
2221
|
+
if (records.length === 0) {
|
|
2222
|
+
log(`wave ${k}: no applying job of it left a run record under ${relativeTo(cwd, dir)}, so there is none to write`);
|
|
2223
|
+
return { ...result, exitCode: CHANT_EXIT.failed };
|
|
2224
|
+
}
|
|
2225
|
+
const port = options.runs ?? (options.gates ? undefined : gitOpWaveRunPort({ cwd }));
|
|
2226
|
+
const codes = [];
|
|
2227
|
+
const releases = [];
|
|
2228
|
+
const holds = [];
|
|
2229
|
+
let lost = false;
|
|
2230
|
+
for (const { name, record: file } of records) {
|
|
2231
|
+
const { handoff, ...record } = file;
|
|
2232
|
+
releases.push(...(handoff?.releases ?? []));
|
|
2233
|
+
if (handoff?.heldOn?.deferred)
|
|
2234
|
+
holds.push(handoff.heldOn);
|
|
2235
|
+
const w = record.wave;
|
|
2236
|
+
codes.push(w?.exitCode ?? CHANT_EXIT.failed);
|
|
2237
|
+
result.applied.push(...(w?.applied ?? []));
|
|
2238
|
+
result.failed.push(...(w?.failed ?? []));
|
|
2239
|
+
result.moved.push(...(w?.moved ?? []));
|
|
2240
|
+
if (w?.blocked)
|
|
2241
|
+
result.blocked = { ...result.blocked, ...w.blocked };
|
|
2242
|
+
if (!port)
|
|
2243
|
+
continue;
|
|
2244
|
+
try {
|
|
2245
|
+
const wrote = await port.append(record);
|
|
2246
|
+
if (wrote && wrote.pushWarning)
|
|
2247
|
+
log(`wave ${k}: the run record ${name} was written and not pushed: ${wrote.pushWarning}`);
|
|
2248
|
+
}
|
|
2249
|
+
catch (err) {
|
|
2250
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2251
|
+
const never = err instanceof LifecyclePushLostError;
|
|
2252
|
+
if (never)
|
|
2253
|
+
lost = true;
|
|
2254
|
+
log(`wave ${k}: ${never ? `LifecyclePushLostError: the run record ${name} never reached the remote, so the job fails` : `could not record ${name} on the run ledger`}: ${message}`);
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2257
|
+
// #4992: the release records of the runs the applying jobs applied through the step.
|
|
2258
|
+
const releasePort = options.releases === false ? undefined : (options.releases ?? (options.gates ? undefined : gitOpWaveReleasePort({ cwd })));
|
|
2259
|
+
if (releasePort && releases.length > 0) {
|
|
2260
|
+
const missed = await submitStepReleases(releasePort, releases, `wave ${k}`, log);
|
|
2261
|
+
for (const line of missed)
|
|
2262
|
+
log(line);
|
|
2263
|
+
if (missed.length > 0)
|
|
2264
|
+
lost = true;
|
|
2265
|
+
}
|
|
2266
|
+
// #4992: a hold on another gate an applying job's hook found waiting, recorded under the gate job's rules.
|
|
2267
|
+
for (const held of holds) {
|
|
2268
|
+
const rule = { approval: decision?.approval ?? "ledger", base: decision?.base ?? options.base ?? "HEAD", neverOverMcp: decision?.neverOverMcp === true || options.spec.neverOverMcp === true };
|
|
2269
|
+
await recordDeferredHold(options, cwd, held, rule, log);
|
|
2270
|
+
}
|
|
2271
|
+
result.exitCode = codes.every((c) => c === CHANT_EXIT.done) ? CHANT_EXIT.done : (codes.find((c) => c !== CHANT_EXIT.done && c !== CHANT_EXIT.failed) ?? CHANT_EXIT.failed);
|
|
2272
|
+
if (lost && result.exitCode === CHANT_EXIT.done)
|
|
2273
|
+
result.exitCode = CHANT_EXIT.failed;
|
|
2274
|
+
log(`wave ${k}: wrote the run record${records.length === 1 ? "" : "s"} of ${records.length} applying job${records.length === 1 ? "" : "s"}; the wave ends with exit ${result.exitCode}`);
|
|
2275
|
+
return result;
|
|
2276
|
+
}
|
|
1594
2277
|
/** The wave engine's loop over this module's steps, from `options.wave` on, with the runs in `skip` left out of that wave. */
|
|
1595
2278
|
async function runOpWaveLoop(options, log, applied, skip) {
|
|
1596
2279
|
const { spec } = options;
|
|
@@ -1605,7 +2288,8 @@ async function runOpWaveLoop(options, log, applied, skip) {
|
|
|
1605
2288
|
digest: (_at, planned) => (planned.kind === "share" ? planned.decision.digest : opWaveSetDigest(spec, planned.members, planned.extras)),
|
|
1606
2289
|
decide: (at, planned, digest) => (planned.kind === "share" ? shareUnmoved(planned) : decideOpWave(options, at, planned, digest)),
|
|
1607
2290
|
apply: (_at, planned, decided) => applyOpWave(options, planned, decided),
|
|
1608
|
-
|
|
2291
|
+
// #4987: an applying job of a split wave has no plan of its own to make again.
|
|
2292
|
+
replan: (_at, outcome) => outcome.replan !== undefined && options.phase !== "apply",
|
|
1609
2293
|
maxPasses: OP_WAVE_MAX_REPLANS,
|
|
1610
2294
|
passesSpent({ k }, outcome) {
|
|
1611
2295
|
log(`wave ${k}: its apply asked to plan again ${OP_WAVE_MAX_REPLANS} times, so the wave stops here`);
|
|
@@ -1620,7 +2304,7 @@ async function runOpWaveLoop(options, log, applied, skip) {
|
|
|
1620
2304
|
Object.assign(targets, outcome.targets ?? {});
|
|
1621
2305
|
// #3821: a `rest` wave goes on through the waves its plan command found.
|
|
1622
2306
|
const found = outcome.decision?.discovered?.waves;
|
|
1623
|
-
const onward = k >= spec.waves.length && last.rest === true && options.share === undefined && options.group === undefined && !options.decide && outcome.exitCode === 0 && found !== undefined && k < found;
|
|
2307
|
+
const onward = k >= spec.waves.length && last.rest === true && options.share === undefined && options.group === undefined && !options.decide && options.phase === undefined && outcome.exitCode === 0 && found !== undefined && k < found;
|
|
1624
2308
|
if (rest.length > 0 || onward)
|
|
1625
2309
|
rest.push({ wave: k, exitCode: outcome.exitCode, ...(outcome.decision ? { digest: outcome.decision.digest } : {}) });
|
|
1626
2310
|
if (onward)
|
|
@@ -1646,10 +2330,11 @@ async function runOpWaveLoop(options, log, applied, skip) {
|
|
|
1646
2330
|
*/
|
|
1647
2331
|
function opWaveBefore(options, dir) {
|
|
1648
2332
|
const { spec, wave: k } = options;
|
|
1649
|
-
const wave = spec
|
|
2333
|
+
const wave = opWaveAt(spec, k);
|
|
1650
2334
|
if (!wave)
|
|
1651
2335
|
return { blocked: new Map() };
|
|
1652
|
-
|
|
2336
|
+
// #4987: a split wave's applying job reads its gate job's outcome, as a share reads its deciding job's.
|
|
2337
|
+
if (options.share !== undefined || options.group !== undefined || options.phase === "apply") {
|
|
1653
2338
|
const decided = readWaveOutcomeFile([dir], waveOutcomeFileName(k));
|
|
1654
2339
|
if (!decided)
|
|
1655
2340
|
return { blocked: new Map() };
|
|
@@ -1699,7 +2384,7 @@ function opWaveBefore(options, dir) {
|
|
|
1699
2384
|
}
|
|
1700
2385
|
/** The runs of wave `options.wave` this job holds (#4322): its slice for a share job, every run otherwise. Empty for a discovering wave. */
|
|
1701
2386
|
function jobRuns(options, runs) {
|
|
1702
|
-
const wave = options.spec
|
|
2387
|
+
const wave = opWaveAt(options.spec, options.wave);
|
|
1703
2388
|
if (!wave)
|
|
1704
2389
|
return [];
|
|
1705
2390
|
const all = runs ?? (wave.discover ? [] : wave.runs);
|
|
@@ -1718,7 +2403,8 @@ function jobRuns(options, runs) {
|
|
|
1718
2403
|
function writeOpWaveOutcome(options, dir, result, members) {
|
|
1719
2404
|
const out = { ...(members ?? {}) };
|
|
1720
2405
|
if (!members) {
|
|
1721
|
-
|
|
2406
|
+
// #4987: a split wave's gate job decides the runs its plan job planned, which the spec at the default branch may not list.
|
|
2407
|
+
const found = result.decision?.discovered !== undefined || (options.phase === "gate" && result.decision) ? result.decision.members.map((m) => ({ target: m.member })) : undefined;
|
|
1722
2408
|
const targets = new Set([...jobRuns(options, found), ...result.applied, ...result.failed, ...result.moved, ...Object.keys(result.blocked ?? {})]);
|
|
1723
2409
|
const applied = new Set(result.applied);
|
|
1724
2410
|
const failed = new Set(result.failed);
|
|
@@ -1733,7 +2419,7 @@ function writeOpWaveOutcome(options, dir, result, members) {
|
|
|
1733
2419
|
out[t] = by !== undefined ? { status: "blocked", by } : applied.has(t) ? { status: "applied" } : failed.has(t) ? { status: "failed" } : moved.has(t) && result.decision?.refused?.reason !== "moved" ? { status: "not-applied" } : { status: otherwise };
|
|
1734
2420
|
}
|
|
1735
2421
|
}
|
|
1736
|
-
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}) };
|
|
2422
|
+
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}), ...(options.phase === "apply" ? { apply: true } : {}) };
|
|
1737
2423
|
try {
|
|
1738
2424
|
mkdirSync(dir, { recursive: true });
|
|
1739
2425
|
writeWaveOutcomeFile(dir, options.wave, part, out, options.now ?? new Date().toISOString());
|
|
@@ -1744,12 +2430,80 @@ function writeOpWaveOutcome(options, dir, result, members) {
|
|
|
1744
2430
|
}
|
|
1745
2431
|
/** What a share or job group's job calls itself in the log: `share 2`, `job group "prod"`, `share 2 of job group "prod"`. */
|
|
1746
2432
|
function shareLabel(options) {
|
|
2433
|
+
// #4987: a split wave's own applying job is neither a share nor a group's.
|
|
2434
|
+
if (options.group === undefined && options.share === undefined)
|
|
2435
|
+
return "the applying job";
|
|
1747
2436
|
if (options.group === undefined)
|
|
1748
2437
|
return `share ${options.share}`;
|
|
1749
2438
|
return `${options.share !== undefined ? `share ${options.share} of ` : ""}job group "${options.group}"`;
|
|
1750
2439
|
}
|
|
1751
2440
|
/** How many times one wave job plans again when its apply asks to (#3821). */
|
|
1752
2441
|
export const OP_WAVE_MAX_REPLANS = 10;
|
|
2442
|
+
/**
|
|
2443
|
+
* Read wave `k`'s rules at base and open its decision: the policy, the
|
|
2444
|
+
* approval mode, the hooks and the points, checked as the wave plans
|
|
2445
|
+
* (#3836, #4290, #4870), with the holds the rules give. Ends the wave when
|
|
2446
|
+
* the rules cannot be read or refuse it. The plan job and the gate job of a
|
|
2447
|
+
* split wave both read them (#4987); `evidence: false` leaves the evidence
|
|
2448
|
+
* the commit needs to the gate job.
|
|
2449
|
+
*/
|
|
2450
|
+
async function openWaveDecision(options, ctx, opts) {
|
|
2451
|
+
const { spec, k, wave, cwd, log } = ctx;
|
|
2452
|
+
const rules = readOpWavePolicy(options.specFile, wave.name, options.base ?? spec.base ?? "HEAD^1", cwd, options.show);
|
|
2453
|
+
const { policy, approval, source, base } = rules;
|
|
2454
|
+
const neverOverMcp = rules.neverOverMcp || spec.neverOverMcp === true;
|
|
2455
|
+
ctx.gateRule = { approval, base, neverOverMcp };
|
|
2456
|
+
noteHooksAtBase(ctx, rules);
|
|
2457
|
+
// #3836: a point at base that cannot be asked fails the wave; the ledger never stands in for it.
|
|
2458
|
+
if (rules.pointRefused)
|
|
2459
|
+
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.pointRefused}, so nothing in it was planned or applied`) };
|
|
2460
|
+
// #4290: rules at base that cannot be read fail the wave; fewer approvers never stand in for them.
|
|
2461
|
+
if (rules.quorumRefused)
|
|
2462
|
+
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.quorumRefused}, so nothing in it was planned or applied`) };
|
|
2463
|
+
if (rules.quorum && rules.point)
|
|
2464
|
+
return { end: failed(ctx, `wave ${k} (${wave.name}): approvals.quorum and a decision point are both set at ${base.slice(0, 12)}; a gate that asks a point takes its deciders from the point, so nothing in it was planned or applied`) };
|
|
2465
|
+
// #4870: the checks whose evidence the commit needs, read at base, before anything is planned.
|
|
2466
|
+
if (rules.evidenceRefused)
|
|
2467
|
+
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.evidenceRefused}, so nothing in it was planned or applied`) };
|
|
2468
|
+
if (rules.evidenceRequires && opts.evidence) {
|
|
2469
|
+
const refused = await opWaveEvidenceRefusal(cwd, options.head ?? "HEAD", base, rules.evidenceRequires);
|
|
2470
|
+
if (refused)
|
|
2471
|
+
return { end: failed(ctx, `wave ${k} (${wave.name}): ${refused}, so nothing in it was planned or applied`) };
|
|
2472
|
+
}
|
|
2473
|
+
if (JSON.stringify(spec.approvals?.quorum ?? null) !== JSON.stringify(rules.quorum ?? null)) {
|
|
2474
|
+
log(`wave ${k} (${wave.name}): its approval rules are read at ${base.slice(0, 12)}, and this change's differ; the ones at base hold`);
|
|
2475
|
+
}
|
|
2476
|
+
const own = opWavePointAt(spec, wave.name);
|
|
2477
|
+
if (JSON.stringify(own) !== JSON.stringify(rules.point)) {
|
|
2478
|
+
log(`wave ${k} (${wave.name}): its gate's decision point is read at ${base.slice(0, 12)} (${rules.point?.name ?? "none"}), and this change's differs (${own?.name ?? "none"}); the one at base holds`);
|
|
2479
|
+
}
|
|
2480
|
+
const holds = [];
|
|
2481
|
+
if (rules.hooksRefused) {
|
|
2482
|
+
holds.push(rules.hooksRefused);
|
|
2483
|
+
log(`wave ${k} (${wave.name}): ${rules.hooksRefused}, so none run and the wave waits for an approval`);
|
|
2484
|
+
}
|
|
2485
|
+
const hooks = Object.fromEntries(Object.entries(rules.hooks).filter(([, list]) => list.length > 0));
|
|
2486
|
+
const decision = {
|
|
2487
|
+
wave: k,
|
|
2488
|
+
name: wave.name,
|
|
2489
|
+
op: spec.name,
|
|
2490
|
+
gate: opWaveGateName(spec, k),
|
|
2491
|
+
policy,
|
|
2492
|
+
approval,
|
|
2493
|
+
policySource: source,
|
|
2494
|
+
base,
|
|
2495
|
+
digest: "",
|
|
2496
|
+
members: [],
|
|
2497
|
+
status: "not-required",
|
|
2498
|
+
...(Object.keys(hooks).length > 0 ? { hooks } : {}),
|
|
2499
|
+
...(rules.spend ? { spend: "base" } : spec.approvals?.spend === true ? { spend: "change" } : {}),
|
|
2500
|
+
...(rules.applyHooksRunBy === "apply" ? { applyHooksRunBy: "apply" } : {}),
|
|
2501
|
+
...(spec.digest?.over === "changing" ? { digestOver: "changing" } : {}),
|
|
2502
|
+
...(neverOverMcp ? { neverOverMcp: true } : {}),
|
|
2503
|
+
...(rules.point ? { point: { ...rules.point } } : {}),
|
|
2504
|
+
};
|
|
2505
|
+
return { go: { rules, holds, decision } };
|
|
2506
|
+
}
|
|
1753
2507
|
/**
|
|
1754
2508
|
* Plan wave `k`: read its rules at base and run the before-plan hooks
|
|
1755
2509
|
* around planning its runs, or for a share job, read the deciding job's
|
|
@@ -1786,7 +2540,8 @@ async function planOpWave(options, { k, pass }, skip) {
|
|
|
1786
2540
|
const ctx = { spec, k, wave, cwd, exec, log, result, options, gateRule: { approval: "ledger", base: "HEAD", neverOverMcp: spec.neverOverMcp === true } };
|
|
1787
2541
|
if (pass > 0)
|
|
1788
2542
|
log(`wave ${k} (${wave.name}): plan ${pass + 1}`);
|
|
1789
|
-
|
|
2543
|
+
// #4987: a split wave's applying job applies under its gate job's decision, as a share does.
|
|
2544
|
+
if (options.share !== undefined || group !== undefined || options.phase === "apply") {
|
|
1790
2545
|
let decision;
|
|
1791
2546
|
try {
|
|
1792
2547
|
decision = JSON.parse(readFileSync(recordPath, "utf-8"));
|
|
@@ -1815,6 +2570,22 @@ async function planOpWave(options, { k, pass }, skip) {
|
|
|
1815
2570
|
const slice = whole.filter((r) => !skip?.has(r.target));
|
|
1816
2571
|
if (slice.length === 0)
|
|
1817
2572
|
return { end: result };
|
|
2573
|
+
// #4987: an applying job of a split wave plans nothing and runs no plan hooks (its plan and gate jobs did): it takes
|
|
2574
|
+
// the plan files the plan job saved, and shareUnmoved applies those whose digest the decision holds.
|
|
2575
|
+
if (options.phase === "apply" && spec.step) {
|
|
2576
|
+
// #4992: a step's plan lives in the process that made it (the engine's saved plan, the artifact it hands its
|
|
2577
|
+
// apply), so the applying job plans its runs again through the step, as a share does, and shareUnmoved applies
|
|
2578
|
+
// each whose plan digest the decision holds. Its hooks ran in the plan and gate jobs.
|
|
2579
|
+
ctx.steps = new Map();
|
|
2580
|
+
const { members } = await planWaveRuns(ctx, slice);
|
|
2581
|
+
return { go: { kind: "share", ctx, decision, slice, planned: new Map(members.map((m) => [m.member, m])), holds: [] } };
|
|
2582
|
+
}
|
|
2583
|
+
if (options.phase === "apply") {
|
|
2584
|
+
const saved = savedRunPlans(ctx, slice);
|
|
2585
|
+
if ("failed" in saved)
|
|
2586
|
+
return { end: failed(ctx, saved.failed) };
|
|
2587
|
+
return { go: { kind: "share", ctx, decision, slice, planned: saved.planned, holds: [] } };
|
|
2588
|
+
}
|
|
1818
2589
|
const before = await runBeforePlanHooks(ctx, decision.hooks, slice);
|
|
1819
2590
|
if (before.failed)
|
|
1820
2591
|
return { end: failed(ctx, `a hook before the plan failed, so nothing in ${part} of wave ${k} was planned or applied`) };
|
|
@@ -1836,60 +2607,12 @@ async function planOpWave(options, { k, pass }, skip) {
|
|
|
1836
2607
|
return { go: { kind: "share", ctx, decision, slice, planned, holds: [...before.holds, ...inProcess.holds, ...after.holds] } };
|
|
1837
2608
|
}
|
|
1838
2609
|
// The policy, the approval mode and the hooks, read at base before anything runs.
|
|
1839
|
-
|
|
1840
|
-
const
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
if (rules.pointRefused)
|
|
1846
|
-
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.pointRefused}, so nothing in it was planned or applied`) };
|
|
1847
|
-
// #4290: rules at base that cannot be read fail the wave; fewer approvers never stand in for them.
|
|
1848
|
-
if (rules.quorumRefused)
|
|
1849
|
-
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.quorumRefused}, so nothing in it was planned or applied`) };
|
|
1850
|
-
if (rules.quorum && rules.point)
|
|
1851
|
-
return { end: failed(ctx, `wave ${k} (${wave.name}): approvals.quorum and a decision point are both set at ${base.slice(0, 12)}; a gate that asks a point takes its deciders from the point, so nothing in it was planned or applied`) };
|
|
1852
|
-
// #4870: the checks whose evidence the commit needs, read at base, before anything is planned.
|
|
1853
|
-
if (rules.evidenceRefused)
|
|
1854
|
-
return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.evidenceRefused}, so nothing in it was planned or applied`) };
|
|
1855
|
-
if (rules.evidenceRequires) {
|
|
1856
|
-
const refused = await opWaveEvidenceRefusal(cwd, options.head ?? "HEAD", base, rules.evidenceRequires);
|
|
1857
|
-
if (refused)
|
|
1858
|
-
return { end: failed(ctx, `wave ${k} (${wave.name}): ${refused}, so nothing in it was planned or applied`) };
|
|
1859
|
-
}
|
|
1860
|
-
if (JSON.stringify(spec.approvals?.quorum ?? null) !== JSON.stringify(rules.quorum ?? null)) {
|
|
1861
|
-
log(`wave ${k} (${wave.name}): its approval rules are read at ${base.slice(0, 12)}, and this change's differ; the ones at base hold`);
|
|
1862
|
-
}
|
|
1863
|
-
const own = opWavePointAt(spec, wave.name);
|
|
1864
|
-
if (JSON.stringify(own) !== JSON.stringify(rules.point)) {
|
|
1865
|
-
log(`wave ${k} (${wave.name}): its gate's decision point is read at ${base.slice(0, 12)} (${rules.point?.name ?? "none"}), and this change's differs (${own?.name ?? "none"}); the one at base holds`);
|
|
1866
|
-
}
|
|
1867
|
-
const holds = [];
|
|
1868
|
-
if (rules.hooksRefused) {
|
|
1869
|
-
holds.push(rules.hooksRefused);
|
|
1870
|
-
log(`wave ${k} (${wave.name}): ${rules.hooksRefused}, so none run and the wave waits for an approval`);
|
|
1871
|
-
}
|
|
1872
|
-
const gate = opWaveGateName(spec, k);
|
|
1873
|
-
const hooks = Object.fromEntries(Object.entries(rules.hooks).filter(([, list]) => list.length > 0));
|
|
1874
|
-
const decision = {
|
|
1875
|
-
wave: k,
|
|
1876
|
-
name: wave.name,
|
|
1877
|
-
op: spec.name,
|
|
1878
|
-
gate,
|
|
1879
|
-
policy,
|
|
1880
|
-
approval,
|
|
1881
|
-
policySource: source,
|
|
1882
|
-
base,
|
|
1883
|
-
digest: "",
|
|
1884
|
-
members: [],
|
|
1885
|
-
status: "not-required",
|
|
1886
|
-
...(Object.keys(hooks).length > 0 ? { hooks } : {}),
|
|
1887
|
-
...(rules.spend ? { spend: "base" } : spec.approvals?.spend === true ? { spend: "change" } : {}),
|
|
1888
|
-
...(rules.applyHooksRunBy === "apply" ? { applyHooksRunBy: "apply" } : {}),
|
|
1889
|
-
...(spec.digest?.over === "changing" ? { digestOver: "changing" } : {}),
|
|
1890
|
-
...(neverOverMcp ? { neverOverMcp: true } : {}),
|
|
1891
|
-
...(rules.point ? { point: { ...rules.point } } : {}),
|
|
1892
|
-
};
|
|
2610
|
+
// #4987: a split wave's plan job leaves the evidence its commit needs to the gate job, which can fetch the notes.
|
|
2611
|
+
const opened = await openWaveDecision(options, ctx, { evidence: options.phase !== "plan" });
|
|
2612
|
+
if ("end" in opened)
|
|
2613
|
+
return opened;
|
|
2614
|
+
const { rules, holds, decision } = opened.go;
|
|
2615
|
+
const { base } = rules;
|
|
1893
2616
|
if (skip?.size && !wave.discover) {
|
|
1894
2617
|
// #4322: a run a failure blocked is neither planned nor in the set digest.
|
|
1895
2618
|
wave = { ...wave, runs: wave.runs.filter((r) => !skip.has(r.target)) };
|
|
@@ -1901,6 +2624,11 @@ async function planOpWave(options, { k, pass }, skip) {
|
|
|
1901
2624
|
// #4926: the policy step records no decision; the wave's own step does.
|
|
1902
2625
|
if (before.waitingOn && options.policyOnly)
|
|
1903
2626
|
return { end: failed(ctx, `wave ${k} (${wave.name}): a hook before the plan waits on ${before.waitingOn}, so its policy did not run`) };
|
|
2627
|
+
if (before.waitingOn && options.phase === "plan") {
|
|
2628
|
+
// #4992: the plan job ends the wave waiting on the other gate; its gate job records that gate's pending fact and the decision.
|
|
2629
|
+
markHeldOn(decision, before.waitingOn);
|
|
2630
|
+
return { end: { ...result, decision, exitCode: CHANT_EXIT.waiting } };
|
|
2631
|
+
}
|
|
1904
2632
|
if (before.waitingOn)
|
|
1905
2633
|
return { end: await decided(ctx, decision, recordPath, before.waitingOn) };
|
|
1906
2634
|
if (spec.step)
|
|
@@ -2105,9 +2833,18 @@ async function decideOpWave(options, { prior }, planned, digest) {
|
|
|
2105
2833
|
(holds.length > 0 ? `; held by ${holds.join(", ")}` : ""));
|
|
2106
2834
|
// A wide wave's shares apply under the deciding job's approval and overrides, so they are spent here, once.
|
|
2107
2835
|
if (options.decide) {
|
|
2108
|
-
|
|
2836
|
+
// #4992: a split wave's gate job holds the rows its step's runs change for the applying jobs, before it spends the approval.
|
|
2837
|
+
const lease = options.phase === "gate" && ctx.spec.step ? await leaseSplitRows(options, ctx, decision, recordPath) : undefined;
|
|
2838
|
+
if (lease && "end" in lease)
|
|
2839
|
+
return lease;
|
|
2840
|
+
if (!(await spendApproval(options, ctx, decision)) || !(await spendOverrides(ctx, decision))) {
|
|
2841
|
+
if (lease && !(await lease.release()))
|
|
2842
|
+
log(`wave ${k} (${wave.name}): could not let go of the rows it held; the next apply takes them over once this run is gone`);
|
|
2843
|
+
if (lease)
|
|
2844
|
+
delete decision.lease;
|
|
2109
2845
|
return { end: ctx.result };
|
|
2110
|
-
|
|
2846
|
+
}
|
|
2847
|
+
if (decision.spentAt || decision.overridesSpentAt || decision.lease)
|
|
2111
2848
|
writeRecord(recordPath, decision, result);
|
|
2112
2849
|
return { end: result };
|
|
2113
2850
|
}
|
|
@@ -2268,6 +3005,11 @@ async function decided(ctx, decision, recordPath, heldOn) {
|
|
|
2268
3005
|
log(`The pull request's review does not approve wave ${k}: ${decision.review.reason}`);
|
|
2269
3006
|
}
|
|
2270
3007
|
const hook = await runGateHooks(ctx, decision, recordPath);
|
|
3008
|
+
if (decision.status === "waiting" && decision.heldOn?.spentForRerun) {
|
|
3009
|
+
// #4994: the approval is spent for the held hook's re-run; nobody needs to approve anything more.
|
|
3010
|
+
log(`Nothing in wave ${k} or after it ran. The approval of ${decision.heldOn.op} / ${decision.heldOn.gate} by ${decision.heldOn.spentForRerun.approvedBy} is spent for ${decision.heldOn.hook} to run again; run the wave again from its plan job`);
|
|
3011
|
+
return { ...result, exitCode: CHANT_EXIT.waiting };
|
|
3012
|
+
}
|
|
2271
3013
|
if (decision.status === "waiting") {
|
|
2272
3014
|
log(`Nothing in wave ${k} or after it ran. ${decision.heldOn ? `It waits on ${decision.heldOn.op} / ${decision.heldOn.gate}.` : "Read the plans, then"} approve: ${decision.approve} (with --resume, the approval also starts this job again)`);
|
|
2273
3015
|
return { ...result, exitCode: CHANT_EXIT.waiting };
|
|
@@ -3046,6 +3788,9 @@ async function applyStage(options, ctx, decision, runs, planned, earlierHolds, r
|
|
|
3046
3788
|
}
|
|
3047
3789
|
if (before.waitingOn) {
|
|
3048
3790
|
markHeldOn(decision, before.waitingOn);
|
|
3791
|
+
// #4992: an applying job's hold goes to its record job, which records the other gate's pending fact.
|
|
3792
|
+
if (before.waitingOn.deferred)
|
|
3793
|
+
result.handoff = { ...result.handoff, heldOn: before.waitingOn };
|
|
3049
3794
|
if (recordPath) {
|
|
3050
3795
|
writeRecord(recordPath, decision, result);
|
|
3051
3796
|
await runGateHooks(ctx, decision, recordPath);
|
|
@@ -3059,7 +3804,8 @@ async function applyStage(options, ctx, decision, runs, planned, earlierHolds, r
|
|
|
3059
3804
|
if (!(await settleHolds(options, ctx, decision, holds, recordPath)))
|
|
3060
3805
|
return;
|
|
3061
3806
|
// #4552: on the resource grain the wave holds the rows its runs change, before its approval is spent.
|
|
3062
|
-
|
|
3807
|
+
// #4992: a split wave's gate job holds them for its applying jobs, which hold no write token.
|
|
3808
|
+
const lease = spec.step && options.phase !== "apply" ? await leaseWaveRows(options, ctx, runs) : undefined;
|
|
3063
3809
|
if (lease && "failed" in lease) {
|
|
3064
3810
|
failed(ctx, lease.failed);
|
|
3065
3811
|
return;
|
|
@@ -3412,12 +4158,23 @@ async function applyStepRunsAtOnce(options, ctx, runs, limit) {
|
|
|
3412
4158
|
* 9), which fails the wave by name.
|
|
3413
4159
|
*/
|
|
3414
4160
|
async function recordStepReleases(options, ctx, decision, runs) {
|
|
3415
|
-
const { spec, k, wave, log, result } = ctx;
|
|
3416
4161
|
if (options.releases === false)
|
|
3417
4162
|
return;
|
|
4163
|
+
// #4992: a split wave's applying job holds no write token; its record job writes the releases.
|
|
4164
|
+
if (options.phase === "apply") {
|
|
4165
|
+
const requests = stepReleaseRequests(options, ctx, decision, runs);
|
|
4166
|
+
if (requests.length > 0)
|
|
4167
|
+
ctx.result.handoff = { ...ctx.result.handoff, releases: requests };
|
|
4168
|
+
return;
|
|
4169
|
+
}
|
|
3418
4170
|
const port = options.releases ?? (options.gates ? undefined : gitOpWaveReleasePort({ cwd: ctx.cwd }));
|
|
3419
4171
|
if (!port)
|
|
3420
4172
|
return;
|
|
4173
|
+
await writeStepReleases(port, ctx, stepReleaseRequests(options, ctx, decision, runs));
|
|
4174
|
+
}
|
|
4175
|
+
/** The release requests of the runs this wave applied through its step (#4615). */
|
|
4176
|
+
function stepReleaseRequests(options, ctx, decision, runs) {
|
|
4177
|
+
const { spec, wave, result } = ctx;
|
|
3421
4178
|
const applied = new Set(result.applied);
|
|
3422
4179
|
let gitSha = options.head;
|
|
3423
4180
|
const requests = [];
|
|
@@ -3438,25 +4195,40 @@ async function recordStepReleases(options, ctx, decision, runs) {
|
|
|
3438
4195
|
}
|
|
3439
4196
|
requests.push({ ...names, target: run.target, planDigest: plan.planDigest, ...(gitSha ? { gitSha } : {}), ...(decision.approvedBy ? { approver: decision.approvedBy } : {}) });
|
|
3440
4197
|
}
|
|
4198
|
+
return requests;
|
|
4199
|
+
}
|
|
4200
|
+
/** Write release requests (#4615), failing the wave through `ctx` when one never reached the remote. */
|
|
4201
|
+
async function writeStepReleases(port, ctx, requests) {
|
|
4202
|
+
for (const lost of await submitStepReleases(port, requests, `wave ${ctx.k} (${ctx.wave.name})`, ctx.log))
|
|
4203
|
+
failed(ctx, lost);
|
|
4204
|
+
}
|
|
4205
|
+
/**
|
|
4206
|
+
* Write release requests on their release ledgers (#4615): a line per run
|
|
4207
|
+
* recorded or not. Returns a failure for each one written that never
|
|
4208
|
+
* reached the remote (plywood#22 finding 9), which fails the wave.
|
|
4209
|
+
*/
|
|
4210
|
+
async function submitStepReleases(port, requests, label, log) {
|
|
3441
4211
|
if (requests.length === 0)
|
|
3442
|
-
return;
|
|
4212
|
+
return [];
|
|
3443
4213
|
let outcomes;
|
|
3444
4214
|
try {
|
|
3445
|
-
outcomes = await port.record(requests);
|
|
4215
|
+
outcomes = await port.record([...requests]);
|
|
3446
4216
|
}
|
|
3447
4217
|
catch (err) {
|
|
3448
|
-
log(
|
|
3449
|
-
return;
|
|
4218
|
+
log(`${label}: applied, and no release record was written: ${err instanceof Error ? err.message : String(err)}`);
|
|
4219
|
+
return [];
|
|
3450
4220
|
}
|
|
4221
|
+
const lost = [];
|
|
3451
4222
|
for (const [i, req] of requests.entries()) {
|
|
3452
4223
|
const got = outcomes[i];
|
|
3453
4224
|
if (got?.recorded)
|
|
3454
4225
|
log(`${req.target}: recorded on the "${got.env}" release ledger as ${got.component} ${got.digest}${got.releaseDigest ? ` under release ${got.releaseDigest}` : ""}`);
|
|
3455
4226
|
else if (got && !got.recorded && got.lost)
|
|
3456
|
-
|
|
4227
|
+
lost.push(`${req.target}: applied, and LifecyclePushLostError: the release record for ${req.component} in "${req.env}" was written and never reached the remote, so the wave fails: ${got.reason}`);
|
|
3457
4228
|
else
|
|
3458
4229
|
log(`${req.target}: applied, and no release record was written for ${req.component} in "${req.env}": ${got?.reason ?? "the port gave no answer"}`);
|
|
3459
4230
|
}
|
|
4231
|
+
return lost;
|
|
3460
4232
|
}
|
|
3461
4233
|
/**
|
|
3462
4234
|
* Record that the wave applies under its approval (#3784), which spends it:
|