@intentius/chant 0.124.89 → 0.124.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci-comment-commands.d.ts +18 -2
- package/dist/ci-comment-commands.d.ts.map +1 -1
- package/dist/ci-comment-commands.js +30 -2
- package/dist/ci-job-guard.d.ts +42 -45
- package/dist/ci-job-guard.d.ts.map +1 -1
- package/dist/ci-job-guard.js +206 -3
- package/dist/ci-note-commands.d.ts +41 -1
- package/dist/ci-note-commands.d.ts.map +1 -1
- package/dist/ci-note-commands.js +64 -5
- package/dist/ci-product.d.ts +41 -1
- package/dist/ci-product.d.ts.map +1 -1
- package/dist/ci-product.js +44 -7
- package/dist/cli/handlers/pr-comment.d.ts +13 -1
- package/dist/cli/handlers/pr-comment.d.ts.map +1 -1
- package/dist/cli/handlers/pr-comment.js +90 -1
- package/dist/cli/handlers/pr-notes.d.ts.map +1 -1
- package/dist/cli/handlers/pr-notes.js +23 -1
- package/dist/cli/handlers/pr.js +16 -14
- package/dist/cli/handlers/run-wave.d.ts.map +1 -1
- package/dist/cli/handlers/run-wave.js +154 -4
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +17 -0
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +15 -0
- package/dist/cli/registry.d.ts +8 -1
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/components/pr-checks.d.ts +28 -1
- package/dist/components/pr-checks.d.ts.map +1 -1
- package/dist/components/pr-checks.js +35 -2
- package/dist/components/pr-pipeline.d.ts +3 -2
- package/dist/components/pr-pipeline.d.ts.map +1 -1
- package/dist/components/pr-pipeline.js +4 -2
- package/dist/components/product-pipeline.d.ts +54 -2
- package/dist/components/product-pipeline.d.ts.map +1 -1
- package/dist/components/product-pipeline.js +59 -5
- package/dist/components/product-plan.d.ts +13 -1
- package/dist/components/product-plan.d.ts.map +1 -1
- package/dist/components/product-plan.js +36 -10
- package/dist/floor-table.data.d.ts.map +1 -1
- package/dist/floor-table.data.js +32 -23
- package/dist/lexicon.d.ts +82 -4
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/lifecycle/gate-applied.d.ts +9 -0
- package/dist/lifecycle/gate-applied.d.ts.map +1 -1
- package/dist/op/apply-status.d.ts +84 -0
- package/dist/op/apply-status.d.ts.map +1 -0
- package/dist/op/apply-status.js +106 -0
- package/dist/op/before-merge-decide.d.ts +102 -0
- package/dist/op/before-merge-decide.d.ts.map +1 -0
- package/dist/op/before-merge-decide.js +142 -0
- package/dist/op/before-merge-done.d.ts +56 -0
- package/dist/op/before-merge-done.d.ts.map +1 -0
- package/dist/op/before-merge-done.js +69 -0
- package/dist/op/before-merge-store.d.ts +20 -0
- package/dist/op/before-merge-store.d.ts.map +1 -0
- package/dist/op/before-merge-store.js +48 -0
- package/dist/op/before-merge.d.ts +150 -0
- package/dist/op/before-merge.d.ts.map +1 -0
- package/dist/op/before-merge.js +207 -0
- package/dist/op/follow-on-run.d.ts +117 -0
- package/dist/op/follow-on-run.d.ts.map +1 -0
- package/dist/op/follow-on-run.js +154 -0
- package/dist/op/follow-on-store.d.ts +17 -0
- package/dist/op/follow-on-store.d.ts.map +1 -0
- package/dist/op/follow-on-store.js +54 -0
- package/dist/op/gate.d.ts +6 -0
- package/dist/op/gate.d.ts.map +1 -1
- package/dist/op/gate.js +10 -1
- package/dist/op/index.d.ts +3 -1
- package/dist/op/index.d.ts.map +1 -1
- package/dist/op/index.js +1 -0
- package/dist/op/op-waves-after-plan.d.ts +13 -0
- package/dist/op/op-waves-after-plan.d.ts.map +1 -1
- package/dist/op/op-waves-evidence-jobs.d.ts +4 -1
- package/dist/op/op-waves-evidence-jobs.d.ts.map +1 -1
- package/dist/op/op-waves-evidence-jobs.js +10 -2
- package/dist/op/op-waves-follow-on.d.ts +162 -0
- package/dist/op/op-waves-follow-on.d.ts.map +1 -0
- package/dist/op/op-waves-follow-on.js +355 -0
- package/dist/op/op-waves-run.d.ts +146 -6
- package/dist/op/op-waves-run.d.ts.map +1 -1
- package/dist/op/op-waves-run.js +871 -99
- package/dist/op/op-waves-split.d.ts +122 -0
- package/dist/op/op-waves-split.d.ts.map +1 -0
- package/dist/op/op-waves-split.js +166 -0
- package/dist/op/op-waves.d.ts +20 -0
- package/dist/op/op-waves.d.ts.map +1 -1
- package/dist/op/op-waves.js +15 -1
- package/dist/op/schedule-overdue.d.ts +15 -1
- package/dist/op/schedule-overdue.d.ts.map +1 -1
- package/dist/op/schedule-overdue.js +18 -2
- package/dist/op/stage-hooks.d.ts +36 -1
- package/dist/op/stage-hooks.d.ts.map +1 -1
- package/dist/op/stage-hooks.js +17 -3
- package/dist/op/wave-stop.d.ts +4 -1
- package/dist/op/wave-stop.d.ts.map +1 -1
- package/dist/op/wave-stop.js +4 -2
- package/dist/pr-post.d.ts.map +1 -1
- package/dist/pr-post.js +2 -1
- package/dist/release-floors.d.ts +3 -1
- package/dist/release-floors.d.ts.map +1 -1
- package/dist/release-floors.js +21 -19
- package/dist/wave-blocked.d.ts +7 -1
- package/dist/wave-blocked.d.ts.map +1 -1
- package/dist/wave-blocked.js +7 -2
- package/dist/wave-runtime.mjs +364 -351
- package/package.json +1 -1
- package/src/ci-comment-commands.test.ts +63 -0
- package/src/ci-comment-commands.ts +43 -4
- package/src/ci-image.test.ts +1 -2
- package/src/ci-job-guard.before-merge.test.ts +148 -0
- package/src/ci-job-guard.follow-on-record.test.ts +137 -0
- package/src/ci-job-guard.follow-on.test.ts +144 -0
- package/src/ci-job-guard.ts +221 -3
- package/src/ci-note-commands.test.ts +76 -1
- package/src/ci-note-commands.ts +96 -7
- package/src/ci-product.ts +55 -7
- package/src/cli/handlers/pr-comment.ts +85 -2
- package/src/cli/handlers/pr-notes.ts +30 -6
- package/src/cli/handlers/pr.ts +14 -11
- package/src/cli/handlers/run-wave.ts +144 -4
- package/src/cli/main.ts +17 -0
- package/src/cli/parse-args.ts +11 -0
- package/src/cli/registry.ts +8 -1
- package/src/components/pr-checks.ts +55 -2
- package/src/components/pr-pipeline.ts +5 -3
- package/src/components/product-pipeline.apply-status.test.ts +50 -0
- package/src/components/product-pipeline.ts +113 -8
- package/src/components/product-plan.test.ts +23 -5
- package/src/components/product-plan.ts +43 -12
- package/src/floor-table.data.ts +32 -23
- package/src/lexicon.ts +78 -1
- package/src/lifecycle/gate-applied.ts +7 -0
- package/src/op/apply-status.test.ts +83 -0
- package/src/op/apply-status.ts +132 -0
- package/src/op/before-merge-decide.ts +190 -0
- package/src/op/before-merge-done.ts +99 -0
- package/src/op/before-merge-store.ts +44 -0
- package/src/op/before-merge.test.ts +201 -0
- package/src/op/before-merge.ts +266 -0
- package/src/op/follow-on-run.ts +206 -0
- package/src/op/follow-on-store.ts +54 -0
- package/src/op/gate.ts +16 -1
- package/src/op/index.ts +5 -0
- package/src/op/op-waves-after-plan.ts +13 -0
- package/src/op/op-waves-evidence-jobs.ts +11 -3
- package/src/op/op-waves-follow-on-record.test.ts +152 -0
- package/src/op/op-waves-follow-on.test.ts +247 -0
- package/src/op/op-waves-follow-on.ts +385 -0
- package/src/op/op-waves-render-spec.test.ts +1 -2
- package/src/op/op-waves-run.ts +971 -93
- package/src/op/op-waves-split-product.test.ts +288 -0
- package/src/op/op-waves-split-step.test.ts +298 -0
- package/src/op/op-waves-split.test.ts +196 -0
- package/src/op/op-waves-split.ts +189 -0
- package/src/op/op-waves.ts +28 -3
- package/src/op/schedule-overdue.test.ts +29 -1
- package/src/op/schedule-overdue.ts +27 -3
- package/src/op/stage-hooks.ts +52 -4
- package/src/op/wave-stop.ts +5 -3
- package/src/pr-post.ts +2 -1
- package/src/release-floors.ts +22 -19
- package/src/wave-blocked.ts +9 -4
package/src/op/op-waves-run.ts
CHANGED
|
@@ -51,8 +51,9 @@
|
|
|
51
51
|
|
|
52
52
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
53
53
|
import { createHash } from "node:crypto";
|
|
54
|
-
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
54
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
55
55
|
import { dirname, isAbsolute, join, posix } from "node:path";
|
|
56
|
+
import { OP_WAVE_PHASES, OP_WAVE_RUN_RECORD_FILE, opWaveRunRecordFileName, opWaveSavedPlanPath, type OpWavePhase } from "./op-waves-split";
|
|
56
57
|
import { CHANT_EXIT } from "../exit-codes";
|
|
57
58
|
import type { ResolvedGatePolicySource } from "../policy-source";
|
|
58
59
|
import {
|
|
@@ -76,11 +77,11 @@ import {
|
|
|
76
77
|
writeWaveOutcomeFile,
|
|
77
78
|
type WaveMemberOutcome,
|
|
78
79
|
} from "../wave-blocked";
|
|
79
|
-
import { approveCommand, evaluateGate, gitGateLedgerPort, type GateCheck, type GateDigestMismatch, type GateLedgerPort } from "./gate";
|
|
80
|
+
import { approveCommand, evaluateGate, gitGateLedgerPort, type GateCheck, type GateCheckInput, type GateDigestMismatch, type GateLedgerPort } from "./gate";
|
|
80
81
|
import type { GateAnswerRef, PendingGateInput, PendingGateRecord } from "../lifecycle/gate-ledger";
|
|
81
82
|
import type { GatePointAsker } from "../point-decision";
|
|
82
83
|
import type { WaitingPoint } from "./steward-points";
|
|
83
|
-
import { isApprovalSpent, type AppliedGateRecord, type FinishedGateRecord } from "../lifecycle/gate-applied";
|
|
84
|
+
import { finishedApply, isApprovalSpent, type AppliedGateRecord, type FinishedGateRecord } from "../lifecycle/gate-applied";
|
|
84
85
|
import { samePlanDigest } from "../lifecycle/plan-digest";
|
|
85
86
|
import { movedSincePreview } from "./op-waves-preview";
|
|
86
87
|
import { isLifecycleLocalNote } from "../lifecycle/local-note";
|
|
@@ -106,7 +107,7 @@ import type { RunEventResource } from "../run-event";
|
|
|
106
107
|
import { opWaveReleaseNames, type OpWaveReleaseOutcome, type OpWaveReleasePort, type OpWaveReleaseRequest } from "./op-waves-release";
|
|
107
108
|
import { gitOpWaveReleasePort } from "./op-waves-release-port";
|
|
108
109
|
import type { LockGrain } from "../lifecycle/lock-rows";
|
|
109
|
-
import type { LockLiveness } from "../lifecycle/resource-lock";
|
|
110
|
+
import type { LockLiveness, LockRowsLease } from "../lifecycle/resource-lock";
|
|
110
111
|
import type { GatePolicyExec } from "../gate-policy";
|
|
111
112
|
import { gitOpWaveNewerCommit, opWaveStandDownReason, tipAppliesTargets, type OpWaveNewerCommitPort } from "./op-waves-stand-down";
|
|
112
113
|
import { keyedSerial, mapLimited, type SerialSection } from "../parallelism";
|
|
@@ -141,6 +142,7 @@ import {
|
|
|
141
142
|
assertOpWaveHooks,
|
|
142
143
|
assertOpWavesSpec,
|
|
143
144
|
opWaveApplyHooksRunBy,
|
|
145
|
+
opWaveAt,
|
|
144
146
|
opWaveGateName,
|
|
145
147
|
opWaveHooksAt,
|
|
146
148
|
opWaveHookRuns,
|
|
@@ -173,6 +175,7 @@ import {
|
|
|
173
175
|
holdFinishedPath,
|
|
174
176
|
readHoldFinished,
|
|
175
177
|
readHoldRequest,
|
|
178
|
+
inProcessHookOf,
|
|
176
179
|
runHookList,
|
|
177
180
|
runHookListAsync,
|
|
178
181
|
runStageHooks,
|
|
@@ -186,6 +189,8 @@ import {
|
|
|
186
189
|
type OpWaveHoldRequest,
|
|
187
190
|
type OpWaveHookOutcome,
|
|
188
191
|
type OpWaveHookRun,
|
|
192
|
+
type OpWaveInProcessHook,
|
|
193
|
+
type OpWaveInProcessHookCall,
|
|
189
194
|
type OtherGateHold,
|
|
190
195
|
type RunStageHooksAsyncOptions,
|
|
191
196
|
type RunStageHooksOptions,
|
|
@@ -193,7 +198,7 @@ import {
|
|
|
193
198
|
} from "./stage-hooks";
|
|
194
199
|
|
|
195
200
|
export { OP_WAVE_PENDING_RESERVED, opWaveArgv, runStageHooks, runStageHooksAsync, spawnExecAsync };
|
|
196
|
-
export type { OpWaveAsyncExec, OpWaveExec, OpWaveHoldFinished, OpWaveHoldRequest, OpWaveHookOutcome, OpWaveHookRun, RunStageHooksAsyncOptions, RunStageHooksOptions, StageHooksRun };
|
|
201
|
+
export type { OpWaveAsyncExec, OpWaveExec, OpWaveHoldFinished, OpWaveHoldRequest, OpWaveHookOutcome, OpWaveHookRun, OpWaveInProcessHook, OpWaveInProcessHookCall, RunStageHooksAsyncOptions, RunStageHooksOptions, StageHooksRun };
|
|
197
202
|
|
|
198
203
|
/** What a run's plan command writes to `{plan}`. */
|
|
199
204
|
export interface OpWavePlanFile {
|
|
@@ -421,6 +426,12 @@ export interface OpWaveDecision {
|
|
|
421
426
|
* workspace at base, or no policy ran.
|
|
422
427
|
*/
|
|
423
428
|
ciPolicySource?: ResolvedGatePolicySource;
|
|
429
|
+
/**
|
|
430
|
+
* A split wave with a `step` (#4992): the rows its gate job leased for
|
|
431
|
+
* the applying jobs, held with no heartbeat until its record job lets go
|
|
432
|
+
* of them, or the forge says the run is gone.
|
|
433
|
+
*/
|
|
434
|
+
lease?: OpWaveSplitLease;
|
|
424
435
|
}
|
|
425
436
|
|
|
426
437
|
/**
|
|
@@ -489,6 +500,27 @@ export interface OpWaveHeldOn {
|
|
|
489
500
|
* approved, oldest first (#3847): its approved re-runs asked for this one.
|
|
490
501
|
*/
|
|
491
502
|
after?: OpWaveHeldOnApproval[];
|
|
503
|
+
/**
|
|
504
|
+
* Set by a split wave's plan or apply job (#4992), which holds no write
|
|
505
|
+
* token: the pending fact it read the gate for and did not record, and
|
|
506
|
+
* the file the request carries. The gate job (for a plan job's hold) or
|
|
507
|
+
* the record job (for an apply job's) records them and drops this.
|
|
508
|
+
*
|
|
509
|
+
* `spend` (#4994): the gate is approved and the hold asks chant to
|
|
510
|
+
* `record` it, so its approval is spent before the hook runs again. The
|
|
511
|
+
* plan job cannot spend it; the gate job checks the approval again under
|
|
512
|
+
* its own rules and spends it for a re-run of the hook at `head`, and the
|
|
513
|
+
* wave waits until it runs again from its plan job.
|
|
514
|
+
*/
|
|
515
|
+
deferred?: { input: GateCheckInput; mode: GateApprovalSource; base: string; attachment?: string; spend?: { approvedBy: string; approvedAt: string; head?: string } };
|
|
516
|
+
/**
|
|
517
|
+
* Set by a split wave's plan job (#4994): the applied line the gate job of
|
|
518
|
+
* an earlier run of the same head wrote for this hold, with no finished
|
|
519
|
+
* line yet. The hook runs again under it, and the next gate job writes the
|
|
520
|
+
* finished line. On a decision it says the approval was spent for that
|
|
521
|
+
* re-run and the wave waits for it.
|
|
522
|
+
*/
|
|
523
|
+
spentForRerun?: AppliedGateRecord;
|
|
492
524
|
}
|
|
493
525
|
|
|
494
526
|
/** An approved hold on another gate (#3847), as `CHANT_HOLD_APPROVED` and `CHANT_HOLDS_APPROVED` carry it. */
|
|
@@ -797,6 +829,17 @@ export interface RunOpWaveOptions {
|
|
|
797
829
|
* {@link opWaveChecksDir} and no outcome file or run record.
|
|
798
830
|
*/
|
|
799
831
|
policyOnly?: boolean;
|
|
832
|
+
/**
|
|
833
|
+
* One of the four jobs of a split wave before merge (#4987,
|
|
834
|
+
* `./op-waves-split.ts`). `plan`: plan the wave and save the plan
|
|
835
|
+
* ({@link OpWaveSavedPlan}), writing nothing to `chant/lifecycle`. `gate`:
|
|
836
|
+
* read the rules at base, check the saved plan against its plan files,
|
|
837
|
+
* and decide the gate on it as a deciding job does. `apply`: apply the
|
|
838
|
+
* saved plan files whose digest the decision holds, as a share job does,
|
|
839
|
+
* and leave the run record in a file. `record`: write those run records
|
|
840
|
+
* to the run ledger. A `step` spec cannot run in phases.
|
|
841
|
+
*/
|
|
842
|
+
phase?: OpWavePhase;
|
|
800
843
|
/** Apply this 1-based share of a decided wave, or with `group`, of that job group. */
|
|
801
844
|
share?: number;
|
|
802
845
|
/**
|
|
@@ -927,6 +970,16 @@ export interface RunOpWaveOptions {
|
|
|
927
970
|
* while a hook does. Default: `exec`, which waits with `spawnSync`.
|
|
928
971
|
*/
|
|
929
972
|
hookExec?: OpWaveAsyncExec;
|
|
973
|
+
/**
|
|
974
|
+
* Hooks served in the job's own process (#4994), by the spec hook's
|
|
975
|
+
* `name`: a hook the spec names (read at base) whose name this holds is
|
|
976
|
+
* called with its command's argv and variables in place of running the
|
|
977
|
+
* command, at every moment and in every job that runs it, a split wave's
|
|
978
|
+
* plan, gate and applying jobs included. Its exit code counts as the
|
|
979
|
+
* command's, and its approved re-run after a hold on another gate calls it
|
|
980
|
+
* again. See {@link OpWaveInProcessHook}.
|
|
981
|
+
*/
|
|
982
|
+
inProcessHooks?: Readonly<Record<string, OpWaveInProcessHook>>;
|
|
930
983
|
}
|
|
931
984
|
|
|
932
985
|
/** Where an op wave job's run record goes (#4551). */
|
|
@@ -983,6 +1036,8 @@ export interface RunOpWaveResult {
|
|
|
983
1036
|
plans?: Record<string, OpRunWavePlan>;
|
|
984
1037
|
/** With `policyOnly` (#4926): the `ci.policy` rows the wave's policy hold gave, in run order. Empty when no policy ran. */
|
|
985
1038
|
policyRows?: OpRunPolicyRow[];
|
|
1039
|
+
/** A split wave's applying job (#4992): what its record job writes with the token, beside the run record. */
|
|
1040
|
+
handoff?: OpWaveSplitHandoff;
|
|
986
1041
|
}
|
|
987
1042
|
|
|
988
1043
|
/** What one wave job works with. */
|
|
@@ -1346,7 +1401,8 @@ async function leaseWaveRows(
|
|
|
1346
1401
|
options: RunOpWaveOptions,
|
|
1347
1402
|
ctx: WaveContext,
|
|
1348
1403
|
runs: readonly OpWaveRun[],
|
|
1349
|
-
|
|
1404
|
+
opts: { noBeat?: boolean } = {},
|
|
1405
|
+
): Promise<{ release(): Promise<boolean>; held: OpWaveSplitLease[] } | OpWaveApplyOutcome | { failed: string } | undefined> {
|
|
1350
1406
|
const locks = runs.flatMap((r) => {
|
|
1351
1407
|
const plan = ctx.steps?.get(r.target);
|
|
1352
1408
|
return plan?.lock && !plan.empty ? [plan.lock] : [];
|
|
@@ -1361,7 +1417,7 @@ async function leaseWaveRows(
|
|
|
1361
1417
|
const gate = options.lockGate ?? PR_APPLY_GATE;
|
|
1362
1418
|
const label = `wave ${ctx.k} (${ctx.wave.name})`;
|
|
1363
1419
|
const alive = options.lockLiveness ? { alive: options.lockLiveness } : {};
|
|
1364
|
-
const held:
|
|
1420
|
+
const held: LockRowsLease[] = [];
|
|
1365
1421
|
const releaseAll = async () => {
|
|
1366
1422
|
let ok = true;
|
|
1367
1423
|
for (const l of held.reverse()) ok = (await l.release()) && ok;
|
|
@@ -1381,7 +1437,9 @@ async function leaseWaveRows(
|
|
|
1381
1437
|
},
|
|
1382
1438
|
}
|
|
1383
1439
|
: {};
|
|
1384
|
-
|
|
1440
|
+
// #4992: a split wave's gate job ends before its rows are applied, so its hold carries no heartbeat of its process:
|
|
1441
|
+
// the forge's run, the same in its applying and record jobs, keeps it, as a hold from before #4660 is kept.
|
|
1442
|
+
leased = await leaseLockRows(gate, wanted, { cwd: ctx.cwd, ...alive, ...waitFor, ...(opts.noBeat ? { beat: () => () => {} } : {}) });
|
|
1385
1443
|
} catch (err) {
|
|
1386
1444
|
return { failed: `${label}: could not hold the rows its runs change in ${gate}, so nothing was applied: ${err instanceof Error ? err.message : String(err)}` };
|
|
1387
1445
|
}
|
|
@@ -1403,7 +1461,41 @@ async function leaseWaveRows(
|
|
|
1403
1461
|
return stop;
|
|
1404
1462
|
}
|
|
1405
1463
|
}
|
|
1406
|
-
return { release: releaseAll };
|
|
1464
|
+
return { release: releaseAll, held: held.map((l) => ({ gate: l.gate, run: l.holder.run, keys: [...l.keys] })) };
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
/**
|
|
1468
|
+
* The lease of a split wave with a `step` (#4992), in its gate job: the
|
|
1469
|
+
* rows its runs wait for and change, taken as {@link leaseWaveRows} takes
|
|
1470
|
+
* them before an apply, with no heartbeat, since this job ends before the
|
|
1471
|
+
* applying jobs start. The decision names the lease, and the record job
|
|
1472
|
+
* lets go of it. A held row ends the wave here with the `held` outcome,
|
|
1473
|
+
* exit 5, before the approval is spent.
|
|
1474
|
+
*/
|
|
1475
|
+
async function leaseSplitRows(options: RunOpWaveOptions, ctx: WaveContext, decision: OpWaveDecision, recordPath: string): Promise<{ end: RunOpWaveResult } | { release(): Promise<boolean> } | undefined> {
|
|
1476
|
+
const lease = await leaseWaveRows(options, ctx, ctx.wave.runs, { noBeat: true });
|
|
1477
|
+
if (!lease) return undefined;
|
|
1478
|
+
if ("failed" in lease) return { end: failed(ctx, lease.failed) };
|
|
1479
|
+
if ("outcome" in lease) {
|
|
1480
|
+
ctx.log(`wave ${ctx.k} (${ctx.wave.name}): applied nothing: ${lease.reason}`);
|
|
1481
|
+
ctx.result.outcome = lease;
|
|
1482
|
+
ctx.result.exitCode = CHANT_EXIT.held;
|
|
1483
|
+
writeRecord(recordPath, decision, ctx.result);
|
|
1484
|
+
return { end: ctx.result };
|
|
1485
|
+
}
|
|
1486
|
+
const first = lease.held[0];
|
|
1487
|
+
if (first) {
|
|
1488
|
+
decision.lease = { gate: first.gate, run: first.run, keys: [...new Set(lease.held.flatMap((h) => h.keys))].sort() };
|
|
1489
|
+
ctx.log(`wave ${ctx.k} (${ctx.wave.name}): holds ${decision.lease.keys.length} row${decision.lease.keys.length === 1 ? "" : "s"} of ${first.gate} for its applying jobs, until its record job lets go of them`);
|
|
1490
|
+
}
|
|
1491
|
+
return lease;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
/** Let go of the rows a split wave's gate job leased (#4992), as the holder the decision names. */
|
|
1495
|
+
async function releaseSplitLease(cwd: string, lease: OpWaveSplitLease, k: number, log: (line: string) => void): Promise<void> {
|
|
1496
|
+
const { releaseLockRows } = await import("../lifecycle/resource-lock");
|
|
1497
|
+
if (await releaseLockRows(lease.gate, lease.keys, lease.run, { cwd })) log(`wave ${k}: let go of the ${lease.keys.length} row${lease.keys.length === 1 ? "" : "s"} of ${lease.gate} its gate job held`);
|
|
1498
|
+
else log(`wave ${k}: could not let go of the rows its gate job held on ${lease.gate}; the next apply takes them over once this run is gone`);
|
|
1407
1499
|
}
|
|
1408
1500
|
|
|
1409
1501
|
function inGitWorkTree(cwd: string): boolean {
|
|
@@ -1664,9 +1756,11 @@ async function runHooks(
|
|
|
1664
1756
|
): Promise<HooksOutcome> {
|
|
1665
1757
|
if (!hooks || hooks.length === 0) return { holds: [] };
|
|
1666
1758
|
const runsPath = writeRunsFile(ctx, runs, members, digest);
|
|
1667
|
-
const
|
|
1759
|
+
const own = ctx.options.inProcessHooks;
|
|
1760
|
+
// #4994: a hook the job's process serves runs in it; the other hooks run as before, through hookExec or exec.
|
|
1761
|
+
const hookExec = ctx.options.hookExec ?? (own ? (argv: string[], cwd: string, env?: Record<string, string>) => Promise.resolve(ctx.exec(argv, cwd, env)) : undefined);
|
|
1668
1762
|
const out = hookExec
|
|
1669
|
-
? await runHookListAsync({ ...placeOf(ctx), exec: hookExec }, moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv)
|
|
1763
|
+
? await runHookListAsync({ ...placeOf(ctx), exec: hookExec, ...(own ? { inProcess: own } : {}) }, moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv)
|
|
1670
1764
|
: runHookList(placeOf(ctx), moment, hooks, hookRuns(ctx, runs, members), runsPath, extraEnv);
|
|
1671
1765
|
ctx.result.hooks.push(...out.outcomes);
|
|
1672
1766
|
const back: HooksOutcome = { holds: out.holds, ...(out.failed ? { failed: out.failed } : {}) };
|
|
@@ -1696,7 +1790,22 @@ async function runHooks(
|
|
|
1696
1790
|
// #4623: a hold that asks chant to record its gate is spent here, before the re-run acts under the approval.
|
|
1697
1791
|
let spent: AppliedGateRecord | undefined;
|
|
1698
1792
|
let finishedAt: string | undefined;
|
|
1699
|
-
|
|
1793
|
+
// #4994: the gate job of an earlier run of this head spent the approval for this re-run; the gate job after this
|
|
1794
|
+
// plan job writes the finished line, from what the re-run leaves at CHANT_HOLD_FINISHED.
|
|
1795
|
+
const spentEarlier = hold.request.record && ctx.options.phase === "plan" ? held.spentForRerun : undefined;
|
|
1796
|
+
if (spentEarlier) {
|
|
1797
|
+
spent = spentEarlier;
|
|
1798
|
+
const finishedRel = holdFinishedPath(hold.env.CHANT_HOLD!);
|
|
1799
|
+
finishedAt = join(ctx.cwd, finishedRel);
|
|
1800
|
+
rmSync(finishedAt, { force: true });
|
|
1801
|
+
approvedEnv.CHANT_HOLD_FINISHED = finishedRel;
|
|
1802
|
+
} else if (hold.request.record && noTokenPhase(ctx.options)) {
|
|
1803
|
+
// #4992: the approval is spent before the re-run acts under it, and a job with no write token cannot spend it.
|
|
1804
|
+
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: 1, outcome: "failed" });
|
|
1805
|
+
back.failed = `${hold.label}: ${held.approvedBy} approved ${held.op} / ${held.gate}, and the hold asks chant to record the gate, which spends the approval before the hook runs again; this ${ctx.options.phase} job of a split wave holds no write token, so it cannot spend it and the hook did not run again`;
|
|
1806
|
+
ctx.log(back.failed);
|
|
1807
|
+
return back;
|
|
1808
|
+
} else if (hold.request.record) {
|
|
1700
1809
|
const got = await spendHeldGate(ctx, hold, now);
|
|
1701
1810
|
if ("failed" in got) {
|
|
1702
1811
|
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: 1, outcome: "failed" });
|
|
@@ -1712,12 +1821,17 @@ async function runHooks(
|
|
|
1712
1821
|
}
|
|
1713
1822
|
let code: number;
|
|
1714
1823
|
try {
|
|
1715
|
-
|
|
1824
|
+
// #4994: an in-process hook runs again in this process, as it ran the first time.
|
|
1825
|
+
const again = inProcessHookOf(own, hold.name);
|
|
1826
|
+
code = again
|
|
1827
|
+
? await again({ name: hold.name, moment: hold.moment, ...(hold.target ? { target: hold.target } : {}), argv: hold.argv, cwd: ctx.cwd, env: approvedEnv })
|
|
1828
|
+
: hookExec ? await hookExec(hold.argv, ctx.cwd, approvedEnv) : ctx.exec(hold.argv, ctx.cwd, approvedEnv);
|
|
1716
1829
|
} catch (err) {
|
|
1717
1830
|
ctx.log(`${hold.label} did not run: ${err instanceof Error ? err.message : String(err)}`);
|
|
1718
1831
|
code = 127;
|
|
1719
1832
|
}
|
|
1720
|
-
if (spent && finishedAt)
|
|
1833
|
+
if (spent && finishedAt && spentEarlier) handHoldFinished(ctx, hold, spent, readHoldFinished(finishedAt, ctx.log, hold.label), code);
|
|
1834
|
+
else if (spent && finishedAt) await finishHeldGate(ctx, hold, spent, readHoldFinished(finishedAt, ctx.log, hold.label), code);
|
|
1721
1835
|
if (code === 0) {
|
|
1722
1836
|
ctx.result.hooks.push({ moment, name: hold.name, ...target, exit: code, outcome: "passed" });
|
|
1723
1837
|
ctx.log(`${hold.label}: ${held.approvedBy} approved ${held.op} / ${held.gate} at ${held.digest}, and the hook ran again and passed`);
|
|
@@ -1870,18 +1984,14 @@ async function settleOtherGate(ctx: WaveContext, hold: OtherGateHold, runs: read
|
|
|
1870
1984
|
const { port } = await gatePort(ctx.options, ctx.cwd, hold.request.gate, mode, rules.base);
|
|
1871
1985
|
const neverOverMcp = rules.neverOverMcp;
|
|
1872
1986
|
const now = ctx.options.now ?? new Date().toISOString();
|
|
1987
|
+
// #4992: a split wave's plan and apply jobs hold no write token. They read the other gate and, while it waits, leave its
|
|
1988
|
+
// pending fact and the file the request carries to the gate or record job, which records them (recordDeferredHold).
|
|
1989
|
+
const deferring = noTokenPhase(ctx.options);
|
|
1873
1990
|
// #4623: the file the request carries, stored once before the pending fact, which names its SHA-256.
|
|
1874
1991
|
const attachment = hold.request.attachment;
|
|
1875
|
-
if (attachment !== undefined) {
|
|
1876
|
-
try {
|
|
1877
|
-
if (!port.storeRequestAttachment) throw new Error("this gate ledger keeps no request files");
|
|
1878
|
-
await port.storeRequestAttachment(op, hold.request.digest, attachment);
|
|
1879
|
-
} catch (err) {
|
|
1880
|
-
log(`${op} / ${hold.request.gate}: the file ${hold.label} attached was not stored beside the pending fact: ${err instanceof Error ? err.message : String(err)}`);
|
|
1881
|
-
}
|
|
1882
|
-
}
|
|
1992
|
+
if (attachment !== undefined && !deferring) await storeHoldAttachment(port, op, hold.request.digest, attachment, `${op} / ${hold.request.gate}: the file ${hold.label} attached`, log);
|
|
1883
1993
|
const proof = attachment !== undefined ? { proof: `sha256:${createHash("sha256").update(attachment).digest("hex")}` } : {};
|
|
1884
|
-
const
|
|
1994
|
+
const input: GateCheckInput = {
|
|
1885
1995
|
op,
|
|
1886
1996
|
gate: hold.request.gate,
|
|
1887
1997
|
planDigest: hold.request.digest,
|
|
@@ -1890,14 +2000,40 @@ async function settleOtherGate(ctx: WaveContext, hold: OtherGateHold, runs: read
|
|
|
1890
2000
|
// #3839: the wave a hold belongs to, as a field, whatever the hook's own description says.
|
|
1891
2001
|
fields: { ...hold.request.pending, ...proof, approvalMode: rules.approval, wave: { number: k, name: wave.name, op: spec.name }, heldBy: { hook: hold.name, moment: hold.moment, ...(hold.target ? { target: hold.target } : {}) } },
|
|
1892
2002
|
...(neverOverMcp ? { neverOverMcp: true } : {}),
|
|
1893
|
-
}
|
|
2003
|
+
};
|
|
2004
|
+
const check = await evaluateGate(deferring ? readOnlyGatePort(port) : port, input);
|
|
1894
2005
|
const base = { op, gate: hold.request.gate, digest: hold.request.digest, hook: hold.label };
|
|
1895
2006
|
logRefused(log, `${op} / ${hold.request.gate}`, check);
|
|
1896
|
-
if (check.satisfied) return { ...base, status: "approved", approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp };
|
|
1897
2007
|
// #4574: a per-run hook's hold names its run; else the wave's target, as the wave's own gate does.
|
|
1898
2008
|
const target = hold.target ?? waveApproveTarget(wave.name, runs.map((r) => r.target));
|
|
1899
2009
|
const approve = approveCommand(op, hold.request.gate, undefined, hold.request.digest, mode === "sealed", target);
|
|
2010
|
+
// #4994: a split wave's plan job cannot spend the approval a `record` hold needs spent before its hook runs again.
|
|
2011
|
+
const spendLater = deferring && hold.request.record === true && ctx.options.phase === "plan";
|
|
2012
|
+
if (spendLater) {
|
|
2013
|
+
// The gate job of an earlier run of this commit spent the approval for this very re-run, and no finished line names it yet.
|
|
2014
|
+
const spent = await spentForRerun(port, op, hold.request.gate, hold.request.digest, waveHeadOf(ctx.options, ctx.cwd));
|
|
2015
|
+
if (spent) {
|
|
2016
|
+
log(`${hold.label}: the approval by ${spent.approvedBy} of ${op} / ${hold.request.gate} at ${hold.request.digest} was spent at ${spent.timestamp} for this re-run of the hook, so it runs again under it`);
|
|
2017
|
+
return { ...base, status: "approved", approvedBy: spent.approvedBy, approvedAt: spent.approvedAt, spentForRerun: spent };
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
if (check.satisfied && spendLater) {
|
|
2021
|
+
const head = waveHeadOf(ctx.options, ctx.cwd);
|
|
2022
|
+
log(`wave ${k} (${wave.name}): ${check.resolution.resolvedBy} approved ${op} / ${hold.request.gate} at ${hold.request.digest}, and ${hold.label} asks chant to record the gate, which spends the approval before the hook runs again; this plan job holds no write token, so the gate job spends it and the wave waits until it runs again from its plan job, where the hook runs again under it`);
|
|
2023
|
+
return {
|
|
2024
|
+
...base,
|
|
2025
|
+
status: "waiting",
|
|
2026
|
+
approve,
|
|
2027
|
+
recorded: false,
|
|
2028
|
+
deferred: { input, mode, base: rules.base, ...(attachment !== undefined ? { attachment } : {}), spend: { approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp, ...(head ? { head } : {}) } },
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
if (check.satisfied) return { ...base, status: "approved", approvedBy: check.resolution.resolvedBy, approvedAt: check.resolution.timestamp };
|
|
1900
2032
|
log(`wave ${k} (${wave.name}) waits on ${op} / ${hold.request.gate}, which ${hold.label} holds it on: nobody approved ${hold.request.digest}. Approve: ${approve}`);
|
|
2033
|
+
if (deferring) {
|
|
2034
|
+
log(`wave ${k} (${wave.name}): this job holds no write token, so the ${ctx.options.phase === "plan" ? "gate" : "record"} job records the pending fact of ${op} / ${hold.request.gate}`);
|
|
2035
|
+
return { ...base, status: "waiting", approve, recorded: false, deferred: { input, mode, base: rules.base, ...(attachment !== undefined ? { attachment } : {}) }, ...(check.mismatch ? { mismatch: check.mismatch } : {}) };
|
|
2036
|
+
}
|
|
1901
2037
|
return {
|
|
1902
2038
|
...base,
|
|
1903
2039
|
status: "waiting",
|
|
@@ -1908,6 +2044,225 @@ async function settleOtherGate(ctx: WaveContext, hold: OtherGateHold, runs: read
|
|
|
1908
2044
|
};
|
|
1909
2045
|
}
|
|
1910
2046
|
|
|
2047
|
+
/** The commit a wave job runs (#4994): `options.head`, a split wave's decided head, else the checkout's HEAD; undefined outside a git checkout. */
|
|
2048
|
+
function waveHeadOf(options: Pick<RunOpWaveOptions, "head">, cwd: string): string | undefined {
|
|
2049
|
+
if (options.head) return options.head;
|
|
2050
|
+
try {
|
|
2051
|
+
return execFileSync("git", ["rev-parse", "HEAD"], { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] }).trim() || undefined;
|
|
2052
|
+
} catch {
|
|
2053
|
+
return undefined;
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
/**
|
|
2058
|
+
* The applied line a split wave's gate job wrote for a re-run of a held
|
|
2059
|
+
* hook at `head` (#4994), when no finished line names it yet; else
|
|
2060
|
+
* undefined. A port that cannot read finished lines gives none, so no
|
|
2061
|
+
* approval is used twice.
|
|
2062
|
+
*/
|
|
2063
|
+
async function spentForRerun(port: GateLedgerPort, op: string, gate: string, digest: string, head: string | undefined): Promise<AppliedGateRecord | undefined> {
|
|
2064
|
+
if (!head || !port.readApplied || !port.readFinished) return undefined;
|
|
2065
|
+
const line = (await port.readApplied(op)).filter((a) => a.gate === gate && samePlanDigest(a.planDigest, digest)).at(-1);
|
|
2066
|
+
if (!line || line.rerun?.head !== head) return undefined;
|
|
2067
|
+
return finishedApply(await port.readFinished(op), line) ? undefined : line;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
/** The finished lines a split wave's plan job leaves its gate job to write (#4994), by the job's options. */
|
|
2071
|
+
const handedFinished = new WeakMap<RunOpWaveOptions, OpWaveHandedFinished[]>();
|
|
2072
|
+
|
|
2073
|
+
/**
|
|
2074
|
+
* A held hook's re-run in a split wave's plan job under an approval an
|
|
2075
|
+
* earlier gate job spent (#4994): how it ended, which the gate job after
|
|
2076
|
+
* this plan job writes as the finished line.
|
|
2077
|
+
*/
|
|
2078
|
+
export interface OpWaveHandedFinished {
|
|
2079
|
+
hook: string;
|
|
2080
|
+
/** The applied line the re-run ran under. */
|
|
2081
|
+
applied: Pick<AppliedGateRecord, "op" | "gate" | "planDigest" | "timestamp" | "rerun">;
|
|
2082
|
+
result: "applied" | "failed";
|
|
2083
|
+
detail?: Record<string, string | number | boolean>;
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
/** Keep a re-run's end for the gate job (#4994), in the saved plan. */
|
|
2087
|
+
function handHoldFinished(ctx: WaveContext, hold: OtherGateHold, spent: AppliedGateRecord, wrote: OpWaveHoldFinished | undefined, code: number): void {
|
|
2088
|
+
const list = handedFinished.get(ctx.options) ?? [];
|
|
2089
|
+
list.push({
|
|
2090
|
+
hook: hold.label,
|
|
2091
|
+
applied: { op: spent.op, gate: spent.gate, planDigest: spent.planDigest, timestamp: spent.timestamp, ...(spent.rerun ? { rerun: spent.rerun } : {}) },
|
|
2092
|
+
result: wrote?.result ?? (code === 0 ? "applied" : "failed"),
|
|
2093
|
+
...(wrote?.detail ? { detail: wrote.detail } : {}),
|
|
2094
|
+
});
|
|
2095
|
+
handedFinished.set(ctx.options, list);
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
/**
|
|
2099
|
+
* Write the finished lines a split wave's plan job handed on (#4994), with
|
|
2100
|
+
* the token: each for an applied line the ledger holds, spent for a re-run
|
|
2101
|
+
* at this job's head and not finished yet. The plan job ran the change's
|
|
2102
|
+
* code, so a line it names that is not one of those is left out.
|
|
2103
|
+
*/
|
|
2104
|
+
async function writeHandedFinished(options: RunOpWaveOptions, cwd: string, handed: readonly OpWaveHandedFinished[] | undefined, log: (line: string) => void): Promise<void> {
|
|
2105
|
+
if (!handed?.length) return;
|
|
2106
|
+
const port = options.gates ?? gitGateLedgerPort({ cwd });
|
|
2107
|
+
const head = waveHeadOf(options, cwd);
|
|
2108
|
+
for (const h of handed) {
|
|
2109
|
+
const where = `${h.applied.op} / ${h.applied.gate}`;
|
|
2110
|
+
try {
|
|
2111
|
+
if (!port.appendFinished || !port.readApplied || !port.readFinished) throw new Error("this gate ledger keeps no finished lines");
|
|
2112
|
+
const line = (await port.readApplied(h.applied.op)).find((a) => a.gate === h.applied.gate && samePlanDigest(a.planDigest, h.applied.planDigest) && a.timestamp === h.applied.timestamp);
|
|
2113
|
+
if (!line || !head || line.rerun?.head !== head) {
|
|
2114
|
+
log(`${h.hook}: the plan job says it ran again under ${where} applied at ${h.applied.timestamp}, which the ledger holds no re-run of this commit for, so no finished line is written`);
|
|
2115
|
+
continue;
|
|
2116
|
+
}
|
|
2117
|
+
if (finishedApply(await port.readFinished(h.applied.op), line)) continue;
|
|
2118
|
+
const record: FinishedGateRecord = {
|
|
2119
|
+
version: 1,
|
|
2120
|
+
kind: "finished",
|
|
2121
|
+
op: line.op,
|
|
2122
|
+
gate: line.gate,
|
|
2123
|
+
planDigest: line.planDigest,
|
|
2124
|
+
applied: line.timestamp,
|
|
2125
|
+
result: h.result,
|
|
2126
|
+
timestamp: options.now ?? new Date().toISOString(),
|
|
2127
|
+
...(h.detail ? { detail: h.detail } : {}),
|
|
2128
|
+
};
|
|
2129
|
+
const pushed = await port.appendFinished(record);
|
|
2130
|
+
if (!pushed.pushed && pushed.pushWarning && !isLifecycleLocalNote(pushed.pushWarning)) log(`${h.hook}: the finished line of ${where} was written and not pushed: ${pushed.pushWarning}`);
|
|
2131
|
+
log(`${h.hook}: wrote the finished line of ${where} (${h.result}) for the plan job's re-run`);
|
|
2132
|
+
} catch (err) {
|
|
2133
|
+
log(`${h.hook}: the finished line of ${where} was not written: ${err instanceof Error ? err.message : String(err)}`);
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
/**
|
|
2139
|
+
* Spend the approval a split wave's plan job found for a `record` hold
|
|
2140
|
+
* (#4994), with the token, under this job's rules at base: the gate is read
|
|
2141
|
+
* again, and still approved, an applied line bound to this job's head is
|
|
2142
|
+
* written for the hook's re-run in the next run's plan job. Not approved
|
|
2143
|
+
* any more, the hold is recorded as waiting (`recordDeferredHold`). A digest
|
|
2144
|
+
* already applied under the gate is not spent again.
|
|
2145
|
+
*/
|
|
2146
|
+
async function spendDeferredHold(
|
|
2147
|
+
options: RunOpWaveOptions,
|
|
2148
|
+
ctx: WaveContext,
|
|
2149
|
+
heldOn: OpWaveHeldOn,
|
|
2150
|
+
log: (line: string) => void,
|
|
2151
|
+
): Promise<{ heldOn: OpWaveHeldOn } | { failed: string }> {
|
|
2152
|
+
const deferred = heldOn.deferred!;
|
|
2153
|
+
const rule = ctx.gateRule;
|
|
2154
|
+
const mode: GateApprovalSource = rule.approval === "sealed" ? "sealed" : "ledger";
|
|
2155
|
+
const { port } = await gatePort(options, ctx.cwd, deferred.input.gate, mode, rule.base);
|
|
2156
|
+
const where = `${deferred.input.op} / ${deferred.input.gate}`;
|
|
2157
|
+
const digest = deferred.input.planDigest!;
|
|
2158
|
+
// A re-run this commit's earlier gate job spent the approval for, which the plan job did not take up: the wave waits for it.
|
|
2159
|
+
const pending = await spentForRerun(port, deferred.input.op, deferred.input.gate, digest, waveHeadOf(options, ctx.cwd));
|
|
2160
|
+
if (pending) {
|
|
2161
|
+
log(`${heldOn.hook}: the approval of ${where} at ${digest} is already spent for a re-run of the hook; the wave waits until it runs again from its plan job`);
|
|
2162
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
2163
|
+
return { heldOn: { ...rest, approvedBy: pending.approvedBy, approvedAt: pending.approvedAt, spentForRerun: pending } };
|
|
2164
|
+
}
|
|
2165
|
+
const check = await evaluateGate(readOnlyGatePort(port), { ...deferred.input, now: options.now ?? new Date().toISOString(), ...(rule.neverOverMcp || deferred.input.neverOverMcp ? { neverOverMcp: true } : {}) });
|
|
2166
|
+
if (!check.satisfied) {
|
|
2167
|
+
log(`${heldOn.hook}: ${where} is not approved under this job's rules, so its approval is not spent`);
|
|
2168
|
+
const { spend: _s, ...waiting } = deferred;
|
|
2169
|
+
return { heldOn: await recordDeferredHold(options, ctx.cwd, { ...heldOn, deferred: waiting }, rule, log) };
|
|
2170
|
+
}
|
|
2171
|
+
const head = waveHeadOf(options, ctx.cwd);
|
|
2172
|
+
if (!head) return { failed: `${heldOn.hook}: this job cannot tell which commit the wave runs, so it does not spend the approval of ${where} for the hook's re-run` };
|
|
2173
|
+
if (deferred.spend?.head && deferred.spend.head !== head) return { failed: `${heldOn.hook}: the plan job ran at ${deferred.spend.head.slice(0, 12)} and this job at ${head.slice(0, 12)}, so the approval of ${where} is not spent` };
|
|
2174
|
+
try {
|
|
2175
|
+
if (!port.appendApplied || !port.readApplied) throw new Error("this gate ledger cannot record an applied approval");
|
|
2176
|
+
const prior = (await port.readApplied(deferred.input.op)).filter((a) => a.gate === deferred.input.gate && samePlanDigest(a.planDigest, digest)).at(-1);
|
|
2177
|
+
if (prior) return { failed: `${heldOn.hook}: ${where} at ${digest} applied already under the approval by ${prior.approvedBy} (${prior.timestamp}); a held request applies once, so it is not spent again` };
|
|
2178
|
+
const commit = typeof deferred.input.fields?.commit === "string" ? deferred.input.fields.commit : undefined;
|
|
2179
|
+
const runId = ciRunIdFromEnv();
|
|
2180
|
+
const record: AppliedGateRecord = {
|
|
2181
|
+
version: 1,
|
|
2182
|
+
kind: "applied",
|
|
2183
|
+
op: deferred.input.op,
|
|
2184
|
+
gate: deferred.input.gate,
|
|
2185
|
+
planDigest: digest,
|
|
2186
|
+
approvedAt: check.resolution.timestamp,
|
|
2187
|
+
approvedBy: check.resolution.resolvedBy,
|
|
2188
|
+
timestamp: options.now ?? new Date().toISOString(),
|
|
2189
|
+
wave: { number: ctx.k, name: ctx.wave.name },
|
|
2190
|
+
...(runId ? { runId } : {}),
|
|
2191
|
+
...(commit ? { commit } : {}),
|
|
2192
|
+
rerun: { head },
|
|
2193
|
+
};
|
|
2194
|
+
requireSpendPushed(await port.appendApplied(record));
|
|
2195
|
+
log(`${heldOn.hook}: ${where} applies under the approval by ${record.approvedBy} at ${record.approvedAt}, which is now spent for the hook's re-run at ${head.slice(0, 12)}; run the wave again, and its plan job runs the hook again under it`);
|
|
2196
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
2197
|
+
return { heldOn: { ...rest, approvedBy: record.approvedBy, approvedAt: record.approvedAt, spentForRerun: record } };
|
|
2198
|
+
} catch (err) {
|
|
2199
|
+
return { failed: `${heldOn.hook}: could not record that ${where} applies under the approval by ${check.resolution.resolvedBy}: ${err instanceof Error ? err.message : String(err)}` };
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
/** Whether a split wave's job holds no write token (#4992): its plan and applying jobs run the change's code. */
|
|
2204
|
+
function noTokenPhase(options: Pick<RunOpWaveOptions, "phase">): boolean {
|
|
2205
|
+
return options.phase === "plan" || options.phase === "apply";
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
/** Store the file a held request carries (#4623). A file not stored is logged; the pending fact still names its SHA-256. */
|
|
2209
|
+
async function storeHoldAttachment(port: GateLedgerPort, op: string, digest: string, attachment: string, what: string, log: (line: string) => void): Promise<void> {
|
|
2210
|
+
try {
|
|
2211
|
+
if (!port.storeRequestAttachment) throw new Error("this gate ledger keeps no request files");
|
|
2212
|
+
await port.storeRequestAttachment(op, digest, attachment);
|
|
2213
|
+
} catch (err) {
|
|
2214
|
+
log(`${what} was not stored beside the pending fact: ${err instanceof Error ? err.message : String(err)}`);
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
/**
|
|
2219
|
+
* The gate ledger as a job with no write token reads it (#4992): every read
|
|
2220
|
+
* the port has, and a pending fact that is never written. `evaluateGate`
|
|
2221
|
+
* then says whether the gate is approved, and the fact it would have
|
|
2222
|
+
* recorded goes to a job that can write it.
|
|
2223
|
+
*/
|
|
2224
|
+
function readOnlyGatePort(port: GateLedgerPort): GateLedgerPort {
|
|
2225
|
+
const { appendResolution: _r, appendApplied: _a, appendFinished: _f, storeRequestAttachment: _s, writeCheckpoint: _c, ...reads } = port;
|
|
2226
|
+
return {
|
|
2227
|
+
...reads,
|
|
2228
|
+
async appendPending(input) {
|
|
2229
|
+
return { record: { version: 1, kind: "pending", ...input } as PendingGateRecord, pushed: false, pushWarning: "not recorded: this job holds no write token" };
|
|
2230
|
+
},
|
|
2231
|
+
};
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
/**
|
|
2235
|
+
* Record a hold on another gate that a split wave's plan or apply job left
|
|
2236
|
+
* (#4992): the file its request carries and the gate's pending fact, with
|
|
2237
|
+
* the token, as that job would have without the split. Returns the hold as
|
|
2238
|
+
* the decision records it. A gate approved since the plan or apply job read
|
|
2239
|
+
* it is said so: the wave waits all the same, and running it again from its
|
|
2240
|
+
* plan job goes on under the approval.
|
|
2241
|
+
*/
|
|
2242
|
+
async function recordDeferredHold(
|
|
2243
|
+
options: RunOpWaveOptions,
|
|
2244
|
+
cwd: string,
|
|
2245
|
+
heldOn: OpWaveHeldOn,
|
|
2246
|
+
rule: { approval: GateApprovalSource; base: string; neverOverMcp: boolean },
|
|
2247
|
+
log: (line: string) => void,
|
|
2248
|
+
): Promise<OpWaveHeldOn> {
|
|
2249
|
+
const deferred = heldOn.deferred;
|
|
2250
|
+
if (!deferred) return heldOn;
|
|
2251
|
+
const { deferred: _d, ...rest } = heldOn;
|
|
2252
|
+
// The approval mode, the base and neverOverMcp are this job's own, read at base, whatever the job that left the hold said.
|
|
2253
|
+
const mode: GateApprovalSource = rule.approval === "sealed" ? "sealed" : "ledger";
|
|
2254
|
+
const { port } = await gatePort(options, cwd, deferred.input.gate, mode, rule.base);
|
|
2255
|
+
const where = `${deferred.input.op} / ${deferred.input.gate}`;
|
|
2256
|
+
if (deferred.attachment !== undefined) await storeHoldAttachment(port, deferred.input.op, deferred.input.planDigest!, deferred.attachment, `${where}: the file ${heldOn.hook} attached`, log);
|
|
2257
|
+
const check = await evaluateGate(port, { ...deferred.input, now: options.now ?? new Date().toISOString(), ...(rule.neverOverMcp || deferred.input.neverOverMcp ? { neverOverMcp: true } : {}) });
|
|
2258
|
+
if (check.satisfied) {
|
|
2259
|
+
log(`${where} was approved by ${check.resolution.resolvedBy} after the job that ${heldOn.hook} ran in read it; the wave waits all the same, and goes on when it runs again from its plan job`);
|
|
2260
|
+
return rest;
|
|
2261
|
+
}
|
|
2262
|
+
log(`${where}: ${check.recorded ? "recorded the pending fact" : "the pending fact stands"} that ${heldOn.hook} holds wave ${options.wave} on`);
|
|
2263
|
+
return { ...rest, waitingSince: check.pending.timestamp, recorded: check.recorded };
|
|
2264
|
+
}
|
|
2265
|
+
|
|
1911
2266
|
/**
|
|
1912
2267
|
* Read the hooks file chant hands an apply command (#3804): the one at
|
|
1913
2268
|
* `path`, by default `CHANT_HOOKS`. Null when there is none, as for an apply
|
|
@@ -2055,6 +2410,13 @@ function noteHooksAtBase(ctx: WaveContext, rules: OpWaveBaseRules): void {
|
|
|
2055
2410
|
*/
|
|
2056
2411
|
export async function runOpWave(options: RunOpWaveOptions): Promise<RunOpWaveResult> {
|
|
2057
2412
|
assertOpWavesSpec(options.spec);
|
|
2413
|
+
if (options.phase !== undefined) {
|
|
2414
|
+
// #4987: a split wave's plan and gate jobs are its deciding job, in two; the record job writes the applying jobs' records.
|
|
2415
|
+
const wrong = opWavePhaseProblem(options);
|
|
2416
|
+
if (wrong) throw new Error(`Op waves "${options.spec.name}": ${wrong}`);
|
|
2417
|
+
if (options.phase === "record") return runOpWaveRecordPhase(options);
|
|
2418
|
+
if (options.phase === "plan" || options.phase === "gate") options = { ...options, decide: true };
|
|
2419
|
+
}
|
|
2058
2420
|
const afterPlanWrong = opWaveAfterPlanProblems(options.afterPlan);
|
|
2059
2421
|
if (afterPlanWrong.length > 0) throw new Error(`Op waves "${options.spec.name}": ${afterPlanWrong.join("; ")}`);
|
|
2060
2422
|
const runHooksWrong = opWaveRunHooksProblems(options.runHooks);
|
|
@@ -2209,7 +2571,7 @@ export function opWaveRunRecord(
|
|
|
2209
2571
|
const wave: OpRunWave = {
|
|
2210
2572
|
spec: spec.name,
|
|
2211
2573
|
number: options.wave,
|
|
2212
|
-
name: spec
|
|
2574
|
+
name: opWaveAt(spec, options.wave)?.name ?? `wave-${options.wave}`,
|
|
2213
2575
|
job: options.decide ? "decide" : options.group !== undefined ? "group" : options.share !== undefined ? "share" : "wave",
|
|
2214
2576
|
...(options.share !== undefined ? { share: options.share } : {}),
|
|
2215
2577
|
...(options.group !== undefined ? { group: options.group } : {}),
|
|
@@ -2251,7 +2613,8 @@ export function opWaveRunRecord(
|
|
|
2251
2613
|
*/
|
|
2252
2614
|
async function recordOpWaveRun(options: RunOpWaveOptions, started: string, outcome: { result: RunOpWaveResult } | { error: unknown }): Promise<OpRunRecordInput & { lost?: string }> {
|
|
2253
2615
|
const cwd = options.cwd ?? process.cwd();
|
|
2254
|
-
|
|
2616
|
+
// #4987: a split wave's plan job records nothing (its gate job records the decision), and an applying job leaves its record for the record job.
|
|
2617
|
+
const port = options.phase === "plan" ? undefined : options.phase === "apply" ? fileOpWaveRunPort(options, "result" in outcome ? outcome.result.handoff : undefined) : options.runs ?? (options.gates ? undefined : gitOpWaveRunPort({ cwd }));
|
|
2255
2618
|
const log = options.log ?? ((line: string) => console.error(line));
|
|
2256
2619
|
let gitSha = options.head;
|
|
2257
2620
|
if (gitSha === undefined) {
|
|
@@ -2302,6 +2665,8 @@ async function runOpWaveJob(options: RunOpWaveOptions): Promise<RunOpWaveResult>
|
|
|
2302
2665
|
const cwd = options.cwd ?? process.cwd();
|
|
2303
2666
|
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(spec);
|
|
2304
2667
|
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
2668
|
+
// #4987: the gate job of a split wave reads what the plan job saved, not the outcome files again.
|
|
2669
|
+
if (options.phase === "gate") return runOpWaveGatePhase(options, dir);
|
|
2305
2670
|
// #4322: what the earlier jobs' outcome files say: an earlier wave that stopped stops this one, and a failure blocks its dependents.
|
|
2306
2671
|
const before = opWaveBefore(options, dir);
|
|
2307
2672
|
const blocked = before.blocked;
|
|
@@ -2309,6 +2674,7 @@ async function runOpWaveJob(options: RunOpWaveOptions): Promise<RunOpWaveResult>
|
|
|
2309
2674
|
log(before.stop.line);
|
|
2310
2675
|
const stopped: RunOpWaveResult = { exitCode: before.stop.exitCode, applied: [], failed: [], moved: [], hooks: [], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) };
|
|
2311
2676
|
writeOpWaveOutcome(options, dir, stopped, before.stop.members);
|
|
2677
|
+
if (options.phase === "plan") saveOpWavePlan(options, { end: stopped });
|
|
2312
2678
|
return stopped;
|
|
2313
2679
|
}
|
|
2314
2680
|
if (blocked.size > 0) log(`wave ${options.wave}: ${blockedWords(blocked)}, so not planned or applied`);
|
|
@@ -2317,8 +2683,10 @@ async function runOpWaveJob(options: RunOpWaveOptions): Promise<RunOpWaveResult>
|
|
|
2317
2683
|
// Every run of this job is blocked: nothing to plan, decide or apply.
|
|
2318
2684
|
const none: RunOpWaveResult = { exitCode: options.decide ? 0 : 1, applied: [], failed: [], moved: [], hooks: [], blocked: Object.fromEntries(blocked) };
|
|
2319
2685
|
writeOpWaveOutcome(options, dir, none);
|
|
2686
|
+
if (options.phase === "plan") saveOpWavePlan(options, { end: none });
|
|
2320
2687
|
return none;
|
|
2321
2688
|
}
|
|
2689
|
+
if (options.phase === "plan") return runOpWavePlanPhase(options, dir, blocked);
|
|
2322
2690
|
let result: RunOpWaveResult;
|
|
2323
2691
|
const applied: string[] = [];
|
|
2324
2692
|
try {
|
|
@@ -2327,6 +2695,12 @@ async function runOpWaveJob(options: RunOpWaveOptions): Promise<RunOpWaveResult>
|
|
|
2327
2695
|
writeOpWaveOutcome(options, dir, { exitCode: 1, applied, failed: [], moved: [], hooks: [], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) });
|
|
2328
2696
|
throw err;
|
|
2329
2697
|
}
|
|
2698
|
+
if (options.phase === "apply" && result.replan) {
|
|
2699
|
+
// #4987: the plan job plans; an applying job that would plan again stops, and the wave runs again from its plan job.
|
|
2700
|
+
log(`wave ${options.wave}: the apply applied nothing and asks to plan again${result.replan.reason ? ` (${result.replan.reason})` : ""}; before merge the plan job plans, so run the wave again`);
|
|
2701
|
+
delete result.replan;
|
|
2702
|
+
result.exitCode = CHANT_EXIT.failed;
|
|
2703
|
+
}
|
|
2330
2704
|
const all = { ...Object.fromEntries(blocked), ...(result.blocked ?? {}) };
|
|
2331
2705
|
if (Object.keys(all).length > 0) {
|
|
2332
2706
|
result.blocked = all;
|
|
@@ -2375,6 +2749,421 @@ async function runOpWavePolicyOnly(options: RunOpWaveOptions): Promise<RunOpWave
|
|
|
2375
2749
|
return end(CHANT_EXIT.done, rows, `wave ${options.wave} (${wave.name}): ci.policy ran over its ${rows.length} plans and denied none`);
|
|
2376
2750
|
}
|
|
2377
2751
|
|
|
2752
|
+
/**
|
|
2753
|
+
* What a split wave's plan job saves for its gate and applying jobs
|
|
2754
|
+
* (#4987), at {@link opWaveSavedPlanPath} beside the run plan files: the
|
|
2755
|
+
* plans and their set digest, what the holds and hooks said of them, and
|
|
2756
|
+
* the runs a failure blocked. `end` instead: the plan job ended the wave,
|
|
2757
|
+
* with that result, and the gate job records it.
|
|
2758
|
+
*/
|
|
2759
|
+
export interface OpWaveSavedPlan {
|
|
2760
|
+
version: 1;
|
|
2761
|
+
op: string;
|
|
2762
|
+
wave: number;
|
|
2763
|
+
name: string;
|
|
2764
|
+
end?: RunOpWaveResult;
|
|
2765
|
+
digest?: string;
|
|
2766
|
+
members?: OpWaveMember[];
|
|
2767
|
+
extras?: WaveMember[];
|
|
2768
|
+
pending?: Record<string, unknown>;
|
|
2769
|
+
overriders?: string[];
|
|
2770
|
+
overridersSource?: string;
|
|
2771
|
+
changeSet?: ChangeSetDocument;
|
|
2772
|
+
/** The holds of the rules at base, the wave's holds and the in-process after-plan hooks. */
|
|
2773
|
+
holds?: string[];
|
|
2774
|
+
/** The holds of the before-plan hooks. */
|
|
2775
|
+
before?: string[];
|
|
2776
|
+
blocked?: Record<string, string>;
|
|
2777
|
+
report?: string;
|
|
2778
|
+
answerApprovals?: OpWaveAnswerApproval[];
|
|
2779
|
+
ciPolicySource?: ResolvedGatePolicySource;
|
|
2780
|
+
discovered?: { waves?: number };
|
|
2781
|
+
hooks?: OpWaveHookOutcome[];
|
|
2782
|
+
plans?: Record<string, OpRunWavePlan>;
|
|
2783
|
+
/**
|
|
2784
|
+
* With a `step` (#4992): each run's plan through it, without the engine's
|
|
2785
|
+
* artifact, its change-set part or the wave's group plan. The gate job
|
|
2786
|
+
* leases the rows these name; the applying job plans its runs again
|
|
2787
|
+
* through the step and applies each whose plan digest the decision holds.
|
|
2788
|
+
*/
|
|
2789
|
+
steps?: OpWaveSavedStep[];
|
|
2790
|
+
/**
|
|
2791
|
+
* The held hooks the plan job ran again under an approval an earlier gate
|
|
2792
|
+
* job spent for them (#4994), and how each ended: the gate job writes
|
|
2793
|
+
* their finished lines, whatever else the plan job came to.
|
|
2794
|
+
*/
|
|
2795
|
+
holdsFinished?: OpWaveHandedFinished[];
|
|
2796
|
+
}
|
|
2797
|
+
|
|
2798
|
+
/** A run's step plan as a split wave's plan job saves it (#4992). */
|
|
2799
|
+
export type OpWaveSavedStep = Omit<OpWaveStepPlan, "artifact" | "part" | "group">;
|
|
2800
|
+
|
|
2801
|
+
/**
|
|
2802
|
+
* What a split wave's applying job hands its record job beside its run
|
|
2803
|
+
* record (#4992): the release records of the runs it applied through the
|
|
2804
|
+
* step, and a hold on another gate its hooks found waiting, which the
|
|
2805
|
+
* record job writes with the token.
|
|
2806
|
+
*/
|
|
2807
|
+
export interface OpWaveSplitHandoff {
|
|
2808
|
+
releases?: OpWaveReleaseRequest[];
|
|
2809
|
+
heldOn?: OpWaveHeldOn;
|
|
2810
|
+
}
|
|
2811
|
+
|
|
2812
|
+
/** The lease a split wave's gate job takes for the wave's rows (#4992), which its record job lets go of. */
|
|
2813
|
+
export interface OpWaveSplitLease {
|
|
2814
|
+
gate: string;
|
|
2815
|
+
/** The holder the rows name: the forge's run, the same in every job of it. */
|
|
2816
|
+
run: string;
|
|
2817
|
+
keys: string[];
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
/** Why `options` cannot run its phase (#4987), or undefined. */
|
|
2821
|
+
function opWavePhaseProblem(options: RunOpWaveOptions): string | undefined {
|
|
2822
|
+
const { phase } = options;
|
|
2823
|
+
if (!OP_WAVE_PHASES.includes(phase!)) return `the phase is one of ${OP_WAVE_PHASES.join(", ")}, not ${JSON.stringify(phase)}`;
|
|
2824
|
+
if (options.decide) return "a phase takes the deciding job's place; pass --decide or --phase, not both";
|
|
2825
|
+
if (options.policyOnly && phase !== "plan") return "--policy-only runs in a split wave's plan job alone";
|
|
2826
|
+
if ((options.share !== undefined || options.group !== undefined) && phase !== "apply") return `--${options.share !== undefined ? "share" : "group"} names an applying job; the ${phase} job covers the whole wave`;
|
|
2827
|
+
return undefined;
|
|
2828
|
+
}
|
|
2829
|
+
|
|
2830
|
+
/** Write the plan job's saved plan (#4987). */
|
|
2831
|
+
function saveOpWavePlan(options: RunOpWaveOptions, plan: Omit<OpWaveSavedPlan, "version" | "op" | "wave" | "name">): void {
|
|
2832
|
+
const path = join(options.cwd ?? process.cwd(), opWaveSavedPlanPath(options.spec, options.wave));
|
|
2833
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
2834
|
+
const name = opWaveAt(options.spec, options.wave)?.name ?? `wave-${options.wave}`;
|
|
2835
|
+
// #4994: the re-runs under an approval an earlier gate job spent, whose finished lines the gate job writes.
|
|
2836
|
+
const finished = handedFinished.get(options);
|
|
2837
|
+
const saved: OpWaveSavedPlan = { version: 1, op: options.spec.name, wave: options.wave, name, ...plan, ...(finished?.length ? { holdsFinished: finished } : {}) };
|
|
2838
|
+
writeFileSync(path, JSON.stringify(saved, null, 2) + "\n");
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
/** Read the saved plan the gate job decides on (#4987); throws when the plan job left none. */
|
|
2842
|
+
function readSavedOpWavePlan(options: RunOpWaveOptions): OpWaveSavedPlan {
|
|
2843
|
+
const { spec, wave: k } = options;
|
|
2844
|
+
const rel = opWaveSavedPlanPath(spec, k);
|
|
2845
|
+
let doc: Partial<OpWaveSavedPlan> | null;
|
|
2846
|
+
try {
|
|
2847
|
+
doc = JSON.parse(readFileSync(join(options.cwd ?? process.cwd(), rel), "utf-8")) as Partial<OpWaveSavedPlan> | null;
|
|
2848
|
+
} catch {
|
|
2849
|
+
throw new Error(`wave ${k} found no saved plan at ${rel}: its plan job did not finish, so nothing is decided`);
|
|
2850
|
+
}
|
|
2851
|
+
if (doc?.version !== 1 || doc.op !== spec.name || doc.wave !== k) throw new Error(`${rel} is not the saved plan of wave ${k} of ${spec.name}`);
|
|
2852
|
+
if (doc.end === undefined && (!Array.isArray(doc.members) || typeof doc.digest !== "string")) throw new Error(`${rel} holds neither the plans of wave ${k} nor how its plan job ended`);
|
|
2853
|
+
return doc as OpWaveSavedPlan;
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
/**
|
|
2857
|
+
* The plan job of a split wave (#4987): plan the wave as its deciding job
|
|
2858
|
+
* would, then save the plan for the gate job and stop. It writes nothing to
|
|
2859
|
+
* `chant/lifecycle` and keeps no run record. A wave its plan ends (a hook
|
|
2860
|
+
* failed, a plan failed, the rules at base refuse it) writes its outcome
|
|
2861
|
+
* here, and the saved plan says how it ended.
|
|
2862
|
+
*/
|
|
2863
|
+
async function runOpWavePlanPhase(options: RunOpWaveOptions, dir: string, blocked: ReadonlyMap<string, string>): Promise<RunOpWaveResult> {
|
|
2864
|
+
const { spec, wave: k } = options;
|
|
2865
|
+
const log = options.log ?? ((line: string) => console.error(line));
|
|
2866
|
+
const ended = (r: RunOpWaveResult): RunOpWaveResult => {
|
|
2867
|
+
const all = { ...Object.fromEntries(blocked), ...(r.blocked ?? {}) };
|
|
2868
|
+
if (Object.keys(all).length > 0) r.blocked = all;
|
|
2869
|
+
writeOpWaveOutcome(options, dir, r);
|
|
2870
|
+
saveOpWavePlan(options, { end: r });
|
|
2871
|
+
return r;
|
|
2872
|
+
};
|
|
2873
|
+
let step: WaveStep<OpWavePlanned, RunOpWaveResult>;
|
|
2874
|
+
try {
|
|
2875
|
+
step = await planOpWave(options, { k, pass: 0 }, blocked);
|
|
2876
|
+
} catch (err) {
|
|
2877
|
+
ended({ exitCode: CHANT_EXIT.failed, applied: [], failed: [], moved: [], hooks: [] });
|
|
2878
|
+
throw err;
|
|
2879
|
+
}
|
|
2880
|
+
if ("end" in step) return ended(step.end);
|
|
2881
|
+
const p = step.go;
|
|
2882
|
+
if (p.kind !== "decide") throw new Error(`wave ${k}'s plan job planned as a share; it plans the whole wave`);
|
|
2883
|
+
const digest = opWaveSetDigest(spec, p.members, p.extras);
|
|
2884
|
+
saveOpWavePlan(options, {
|
|
2885
|
+
digest,
|
|
2886
|
+
members: p.members,
|
|
2887
|
+
...(p.extras ? { extras: p.extras } : {}),
|
|
2888
|
+
...(p.pending ? { pending: p.pending } : {}),
|
|
2889
|
+
...(p.overriders ? { overriders: p.overriders } : {}),
|
|
2890
|
+
...(p.overridersSource ? { overridersSource: p.overridersSource } : {}),
|
|
2891
|
+
...(p.changeSet ? { changeSet: p.changeSet } : {}),
|
|
2892
|
+
holds: p.holds,
|
|
2893
|
+
before: p.before,
|
|
2894
|
+
...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}),
|
|
2895
|
+
...(p.ctx.report !== undefined ? { report: p.ctx.report } : {}),
|
|
2896
|
+
...(p.ctx.answerApprovals?.length ? { answerApprovals: p.ctx.answerApprovals } : {}),
|
|
2897
|
+
...(p.decision.ciPolicySource ? { ciPolicySource: p.decision.ciPolicySource } : {}),
|
|
2898
|
+
...(p.decision.discovered ? { discovered: p.decision.discovered } : {}),
|
|
2899
|
+
...(p.ctx.result.hooks.length > 0 ? { hooks: p.ctx.result.hooks } : {}),
|
|
2900
|
+
...(p.ctx.result.plans ? { plans: p.ctx.result.plans } : {}),
|
|
2901
|
+
// #4992: the step plans, for the gate job's lease.
|
|
2902
|
+
...(p.ctx.steps?.size ? { steps: [...p.ctx.steps.values()].map(({ artifact: _a, part: _p, group: _g, ...s }) => s) } : {}),
|
|
2903
|
+
});
|
|
2904
|
+
log(`wave ${k} (${p.ctx.wave.name}): planned ${p.members.length} run${p.members.length === 1 ? "" : "s"} to digest ${digest}; the plan is saved at ${opWaveSavedPlanPath(spec, k)} for the gate job`);
|
|
2905
|
+
return { exitCode: CHANT_EXIT.done, applied: [], failed: [], moved: [], hooks: [...p.ctx.result.hooks], ...(blocked.size > 0 ? { blocked: Object.fromEntries(blocked) } : {}) };
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
/** The runs whose plan file no longer gives the digest the saved plan names (#4987). */
|
|
2909
|
+
function savedPlansMoved(ctx: WaveContext, members: readonly OpWaveMember[]): string[] {
|
|
2910
|
+
return members.filter((m) => readPlanFileField(join(ctx.cwd, opWavePlanPath(ctx.spec, ctx.k, m.member)), "planDigest") !== m.planDigest).map((m) => m.member);
|
|
2911
|
+
}
|
|
2912
|
+
|
|
2913
|
+
/** An applying job's plans of `runs` (#4987), read from the plan files its plan job saved. */
|
|
2914
|
+
function savedRunPlans(ctx: WaveContext, runs: readonly OpWaveRun[]): { planned: Map<string, OpWaveMember> } | { failed: string } {
|
|
2915
|
+
const planned = new Map<string, OpWaveMember>();
|
|
2916
|
+
for (const run of runs) {
|
|
2917
|
+
const path = opWavePlanPath(ctx.spec, ctx.k, run.target);
|
|
2918
|
+
try {
|
|
2919
|
+
const file = JSON.parse(readFileSync(join(ctx.cwd, path), "utf-8")) as OpWavePlanFile;
|
|
2920
|
+
if (typeof file.planDigest !== "string" || file.planDigest === "") throw new Error("it has no planDigest");
|
|
2921
|
+
planned.set(run.target, member(run.target, run.op ?? ctx.spec.op, file));
|
|
2922
|
+
} catch (err) {
|
|
2923
|
+
return { failed: `${shareLabel(ctx.options)} of wave ${ctx.k} (${ctx.wave.name}) found no saved plan of ${run.target} at ${path} (${err instanceof Error ? err.message : String(err)}), so nothing in it was applied` };
|
|
2924
|
+
}
|
|
2925
|
+
}
|
|
2926
|
+
return { planned };
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
/**
|
|
2930
|
+
* The gate job of a split wave (#4987), at the default branch's code with
|
|
2931
|
+
* the token: read the rules at base, take the plan its plan job saved,
|
|
2932
|
+
* refuse it (exit 4) when its set digest is not the one its plan files
|
|
2933
|
+
* give, then decide the gate on it as a wide wave's deciding job does:
|
|
2934
|
+
* write the pending fact, spend the approval and the overrides, keep the
|
|
2935
|
+
* decision record and the outcome. A wave its plan job ended is not
|
|
2936
|
+
* decided again; that end is this job's result, which its run record keeps.
|
|
2937
|
+
*/
|
|
2938
|
+
async function runOpWaveGatePhase(options: RunOpWaveOptions, dir: string): Promise<RunOpWaveResult> {
|
|
2939
|
+
const { spec, wave: k } = options;
|
|
2940
|
+
const cwd = options.cwd ?? process.cwd();
|
|
2941
|
+
const log = options.log ?? ((line: string) => console.error(line));
|
|
2942
|
+
const saved = readSavedOpWavePlan(options);
|
|
2943
|
+
// #4994: the plan job ran a held hook again under an approval an earlier gate job spent; this job writes how it ended.
|
|
2944
|
+
await writeHandedFinished(options, cwd, saved.holdsFinished, log);
|
|
2945
|
+
// #4992: a follow-on run's wave past the spec's is the rest wave's.
|
|
2946
|
+
const at = opWaveAt(spec, k);
|
|
2947
|
+
// #4992: a hook in the plan job held the wave on another gate it could not record; this job records it.
|
|
2948
|
+
const heldOn = saved.end?.exitCode === CHANT_EXIT.waiting ? saved.end.decision?.heldOn : undefined;
|
|
2949
|
+
if (saved.end && !(heldOn?.deferred && at)) {
|
|
2950
|
+
log(`wave ${k}: its plan job ended the wave with exit ${saved.end.exitCode}, so there is nothing to decide`);
|
|
2951
|
+
return saved.end;
|
|
2952
|
+
}
|
|
2953
|
+
if (!at) throw new Error(`Op waves "${spec.name}" has ${spec.waves.length} waves; there is no wave ${k}.`);
|
|
2954
|
+
const members = saved.members ?? [];
|
|
2955
|
+
// The runs the plan job planned, which a discovering wave found and the spec at the default branch may not list.
|
|
2956
|
+
const wave: OpWave = saved.end ? at : { ...at, runs: members.map((m) => ({ target: m.member, ...(m.op !== spec.op ? { op: m.op } : {}) })) };
|
|
2957
|
+
const blocked = saved.blocked ?? {};
|
|
2958
|
+
const result: RunOpWaveResult = { exitCode: 0, applied: [], failed: [], moved: [], hooks: [...(saved.end?.hooks ?? saved.hooks ?? [])], ...(saved.plans ? { plans: saved.plans } : {}) };
|
|
2959
|
+
const ctx: WaveContext = { spec, k, wave, cwd, exec: options.exec ?? spawnExec, log, result, options, gateRule: { approval: "ledger", base: "HEAD", neverOverMcp: spec.neverOverMcp === true } };
|
|
2960
|
+
// #4992: the rows a step's runs lease, as the plan job planned them.
|
|
2961
|
+
if (saved.steps?.length) ctx.steps = new Map(saved.steps.map((s) => [s.target, s]));
|
|
2962
|
+
const end = (r: RunOpWaveResult): RunOpWaveResult => {
|
|
2963
|
+
if (Object.keys(blocked).length > 0) r.blocked = { ...blocked, ...(r.blocked ?? {}) };
|
|
2964
|
+
writeOpWaveOutcome(options, dir, r);
|
|
2965
|
+
return r;
|
|
2966
|
+
};
|
|
2967
|
+
if (heldOn?.deferred) {
|
|
2968
|
+
// The decision is this job's own, from the rules at base: of the plan job's end it takes only the hold.
|
|
2969
|
+
const opened = await openWaveDecision(options, ctx, { evidence: false });
|
|
2970
|
+
if ("end" in opened) return end(opened.end);
|
|
2971
|
+
// #4994: an approved `record` hold, whose approval this job spends for the hook's re-run in the next run's plan job.
|
|
2972
|
+
if (heldOn.deferred.spend) {
|
|
2973
|
+
const spent = await spendDeferredHold(options, ctx, heldOn, log);
|
|
2974
|
+
if ("failed" in spent) return end(failed(ctx, `wave ${k} (${at.name}): ${spent.failed}`));
|
|
2975
|
+
return end(await decided(ctx, opened.go.decision, join(cwd, opWaveRecordPath(spec, k)), spent.heldOn));
|
|
2976
|
+
}
|
|
2977
|
+
const recorded = await recordDeferredHold(options, cwd, heldOn, ctx.gateRule, log);
|
|
2978
|
+
return end(await decided(ctx, opened.go.decision, join(cwd, opWaveRecordPath(spec, k)), recorded));
|
|
2979
|
+
}
|
|
2980
|
+
const opened = await openWaveDecision(options, ctx, { evidence: true });
|
|
2981
|
+
if ("end" in opened) return end(opened.end);
|
|
2982
|
+
const { rules, holds, decision } = opened.go;
|
|
2983
|
+
const digest = opWaveSetDigest(spec, members, saved.extras);
|
|
2984
|
+
const moved = savedPlansMoved(ctx, members);
|
|
2985
|
+
if (digest !== saved.digest || moved.length > 0) {
|
|
2986
|
+
log(
|
|
2987
|
+
`wave ${k} (${wave.name}): ` +
|
|
2988
|
+
(moved.length > 0 ? `the saved plan names plans its plan files do not hold (${moved.join(", ")})` : `the saved plan says digest ${saved.digest}, and its plans give ${digest}`) +
|
|
2989
|
+
", so nothing is decided or applied; run the wave again",
|
|
2990
|
+
);
|
|
2991
|
+
return end({ ...result, exitCode: CHANT_EXIT.moved, moved: moved.length > 0 ? moved : members.map((m) => m.member) });
|
|
2992
|
+
}
|
|
2993
|
+
if (saved.discovered) decision.discovered = saved.discovered;
|
|
2994
|
+
if (saved.ciPolicySource) decision.ciPolicySource = saved.ciPolicySource;
|
|
2995
|
+
if (saved.report !== undefined) ctx.report = saved.report;
|
|
2996
|
+
if (saved.answerApprovals?.length) ctx.answerApprovals = saved.answerApprovals;
|
|
2997
|
+
const planned: Extract<OpWavePlanned, { kind: "decide" }> = {
|
|
2998
|
+
kind: "decide",
|
|
2999
|
+
ctx,
|
|
3000
|
+
recordPath: join(cwd, opWaveRecordPath(spec, k)),
|
|
3001
|
+
rules,
|
|
3002
|
+
decision,
|
|
3003
|
+
holds: [...new Set([...holds, ...(saved.holds ?? [])])],
|
|
3004
|
+
before: saved.before ?? [],
|
|
3005
|
+
members,
|
|
3006
|
+
planned: new Map(members.map((m) => [m.member, m])),
|
|
3007
|
+
...(saved.pending ? { pending: saved.pending } : {}),
|
|
3008
|
+
...(saved.overriders ? { overriders: saved.overriders } : {}),
|
|
3009
|
+
...(saved.overridersSource ? { overridersSource: saved.overridersSource } : {}),
|
|
3010
|
+
...(saved.extras ? { extras: saved.extras } : {}),
|
|
3011
|
+
...(saved.changeSet ? { changeSet: saved.changeSet } : {}),
|
|
3012
|
+
};
|
|
3013
|
+
const step = await decideOpWave(options, { k, pass: 0 }, planned, digest);
|
|
3014
|
+
return end("end" in step ? step.end : result);
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
/**
|
|
3018
|
+
* An applying job's run record port (#4987): a file in the outcome
|
|
3019
|
+
* directory, for the record job, in place of the ledger. #4992: with what
|
|
3020
|
+
* the job hands its record job under `handoff`, which the record job takes
|
|
3021
|
+
* off before it writes the record.
|
|
3022
|
+
*/
|
|
3023
|
+
function fileOpWaveRunPort(options: RunOpWaveOptions, handoff?: OpWaveSplitHandoff): OpWaveRunPort {
|
|
3024
|
+
const cwd = options.cwd ?? process.cwd();
|
|
3025
|
+
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(options.spec);
|
|
3026
|
+
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
3027
|
+
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}) };
|
|
3028
|
+
return {
|
|
3029
|
+
async append(record) {
|
|
3030
|
+
mkdirSync(dir, { recursive: true });
|
|
3031
|
+
const file: OpWaveRunRecordFile = { ...record, ...(handoff && (handoff.releases?.length || handoff.heldOn) ? { handoff } : {}) };
|
|
3032
|
+
writeFileSync(join(dir, opWaveRunRecordFileName(options.wave, part)), JSON.stringify(file, null, 2) + "\n");
|
|
3033
|
+
},
|
|
3034
|
+
};
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
/** A run record file an applying job leaves (#4987), with its hand-off to the record job (#4992). */
|
|
3038
|
+
type OpWaveRunRecordFile = OpRunRecordInput & { handoff?: OpWaveSplitHandoff };
|
|
3039
|
+
|
|
3040
|
+
/** The run records wave `k`'s applying jobs left under `dir` (#4987), at any depth up to four, the newest of each name, by name. */
|
|
3041
|
+
function waveRunRecordFiles(dir: string, k: number, log: (line: string) => void): Array<{ name: string; record: OpWaveRunRecordFile }> {
|
|
3042
|
+
const newest = new Map<string, OpWaveRunRecordFile>();
|
|
3043
|
+
const walk = (at: string, depth: number): void => {
|
|
3044
|
+
let names: string[];
|
|
3045
|
+
try {
|
|
3046
|
+
names = readdirSync(at);
|
|
3047
|
+
} catch {
|
|
3048
|
+
return;
|
|
3049
|
+
}
|
|
3050
|
+
for (const name of names) {
|
|
3051
|
+
const abs = join(at, name);
|
|
3052
|
+
let isDir = false;
|
|
3053
|
+
try {
|
|
3054
|
+
isDir = statSync(abs).isDirectory();
|
|
3055
|
+
} catch {
|
|
3056
|
+
continue;
|
|
3057
|
+
}
|
|
3058
|
+
if (isDir) {
|
|
3059
|
+
if (depth < 4) walk(abs, depth + 1);
|
|
3060
|
+
continue;
|
|
3061
|
+
}
|
|
3062
|
+
if (OP_WAVE_RUN_RECORD_FILE.exec(name)?.[1] !== String(k)) continue;
|
|
3063
|
+
let record: OpWaveRunRecordFile;
|
|
3064
|
+
try {
|
|
3065
|
+
record = JSON.parse(readFileSync(abs, "utf-8")) as OpWaveRunRecordFile;
|
|
3066
|
+
} catch (err) {
|
|
3067
|
+
log(`wave ${k}: the run record ${abs} does not parse, so it is not written: ${err instanceof Error ? err.message : String(err)}`);
|
|
3068
|
+
continue;
|
|
3069
|
+
}
|
|
3070
|
+
const had = newest.get(name);
|
|
3071
|
+
if (!had || had.ended <= record.ended) newest.set(name, record);
|
|
3072
|
+
}
|
|
3073
|
+
};
|
|
3074
|
+
walk(dir, 0);
|
|
3075
|
+
return [...newest].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([name, record]) => ({ name, record }));
|
|
3076
|
+
}
|
|
3077
|
+
|
|
3078
|
+
/**
|
|
3079
|
+
* The record job of a split wave (#4987), at the default branch's code with
|
|
3080
|
+
* the token: write each run record the wave's applying jobs left to the run
|
|
3081
|
+
* ledger, and end as they did: 0 when each applied, else the first exit that
|
|
3082
|
+
* was not a failed run (a moved plan, a held or stood-down apply), else 1.
|
|
3083
|
+
* A record whose push never landed fails the job, as it fails a wave job.
|
|
3084
|
+
*
|
|
3085
|
+
* #4992: it also writes what the applying jobs handed it: the release
|
|
3086
|
+
* records of the runs they applied through the step, and the pending fact
|
|
3087
|
+
* of another gate a hook held the wave on (the wave then waits, exit 3).
|
|
3088
|
+
* Last it lets go of the rows the gate job leased, which the decision names.
|
|
3089
|
+
*/
|
|
3090
|
+
async function runOpWaveRecordPhase(options: RunOpWaveOptions): Promise<RunOpWaveResult> {
|
|
3091
|
+
const { spec, wave: k } = options;
|
|
3092
|
+
const cwd = options.cwd ?? process.cwd();
|
|
3093
|
+
const log = options.log ?? ((line: string) => console.error(line));
|
|
3094
|
+
const outcomeDir = options.outcomeDir ?? opWaveOutcomeDir(spec);
|
|
3095
|
+
const dir = isAbsolute(outcomeDir) ? outcomeDir : join(cwd, outcomeDir);
|
|
3096
|
+
const result: RunOpWaveResult = { exitCode: CHANT_EXIT.done, applied: [], failed: [], moved: [], hooks: [] };
|
|
3097
|
+
// The gate job's decision: the lease it took, and the rules a hold on another gate is recorded under.
|
|
3098
|
+
let decision: OpWaveDecision | undefined;
|
|
3099
|
+
try {
|
|
3100
|
+
decision = JSON.parse(readFileSync(join(cwd, opWaveRecordPath(spec, k)), "utf-8")) as OpWaveDecision;
|
|
3101
|
+
} catch {
|
|
3102
|
+
decision = undefined;
|
|
3103
|
+
}
|
|
3104
|
+
try {
|
|
3105
|
+
return await writeOpWaveRecords(options, { cwd, dir, log, result, decision });
|
|
3106
|
+
} finally {
|
|
3107
|
+
if (decision?.lease?.keys.length) await releaseSplitLease(cwd, decision.lease, k, log);
|
|
3108
|
+
}
|
|
3109
|
+
}
|
|
3110
|
+
|
|
3111
|
+
/** {@link runOpWaveRecordPhase}'s writes, before it lets go of the lease. */
|
|
3112
|
+
async function writeOpWaveRecords(
|
|
3113
|
+
options: RunOpWaveOptions,
|
|
3114
|
+
at: { cwd: string; dir: string; log: (line: string) => void; result: RunOpWaveResult; decision?: OpWaveDecision },
|
|
3115
|
+
): Promise<RunOpWaveResult> {
|
|
3116
|
+
const { wave: k } = options;
|
|
3117
|
+
const { cwd, dir, log, result, decision } = at;
|
|
3118
|
+
const records = waveRunRecordFiles(dir, k, log);
|
|
3119
|
+
if (records.length === 0) {
|
|
3120
|
+
log(`wave ${k}: no applying job of it left a run record under ${relativeTo(cwd, dir)}, so there is none to write`);
|
|
3121
|
+
return { ...result, exitCode: CHANT_EXIT.failed };
|
|
3122
|
+
}
|
|
3123
|
+
const port = options.runs ?? (options.gates ? undefined : gitOpWaveRunPort({ cwd }));
|
|
3124
|
+
const codes: number[] = [];
|
|
3125
|
+
const releases: OpWaveReleaseRequest[] = [];
|
|
3126
|
+
const holds: OpWaveHeldOn[] = [];
|
|
3127
|
+
let lost = false;
|
|
3128
|
+
for (const { name, record: file } of records) {
|
|
3129
|
+
const { handoff, ...record } = file;
|
|
3130
|
+
releases.push(...(handoff?.releases ?? []));
|
|
3131
|
+
if (handoff?.heldOn?.deferred) holds.push(handoff.heldOn);
|
|
3132
|
+
const w = record.wave;
|
|
3133
|
+
codes.push(w?.exitCode ?? CHANT_EXIT.failed);
|
|
3134
|
+
result.applied.push(...(w?.applied ?? []));
|
|
3135
|
+
result.failed.push(...(w?.failed ?? []));
|
|
3136
|
+
result.moved.push(...(w?.moved ?? []));
|
|
3137
|
+
if (w?.blocked) result.blocked = { ...result.blocked, ...w.blocked };
|
|
3138
|
+
if (!port) continue;
|
|
3139
|
+
try {
|
|
3140
|
+
const wrote = await port.append(record);
|
|
3141
|
+
if (wrote && wrote.pushWarning) log(`wave ${k}: the run record ${name} was written and not pushed: ${wrote.pushWarning}`);
|
|
3142
|
+
} catch (err) {
|
|
3143
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
3144
|
+
const never = err instanceof LifecyclePushLostError;
|
|
3145
|
+
if (never) lost = true;
|
|
3146
|
+
log(`wave ${k}: ${never ? `LifecyclePushLostError: the run record ${name} never reached the remote, so the job fails` : `could not record ${name} on the run ledger`}: ${message}`);
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
// #4992: the release records of the runs the applying jobs applied through the step.
|
|
3150
|
+
const releasePort = options.releases === false ? undefined : (options.releases ?? (options.gates ? undefined : gitOpWaveReleasePort({ cwd })));
|
|
3151
|
+
if (releasePort && releases.length > 0) {
|
|
3152
|
+
const missed = await submitStepReleases(releasePort, releases, `wave ${k}`, log);
|
|
3153
|
+
for (const line of missed) log(line);
|
|
3154
|
+
if (missed.length > 0) lost = true;
|
|
3155
|
+
}
|
|
3156
|
+
// #4992: a hold on another gate an applying job's hook found waiting, recorded under the gate job's rules.
|
|
3157
|
+
for (const held of holds) {
|
|
3158
|
+
const rule = { approval: decision?.approval ?? "ledger", base: decision?.base ?? options.base ?? "HEAD", neverOverMcp: decision?.neverOverMcp === true || options.spec.neverOverMcp === true };
|
|
3159
|
+
await recordDeferredHold(options, cwd, held, rule, log);
|
|
3160
|
+
}
|
|
3161
|
+
result.exitCode = codes.every((c) => c === CHANT_EXIT.done) ? CHANT_EXIT.done : (codes.find((c) => c !== CHANT_EXIT.done && c !== CHANT_EXIT.failed) ?? CHANT_EXIT.failed);
|
|
3162
|
+
if (lost && result.exitCode === CHANT_EXIT.done) result.exitCode = CHANT_EXIT.failed;
|
|
3163
|
+
log(`wave ${k}: wrote the run record${records.length === 1 ? "" : "s"} of ${records.length} applying job${records.length === 1 ? "" : "s"}; the wave ends with exit ${result.exitCode}`);
|
|
3164
|
+
return result;
|
|
3165
|
+
}
|
|
3166
|
+
|
|
2378
3167
|
/** The wave engine's loop over this module's steps, from `options.wave` on, with the runs in `skip` left out of that wave. */
|
|
2379
3168
|
async function runOpWaveLoop(options: RunOpWaveOptions, log: (line: string) => void, applied: string[], skip: ReadonlyMap<string, string>): Promise<RunOpWaveResult> {
|
|
2380
3169
|
const { spec } = options;
|
|
@@ -2389,7 +3178,8 @@ async function runOpWaveLoop(options: RunOpWaveOptions, log: (line: string) => v
|
|
|
2389
3178
|
digest: (_at, planned) => (planned.kind === "share" ? planned.decision.digest : opWaveSetDigest(spec, planned.members, planned.extras)),
|
|
2390
3179
|
decide: (at, planned, digest) => (planned.kind === "share" ? shareUnmoved(planned) : decideOpWave(options, at, planned, digest)),
|
|
2391
3180
|
apply: (_at, planned, decided) => applyOpWave(options, planned, decided),
|
|
2392
|
-
|
|
3181
|
+
// #4987: an applying job of a split wave has no plan of its own to make again.
|
|
3182
|
+
replan: (_at, outcome) => outcome.replan !== undefined && options.phase !== "apply",
|
|
2393
3183
|
maxPasses: OP_WAVE_MAX_REPLANS,
|
|
2394
3184
|
passesSpent({ k }, outcome) {
|
|
2395
3185
|
log(`wave ${k}: its apply asked to plan again ${OP_WAVE_MAX_REPLANS} times, so the wave stops here`);
|
|
@@ -2404,7 +3194,7 @@ async function runOpWaveLoop(options: RunOpWaveOptions, log: (line: string) => v
|
|
|
2404
3194
|
Object.assign(targets, outcome.targets ?? {});
|
|
2405
3195
|
// #3821: a `rest` wave goes on through the waves its plan command found.
|
|
2406
3196
|
const found = outcome.decision?.discovered?.waves;
|
|
2407
|
-
const onward = k >= spec.waves.length && last.rest === true && options.share === undefined && options.group === undefined && !options.decide && outcome.exitCode === 0 && found !== undefined && k < found;
|
|
3197
|
+
const onward = k >= spec.waves.length && last.rest === true && options.share === undefined && options.group === undefined && !options.decide && options.phase === undefined && outcome.exitCode === 0 && found !== undefined && k < found;
|
|
2408
3198
|
if (rest.length > 0 || onward) rest.push({ wave: k, exitCode: outcome.exitCode, ...(outcome.decision ? { digest: outcome.decision.digest } : {}) });
|
|
2409
3199
|
if (onward) log(`wave ${k} applied; the plan found ${found} waves, so this job goes on to wave ${k + 1}`);
|
|
2410
3200
|
return onward;
|
|
@@ -2437,9 +3227,10 @@ interface OpWaveBefore {
|
|
|
2437
3227
|
*/
|
|
2438
3228
|
function opWaveBefore(options: RunOpWaveOptions, dir: string): OpWaveBefore {
|
|
2439
3229
|
const { spec, wave: k } = options;
|
|
2440
|
-
const wave = spec
|
|
3230
|
+
const wave = opWaveAt(spec, k);
|
|
2441
3231
|
if (!wave) return { blocked: new Map() };
|
|
2442
|
-
|
|
3232
|
+
// #4987: a split wave's applying job reads its gate job's outcome, as a share reads its deciding job's.
|
|
3233
|
+
if (options.share !== undefined || options.group !== undefined || options.phase === "apply") {
|
|
2443
3234
|
const decided = readWaveOutcomeFile([dir], waveOutcomeFileName(k));
|
|
2444
3235
|
if (!decided) return { blocked: new Map() };
|
|
2445
3236
|
const blocked = new Map<string, string>();
|
|
@@ -2482,7 +3273,7 @@ function opWaveBefore(options: RunOpWaveOptions, dir: string): OpWaveBefore {
|
|
|
2482
3273
|
|
|
2483
3274
|
/** The runs of wave `options.wave` this job holds (#4322): its slice for a share job, every run otherwise. Empty for a discovering wave. */
|
|
2484
3275
|
function jobRuns(options: RunOpWaveOptions, runs?: readonly OpWaveRun[]): string[] {
|
|
2485
|
-
const wave = options.spec
|
|
3276
|
+
const wave = opWaveAt(options.spec, options.wave);
|
|
2486
3277
|
if (!wave) return [];
|
|
2487
3278
|
const all = runs ?? (wave.discover ? [] : wave.runs);
|
|
2488
3279
|
if (options.share === undefined && options.group === undefined) return all.map((r) => r.target);
|
|
@@ -2499,7 +3290,8 @@ function jobRuns(options: RunOpWaveOptions, runs?: readonly OpWaveRun[]): string
|
|
|
2499
3290
|
function writeOpWaveOutcome(options: RunOpWaveOptions, dir: string, result: RunOpWaveResult, members?: Record<string, WaveMemberOutcome>): void {
|
|
2500
3291
|
const out: Record<string, WaveMemberOutcome> = { ...(members ?? {}) };
|
|
2501
3292
|
if (!members) {
|
|
2502
|
-
|
|
3293
|
+
// #4987: a split wave's gate job decides the runs its plan job planned, which the spec at the default branch may not list.
|
|
3294
|
+
const found = result.decision?.discovered !== undefined || (options.phase === "gate" && result.decision) ? result.decision.members.map((m) => ({ target: m.member })) : undefined;
|
|
2503
3295
|
const targets = new Set([...jobRuns(options, found), ...result.applied, ...result.failed, ...result.moved, ...Object.keys(result.blocked ?? {})]);
|
|
2504
3296
|
const applied = new Set(result.applied);
|
|
2505
3297
|
const failed = new Set(result.failed);
|
|
@@ -2515,7 +3307,7 @@ function writeOpWaveOutcome(options: RunOpWaveOptions, dir: string, result: RunO
|
|
|
2515
3307
|
out[t] = by !== undefined ? { status: "blocked", by } : applied.has(t) ? { status: "applied" } : failed.has(t) ? { status: "failed" } : moved.has(t) && result.decision?.refused?.reason !== "moved" ? { status: "not-applied" } : { status: otherwise };
|
|
2516
3308
|
}
|
|
2517
3309
|
}
|
|
2518
|
-
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}) };
|
|
3310
|
+
const part = { ...(options.group !== undefined ? { group: options.group } : {}), ...(options.share !== undefined ? { share: options.share } : {}), ...(options.phase === "apply" ? { apply: true } : {}) };
|
|
2519
3311
|
try {
|
|
2520
3312
|
mkdirSync(dir, { recursive: true });
|
|
2521
3313
|
writeWaveOutcomeFile(dir, options.wave, part, out, options.now ?? new Date().toISOString());
|
|
@@ -2526,6 +3318,8 @@ function writeOpWaveOutcome(options: RunOpWaveOptions, dir: string, result: RunO
|
|
|
2526
3318
|
|
|
2527
3319
|
/** What a share or job group's job calls itself in the log: `share 2`, `job group "prod"`, `share 2 of job group "prod"`. */
|
|
2528
3320
|
function shareLabel(options: Pick<RunOpWaveOptions, "share" | "group">): string {
|
|
3321
|
+
// #4987: a split wave's own applying job is neither a share nor a group's.
|
|
3322
|
+
if (options.group === undefined && options.share === undefined) return "the applying job";
|
|
2529
3323
|
if (options.group === undefined) return `share ${options.share}`;
|
|
2530
3324
|
return `${options.share !== undefined ? `share ${options.share} of ` : ""}job group "${options.group}"`;
|
|
2531
3325
|
}
|
|
@@ -2575,6 +3369,71 @@ interface OpWaveDecided {
|
|
|
2575
3369
|
recordPath?: string;
|
|
2576
3370
|
}
|
|
2577
3371
|
|
|
3372
|
+
/**
|
|
3373
|
+
* Read wave `k`'s rules at base and open its decision: the policy, the
|
|
3374
|
+
* approval mode, the hooks and the points, checked as the wave plans
|
|
3375
|
+
* (#3836, #4290, #4870), with the holds the rules give. Ends the wave when
|
|
3376
|
+
* the rules cannot be read or refuse it. The plan job and the gate job of a
|
|
3377
|
+
* split wave both read them (#4987); `evidence: false` leaves the evidence
|
|
3378
|
+
* the commit needs to the gate job.
|
|
3379
|
+
*/
|
|
3380
|
+
async function openWaveDecision(
|
|
3381
|
+
options: RunOpWaveOptions,
|
|
3382
|
+
ctx: WaveContext,
|
|
3383
|
+
opts: { evidence: boolean },
|
|
3384
|
+
): Promise<WaveStep<{ rules: OpWaveBaseRules; holds: string[]; decision: OpWaveDecision }, RunOpWaveResult>> {
|
|
3385
|
+
const { spec, k, wave, cwd, log } = ctx;
|
|
3386
|
+
const rules = readOpWavePolicy(options.specFile, wave.name, options.base ?? spec.base ?? "HEAD^1", cwd, options.show);
|
|
3387
|
+
const { policy, approval, source, base } = rules;
|
|
3388
|
+
const neverOverMcp = rules.neverOverMcp || spec.neverOverMcp === true;
|
|
3389
|
+
ctx.gateRule = { approval, base, neverOverMcp };
|
|
3390
|
+
noteHooksAtBase(ctx, rules);
|
|
3391
|
+
// #3836: a point at base that cannot be asked fails the wave; the ledger never stands in for it.
|
|
3392
|
+
if (rules.pointRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.pointRefused}, so nothing in it was planned or applied`) };
|
|
3393
|
+
// #4290: rules at base that cannot be read fail the wave; fewer approvers never stand in for them.
|
|
3394
|
+
if (rules.quorumRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.quorumRefused}, so nothing in it was planned or applied`) };
|
|
3395
|
+
if (rules.quorum && rules.point) return { end: failed(ctx, `wave ${k} (${wave.name}): approvals.quorum and a decision point are both set at ${base.slice(0, 12)}; a gate that asks a point takes its deciders from the point, so nothing in it was planned or applied`) };
|
|
3396
|
+
// #4870: the checks whose evidence the commit needs, read at base, before anything is planned.
|
|
3397
|
+
if (rules.evidenceRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.evidenceRefused}, so nothing in it was planned or applied`) };
|
|
3398
|
+
if (rules.evidenceRequires && opts.evidence) {
|
|
3399
|
+
const refused = await opWaveEvidenceRefusal(cwd, options.head ?? "HEAD", base, rules.evidenceRequires);
|
|
3400
|
+
if (refused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${refused}, so nothing in it was planned or applied`) };
|
|
3401
|
+
}
|
|
3402
|
+
if (JSON.stringify(spec.approvals?.quorum ?? null) !== JSON.stringify(rules.quorum ?? null)) {
|
|
3403
|
+
log(`wave ${k} (${wave.name}): its approval rules are read at ${base.slice(0, 12)}, and this change's differ; the ones at base hold`);
|
|
3404
|
+
}
|
|
3405
|
+
const own = opWavePointAt(spec, wave.name);
|
|
3406
|
+
if (JSON.stringify(own) !== JSON.stringify(rules.point)) {
|
|
3407
|
+
log(`wave ${k} (${wave.name}): its gate's decision point is read at ${base.slice(0, 12)} (${rules.point?.name ?? "none"}), and this change's differs (${own?.name ?? "none"}); the one at base holds`);
|
|
3408
|
+
}
|
|
3409
|
+
const holds: string[] = [];
|
|
3410
|
+
if (rules.hooksRefused) {
|
|
3411
|
+
holds.push(rules.hooksRefused);
|
|
3412
|
+
log(`wave ${k} (${wave.name}): ${rules.hooksRefused}, so none run and the wave waits for an approval`);
|
|
3413
|
+
}
|
|
3414
|
+
const hooks = Object.fromEntries(Object.entries(rules.hooks).filter(([, list]) => list.length > 0));
|
|
3415
|
+
const decision: OpWaveDecision = {
|
|
3416
|
+
wave: k,
|
|
3417
|
+
name: wave.name,
|
|
3418
|
+
op: spec.name,
|
|
3419
|
+
gate: opWaveGateName(spec, k),
|
|
3420
|
+
policy,
|
|
3421
|
+
approval,
|
|
3422
|
+
policySource: source,
|
|
3423
|
+
base,
|
|
3424
|
+
digest: "",
|
|
3425
|
+
members: [],
|
|
3426
|
+
status: "not-required",
|
|
3427
|
+
...(Object.keys(hooks).length > 0 ? { hooks } : {}),
|
|
3428
|
+
...(rules.spend ? { spend: "base" as const } : spec.approvals?.spend === true ? { spend: "change" as const } : {}),
|
|
3429
|
+
...(rules.applyHooksRunBy === "apply" ? { applyHooksRunBy: "apply" as const } : {}),
|
|
3430
|
+
...(spec.digest?.over === "changing" ? { digestOver: "changing" as const } : {}),
|
|
3431
|
+
...(neverOverMcp ? { neverOverMcp: true as const } : {}),
|
|
3432
|
+
...(rules.point ? { point: { ...rules.point } } : {}),
|
|
3433
|
+
};
|
|
3434
|
+
return { go: { rules, holds, decision } };
|
|
3435
|
+
}
|
|
3436
|
+
|
|
2578
3437
|
/**
|
|
2579
3438
|
* Plan wave `k`: read its rules at base and run the before-plan hooks
|
|
2580
3439
|
* around planning its runs, or for a share job, read the deciding job's
|
|
@@ -2610,7 +3469,8 @@ async function planOpWave(options: RunOpWaveOptions, { k, pass }: WaveAttempt<Ru
|
|
|
2610
3469
|
const ctx: WaveContext = { spec, k, wave, cwd, exec, log, result, options, gateRule: { approval: "ledger", base: "HEAD", neverOverMcp: spec.neverOverMcp === true } };
|
|
2611
3470
|
if (pass > 0) log(`wave ${k} (${wave.name}): plan ${pass + 1}`);
|
|
2612
3471
|
|
|
2613
|
-
|
|
3472
|
+
// #4987: a split wave's applying job applies under its gate job's decision, as a share does.
|
|
3473
|
+
if (options.share !== undefined || group !== undefined || options.phase === "apply") {
|
|
2614
3474
|
let decision: OpWaveDecision;
|
|
2615
3475
|
try {
|
|
2616
3476
|
decision = JSON.parse(readFileSync(recordPath, "utf-8")) as OpWaveDecision;
|
|
@@ -2637,6 +3497,21 @@ async function planOpWave(options: RunOpWaveOptions, { k, pass }: WaveAttempt<Ru
|
|
|
2637
3497
|
// #4322: the runs its deciding job blocked are neither planned nor applied here.
|
|
2638
3498
|
const slice = whole.filter((r) => !skip?.has(r.target));
|
|
2639
3499
|
if (slice.length === 0) return { end: result };
|
|
3500
|
+
// #4987: an applying job of a split wave plans nothing and runs no plan hooks (its plan and gate jobs did): it takes
|
|
3501
|
+
// the plan files the plan job saved, and shareUnmoved applies those whose digest the decision holds.
|
|
3502
|
+
if (options.phase === "apply" && spec.step) {
|
|
3503
|
+
// #4992: a step's plan lives in the process that made it (the engine's saved plan, the artifact it hands its
|
|
3504
|
+
// apply), so the applying job plans its runs again through the step, as a share does, and shareUnmoved applies
|
|
3505
|
+
// each whose plan digest the decision holds. Its hooks ran in the plan and gate jobs.
|
|
3506
|
+
ctx.steps = new Map();
|
|
3507
|
+
const { members } = await planWaveRuns(ctx, slice);
|
|
3508
|
+
return { go: { kind: "share", ctx, decision, slice, planned: new Map(members.map((m) => [m.member, m])), holds: [] } };
|
|
3509
|
+
}
|
|
3510
|
+
if (options.phase === "apply") {
|
|
3511
|
+
const saved = savedRunPlans(ctx, slice);
|
|
3512
|
+
if ("failed" in saved) return { end: failed(ctx, saved.failed) };
|
|
3513
|
+
return { go: { kind: "share", ctx, decision, slice, planned: saved.planned, holds: [] } };
|
|
3514
|
+
}
|
|
2640
3515
|
const before = await runBeforePlanHooks(ctx, decision.hooks, slice);
|
|
2641
3516
|
if (before.failed) return { end: failed(ctx, `a hook before the plan failed, so nothing in ${part} of wave ${k} was planned or applied`) };
|
|
2642
3517
|
if (before.waitingOn) return { end: await waitOnOther(ctx, decision, before.waitingOn) };
|
|
@@ -2653,55 +3528,11 @@ async function planOpWave(options: RunOpWaveOptions, { k, pass }: WaveAttempt<Ru
|
|
|
2653
3528
|
}
|
|
2654
3529
|
|
|
2655
3530
|
// The policy, the approval mode and the hooks, read at base before anything runs.
|
|
2656
|
-
|
|
2657
|
-
const
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
// #3836: a point at base that cannot be asked fails the wave; the ledger never stands in for it.
|
|
2662
|
-
if (rules.pointRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.pointRefused}, so nothing in it was planned or applied`) };
|
|
2663
|
-
// #4290: rules at base that cannot be read fail the wave; fewer approvers never stand in for them.
|
|
2664
|
-
if (rules.quorumRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.quorumRefused}, so nothing in it was planned or applied`) };
|
|
2665
|
-
if (rules.quorum && rules.point) return { end: failed(ctx, `wave ${k} (${wave.name}): approvals.quorum and a decision point are both set at ${base.slice(0, 12)}; a gate that asks a point takes its deciders from the point, so nothing in it was planned or applied`) };
|
|
2666
|
-
// #4870: the checks whose evidence the commit needs, read at base, before anything is planned.
|
|
2667
|
-
if (rules.evidenceRefused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${rules.evidenceRefused}, so nothing in it was planned or applied`) };
|
|
2668
|
-
if (rules.evidenceRequires) {
|
|
2669
|
-
const refused = await opWaveEvidenceRefusal(cwd, options.head ?? "HEAD", base, rules.evidenceRequires);
|
|
2670
|
-
if (refused) return { end: failed(ctx, `wave ${k} (${wave.name}): ${refused}, so nothing in it was planned or applied`) };
|
|
2671
|
-
}
|
|
2672
|
-
if (JSON.stringify(spec.approvals?.quorum ?? null) !== JSON.stringify(rules.quorum ?? null)) {
|
|
2673
|
-
log(`wave ${k} (${wave.name}): its approval rules are read at ${base.slice(0, 12)}, and this change's differ; the ones at base hold`);
|
|
2674
|
-
}
|
|
2675
|
-
const own = opWavePointAt(spec, wave.name);
|
|
2676
|
-
if (JSON.stringify(own) !== JSON.stringify(rules.point)) {
|
|
2677
|
-
log(`wave ${k} (${wave.name}): its gate's decision point is read at ${base.slice(0, 12)} (${rules.point?.name ?? "none"}), and this change's differs (${own?.name ?? "none"}); the one at base holds`);
|
|
2678
|
-
}
|
|
2679
|
-
const holds: string[] = [];
|
|
2680
|
-
if (rules.hooksRefused) {
|
|
2681
|
-
holds.push(rules.hooksRefused);
|
|
2682
|
-
log(`wave ${k} (${wave.name}): ${rules.hooksRefused}, so none run and the wave waits for an approval`);
|
|
2683
|
-
}
|
|
2684
|
-
const gate = opWaveGateName(spec, k);
|
|
2685
|
-
const hooks = Object.fromEntries(Object.entries(rules.hooks).filter(([, list]) => list.length > 0));
|
|
2686
|
-
const decision: OpWaveDecision = {
|
|
2687
|
-
wave: k,
|
|
2688
|
-
name: wave.name,
|
|
2689
|
-
op: spec.name,
|
|
2690
|
-
gate,
|
|
2691
|
-
policy,
|
|
2692
|
-
approval,
|
|
2693
|
-
policySource: source,
|
|
2694
|
-
base,
|
|
2695
|
-
digest: "",
|
|
2696
|
-
members: [],
|
|
2697
|
-
status: "not-required",
|
|
2698
|
-
...(Object.keys(hooks).length > 0 ? { hooks } : {}),
|
|
2699
|
-
...(rules.spend ? { spend: "base" as const } : spec.approvals?.spend === true ? { spend: "change" as const } : {}),
|
|
2700
|
-
...(rules.applyHooksRunBy === "apply" ? { applyHooksRunBy: "apply" as const } : {}),
|
|
2701
|
-
...(spec.digest?.over === "changing" ? { digestOver: "changing" as const } : {}),
|
|
2702
|
-
...(neverOverMcp ? { neverOverMcp: true as const } : {}),
|
|
2703
|
-
...(rules.point ? { point: { ...rules.point } } : {}),
|
|
2704
|
-
};
|
|
3531
|
+
// #4987: a split wave's plan job leaves the evidence its commit needs to the gate job, which can fetch the notes.
|
|
3532
|
+
const opened = await openWaveDecision(options, ctx, { evidence: options.phase !== "plan" });
|
|
3533
|
+
if ("end" in opened) return opened;
|
|
3534
|
+
const { rules, holds, decision } = opened.go;
|
|
3535
|
+
const { base } = rules;
|
|
2705
3536
|
if (skip?.size && !wave.discover) {
|
|
2706
3537
|
// #4322: a run a failure blocked is neither planned nor in the set digest.
|
|
2707
3538
|
wave = { ...wave, runs: wave.runs.filter((r) => !skip.has(r.target)) };
|
|
@@ -2711,6 +3542,11 @@ async function planOpWave(options: RunOpWaveOptions, { k, pass }: WaveAttempt<Ru
|
|
|
2711
3542
|
if (before.failed) return { end: failed(ctx, `a hook before the plan failed, so nothing in wave ${k} was planned or applied`) };
|
|
2712
3543
|
// #4926: the policy step records no decision; the wave's own step does.
|
|
2713
3544
|
if (before.waitingOn && options.policyOnly) return { end: failed(ctx, `wave ${k} (${wave.name}): a hook before the plan waits on ${before.waitingOn}, so its policy did not run`) };
|
|
3545
|
+
if (before.waitingOn && options.phase === "plan") {
|
|
3546
|
+
// #4992: the plan job ends the wave waiting on the other gate; its gate job records that gate's pending fact and the decision.
|
|
3547
|
+
markHeldOn(decision, before.waitingOn);
|
|
3548
|
+
return { end: { ...result, decision, exitCode: CHANT_EXIT.waiting } };
|
|
3549
|
+
}
|
|
2714
3550
|
if (before.waitingOn) return { end: await decided(ctx, decision, recordPath, before.waitingOn) };
|
|
2715
3551
|
if (spec.step) ctx.steps = new Map();
|
|
2716
3552
|
const { members, pending: planPending, overriders: planOverriders, overridersSource: planOverridersSource, extras: planExtras, waves: found, changeSet, parts } = await planWaveRuns(ctx, wave.runs, wave.discover === true);
|
|
@@ -2887,8 +3723,15 @@ async function decideOpWave(
|
|
|
2887
3723
|
);
|
|
2888
3724
|
// A wide wave's shares apply under the deciding job's approval and overrides, so they are spent here, once.
|
|
2889
3725
|
if (options.decide) {
|
|
2890
|
-
|
|
2891
|
-
|
|
3726
|
+
// #4992: a split wave's gate job holds the rows its step's runs change for the applying jobs, before it spends the approval.
|
|
3727
|
+
const lease = options.phase === "gate" && ctx.spec.step ? await leaseSplitRows(options, ctx, decision, recordPath) : undefined;
|
|
3728
|
+
if (lease && "end" in lease) return lease;
|
|
3729
|
+
if (!(await spendApproval(options, ctx, decision)) || !(await spendOverrides(ctx, decision))) {
|
|
3730
|
+
if (lease && !(await lease.release())) log(`wave ${k} (${wave.name}): could not let go of the rows it held; the next apply takes them over once this run is gone`);
|
|
3731
|
+
if (lease) delete decision.lease;
|
|
3732
|
+
return { end: ctx.result };
|
|
3733
|
+
}
|
|
3734
|
+
if (decision.spentAt || decision.overridesSpentAt || decision.lease) writeRecord(recordPath, decision, result);
|
|
2892
3735
|
return { end: result };
|
|
2893
3736
|
}
|
|
2894
3737
|
return { go: { runs: wave.runs, earlierHolds: [], recordPath } };
|
|
@@ -3044,6 +3887,11 @@ async function decided(ctx: WaveContext, decision: OpWaveDecision, recordPath: s
|
|
|
3044
3887
|
log(`The pull request's review does not approve wave ${k}: ${decision.review.reason}`);
|
|
3045
3888
|
}
|
|
3046
3889
|
const hook = await runGateHooks(ctx, decision, recordPath);
|
|
3890
|
+
if (decision.status === "waiting" && decision.heldOn?.spentForRerun) {
|
|
3891
|
+
// #4994: the approval is spent for the held hook's re-run; nobody needs to approve anything more.
|
|
3892
|
+
log(`Nothing in wave ${k} or after it ran. The approval of ${decision.heldOn.op} / ${decision.heldOn.gate} by ${decision.heldOn.spentForRerun.approvedBy} is spent for ${decision.heldOn.hook} to run again; run the wave again from its plan job`);
|
|
3893
|
+
return { ...result, exitCode: CHANT_EXIT.waiting };
|
|
3894
|
+
}
|
|
3047
3895
|
if (decision.status === "waiting") {
|
|
3048
3896
|
log(`Nothing in wave ${k} or after it ran. ${decision.heldOn ? `It waits on ${decision.heldOn.op} / ${decision.heldOn.gate}.` : "Read the plans, then"} approve: ${decision.approve} (with --resume, the approval also starts this job again)`);
|
|
3049
3897
|
return { ...result, exitCode: CHANT_EXIT.waiting };
|
|
@@ -3821,6 +4669,8 @@ async function applyStage(
|
|
|
3821
4669
|
}
|
|
3822
4670
|
if (before.waitingOn) {
|
|
3823
4671
|
markHeldOn(decision, before.waitingOn);
|
|
4672
|
+
// #4992: an applying job's hold goes to its record job, which records the other gate's pending fact.
|
|
4673
|
+
if (before.waitingOn.deferred) result.handoff = { ...result.handoff, heldOn: before.waitingOn };
|
|
3824
4674
|
if (recordPath) {
|
|
3825
4675
|
writeRecord(recordPath, decision, result);
|
|
3826
4676
|
await runGateHooks(ctx, decision, recordPath);
|
|
@@ -3833,7 +4683,8 @@ async function applyStage(
|
|
|
3833
4683
|
}
|
|
3834
4684
|
if (!(await settleHolds(options, ctx, decision, holds, recordPath))) return;
|
|
3835
4685
|
// #4552: on the resource grain the wave holds the rows its runs change, before its approval is spent.
|
|
3836
|
-
|
|
4686
|
+
// #4992: a split wave's gate job holds them for its applying jobs, which hold no write token.
|
|
4687
|
+
const lease = spec.step && options.phase !== "apply" ? await leaseWaveRows(options, ctx, runs) : undefined;
|
|
3837
4688
|
if (lease && "failed" in lease) {
|
|
3838
4689
|
failed(ctx, lease.failed);
|
|
3839
4690
|
return;
|
|
@@ -4158,10 +5009,21 @@ async function applyStepRunsAtOnce(options: RunOpWaveOptions, ctx: WaveContext,
|
|
|
4158
5009
|
* 9), which fails the wave by name.
|
|
4159
5010
|
*/
|
|
4160
5011
|
async function recordStepReleases(options: RunOpWaveOptions, ctx: WaveContext, decision: OpWaveDecision, runs: readonly OpWaveRun[]): Promise<void> {
|
|
4161
|
-
const { spec, k, wave, log, result } = ctx;
|
|
4162
5012
|
if (options.releases === false) return;
|
|
5013
|
+
// #4992: a split wave's applying job holds no write token; its record job writes the releases.
|
|
5014
|
+
if (options.phase === "apply") {
|
|
5015
|
+
const requests = stepReleaseRequests(options, ctx, decision, runs);
|
|
5016
|
+
if (requests.length > 0) ctx.result.handoff = { ...ctx.result.handoff, releases: requests };
|
|
5017
|
+
return;
|
|
5018
|
+
}
|
|
4163
5019
|
const port = options.releases ?? (options.gates ? undefined : gitOpWaveReleasePort({ cwd: ctx.cwd }));
|
|
4164
5020
|
if (!port) return;
|
|
5021
|
+
await writeStepReleases(port, ctx, stepReleaseRequests(options, ctx, decision, runs));
|
|
5022
|
+
}
|
|
5023
|
+
|
|
5024
|
+
/** The release requests of the runs this wave applied through its step (#4615). */
|
|
5025
|
+
function stepReleaseRequests(options: RunOpWaveOptions, ctx: WaveContext, decision: OpWaveDecision, runs: readonly OpWaveRun[]): OpWaveReleaseRequest[] {
|
|
5026
|
+
const { spec, wave, result } = ctx;
|
|
4165
5027
|
const applied = new Set(result.applied);
|
|
4166
5028
|
let gitSha = options.head;
|
|
4167
5029
|
const requests: OpWaveReleaseRequest[] = [];
|
|
@@ -4179,20 +5041,36 @@ async function recordStepReleases(options: RunOpWaveOptions, ctx: WaveContext, d
|
|
|
4179
5041
|
}
|
|
4180
5042
|
requests.push({ ...names, target: run.target, planDigest: plan.planDigest, ...(gitSha ? { gitSha } : {}), ...(decision.approvedBy ? { approver: decision.approvedBy } : {}) });
|
|
4181
5043
|
}
|
|
4182
|
-
|
|
5044
|
+
return requests;
|
|
5045
|
+
}
|
|
5046
|
+
|
|
5047
|
+
/** Write release requests (#4615), failing the wave through `ctx` when one never reached the remote. */
|
|
5048
|
+
async function writeStepReleases(port: OpWaveReleasePort, ctx: WaveContext, requests: readonly OpWaveReleaseRequest[]): Promise<void> {
|
|
5049
|
+
for (const lost of await submitStepReleases(port, requests, `wave ${ctx.k} (${ctx.wave.name})`, ctx.log)) failed(ctx, lost);
|
|
5050
|
+
}
|
|
5051
|
+
|
|
5052
|
+
/**
|
|
5053
|
+
* Write release requests on their release ledgers (#4615): a line per run
|
|
5054
|
+
* recorded or not. Returns a failure for each one written that never
|
|
5055
|
+
* reached the remote (plywood#22 finding 9), which fails the wave.
|
|
5056
|
+
*/
|
|
5057
|
+
async function submitStepReleases(port: OpWaveReleasePort, requests: readonly OpWaveReleaseRequest[], label: string, log: (line: string) => void): Promise<string[]> {
|
|
5058
|
+
if (requests.length === 0) return [];
|
|
4183
5059
|
let outcomes: OpWaveReleaseOutcome[];
|
|
4184
5060
|
try {
|
|
4185
|
-
outcomes = await port.record(requests);
|
|
5061
|
+
outcomes = await port.record([...requests]);
|
|
4186
5062
|
} catch (err) {
|
|
4187
|
-
log(
|
|
4188
|
-
return;
|
|
5063
|
+
log(`${label}: applied, and no release record was written: ${err instanceof Error ? err.message : String(err)}`);
|
|
5064
|
+
return [];
|
|
4189
5065
|
}
|
|
5066
|
+
const lost: string[] = [];
|
|
4190
5067
|
for (const [i, req] of requests.entries()) {
|
|
4191
5068
|
const got = outcomes[i];
|
|
4192
5069
|
if (got?.recorded) log(`${req.target}: recorded on the "${got.env}" release ledger as ${got.component} ${got.digest}${got.releaseDigest ? ` under release ${got.releaseDigest}` : ""}`);
|
|
4193
|
-
else if (got && !got.recorded && got.lost)
|
|
5070
|
+
else if (got && !got.recorded && got.lost) lost.push(`${req.target}: applied, and LifecyclePushLostError: the release record for ${req.component} in "${req.env}" was written and never reached the remote, so the wave fails: ${got.reason}`);
|
|
4194
5071
|
else log(`${req.target}: applied, and no release record was written for ${req.component} in "${req.env}": ${got?.reason ?? "the port gave no answer"}`);
|
|
4195
5072
|
}
|
|
5073
|
+
return lost;
|
|
4196
5074
|
}
|
|
4197
5075
|
|
|
4198
5076
|
/**
|