@intentius/chant 0.124.80 → 0.124.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@intentius/chant",
3
- "version": "0.124.80",
3
+ "version": "0.124.81",
4
4
  "description": "Declarative infrastructure-as-code toolkit — TypeScript on Node.js",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://intentius.io/chant",
@@ -148,6 +148,17 @@ export interface DeployContext {
148
148
  * nothing to call.
149
149
  */
150
150
  afterInit?: () => Promise<void>;
151
+ /**
152
+ * Run `fn` alone among the runs beside this one that call it with the
153
+ * same `key` (#4864), and return what it returns. An op wave whose
154
+ * `parallelism` plans and applies several runs at once gives each run the
155
+ * wave's one section, so a capability keeps the part that cannot run twice
156
+ * at once to one run at a time, such as `terraform init` writing the
157
+ * shared provider cache (`TF_PLUGIN_CACHE_DIR`), keyed on that directory.
158
+ * Never nest two calls with the same key: the inner one waits on the
159
+ * outer. Absent where runs go one at a time: then nothing runs beside it.
160
+ */
161
+ serial?: <T>(key: string, fn: () => Promise<T>) => Promise<T>;
151
162
  /**
152
163
  * The run's row lease (#4594), set by an apply path that plans inside its
153
164
  * steps (`run --components`, `components fan-out`, a promote). The driver
package/src/op/index.ts CHANGED
@@ -124,6 +124,7 @@ export type {
124
124
  export {
125
125
  OP_WAVE_COST_FORMATS,
126
126
  OP_WAVE_COST_MEMBER,
127
+ OP_WAVE_PARALLEL_STEP_FLOOR,
127
128
  OP_WAVE_STEP_WAVE_FLOOR,
128
129
  OP_WAVE_TOGETHER_MEMBER,
129
130
  opWaveCostVerdict,
@@ -48,7 +48,6 @@
48
48
  */
49
49
 
50
50
  import { isAbsolute } from "node:path";
51
- import { NEXT_RELEASE } from "../release-floors";
52
51
  import type { ChangeSetPart } from "../change-set";
53
52
  import type { OpRunPolicyRow } from "./runtime";
54
53
  import type { WaveMember } from "../wave-engine";
@@ -59,7 +58,7 @@ import { OP_WAVE_PENDING_RESERVED, type OpWaveHookOutcome } from "./stage-hooks"
59
58
  export const OP_WAVE_AFTER_PLAN_WAVE_FLOOR = "0.124.79";
60
59
 
61
60
  /** The first chant whose wave answer takes `approvals`, recorded at the wave's gate before chant decides it (terragucci issue 1303). */
62
- export const OP_WAVE_ANSWER_APPROVALS_FLOOR = NEXT_RELEASE;
61
+ export const OP_WAVE_ANSWER_APPROVALS_FLOOR = "0.124.80";
63
62
 
64
63
  /**
65
64
  * An approval a product settled before the gate (terragucci issue 1303),
@@ -84,6 +84,7 @@ import type { ResolvedGatePolicySource } from "../policy-source";
84
84
  import type { WaveMember } from "../wave-engine";
85
85
  import type { OpWave, OpWaveRun, OpWavesSpec } from "./op-waves";
86
86
  import { resolvePlannedLock, type LockGrain, type PlannedLock } from "../lifecycle/lock-rows";
87
+ import { NEXT_RELEASE } from "../release-floors";
87
88
 
88
89
  import { OP_WAVE_COST_FORMATS, OP_WAVE_COST_MEMBER, type OpWaveCostFormat, type OpWaveCostHold, type OpWaveStep } from "./op-waves-capability-spec";
89
90
 
@@ -170,6 +171,14 @@ export const OP_WAVE_TOGETHER_MEMBER = "(wave plan)";
170
171
  /** The release that has `discover` with a `step`, and `together` waves (#4842). */
171
172
  export const OP_WAVE_STEP_WAVE_FLOOR = "0.124.79";
172
173
 
174
+ /**
175
+ * The first chant whose step plans a wave's runs `parallelism` at once, as it
176
+ * applies them, and hands each run the wave's serial section
177
+ * (`DeployContext.serial`), where `terraform-apply` and `terragrunt-apply`
178
+ * take their inits one at a time on a shared provider cache (#4864).
179
+ */
180
+ export const OP_WAVE_PARALLEL_STEP_FLOOR = NEXT_RELEASE;
181
+
173
182
  /** Whether a part changes nothing, and whether it destroys or replaces something. A plan that changes only outputs changes something (#4849). */
174
183
  export function partEffects(part: ChangeSetPart | undefined): { empty: boolean; destructive: boolean } {
175
184
  if (!part) return { empty: false, destructive: false };
@@ -184,7 +193,7 @@ function resolveCapability(registry: CapabilityRegistry, kind: string): Capabili
184
193
 
185
194
  /** Plan one run through the spec's step (#4545), writing its plan file at `planPath` (relative to `cwd`) as a per-run `plan` command would. */
186
195
  export async function planOpWaveStep(
187
- at: { spec: OpWavesSpec; wave: OpWave; cwd: string; registry: CapabilityRegistry; planPath: string; lockGrain?: LockGrain | null; policyBase?: string; processEnv?: Record<string, string>; afterInit?: () => Promise<void> },
196
+ at: { spec: OpWavesSpec; wave: OpWave; cwd: string; registry: CapabilityRegistry; planPath: string; lockGrain?: LockGrain | null; policyBase?: string; processEnv?: Record<string, string>; afterInit?: () => Promise<void>; serial?: DeployContext["serial"] },
188
197
  run: OpWaveRun,
189
198
  ): Promise<OpWaveStepPlan> {
190
199
  const { spec, wave, cwd, registry } = at;
@@ -195,7 +204,7 @@ export async function planOpWaveStep(
195
204
  let lock: PlannedLock;
196
205
  try {
197
206
  // #4842: the run hooks' environment and check after init.
198
- planned = await capability.plan({ ...opWaveStepContext(spec, wave, run, cwd), ...(at.lockGrain ? { lockGrain: at.lockGrain } : {}), ...(at.policyBase ? { policyBase: at.policyBase } : {}), ...(at.processEnv ? { processEnv: at.processEnv } : {}), ...(at.afterInit ? { afterInit: at.afterInit } : {}) }, input as never);
207
+ planned = await capability.plan({ ...opWaveStepContext(spec, wave, run, cwd), ...(at.lockGrain ? { lockGrain: at.lockGrain } : {}), ...(at.policyBase ? { policyBase: at.policyBase } : {}), ...(at.processEnv ? { processEnv: at.processEnv } : {}), ...(at.afterInit ? { afterInit: at.afterInit } : {}), ...(at.serial ? { serial: at.serial } : {}) }, input as never);
199
208
  // #4552: the run's grain (declared at base, else its lexicon's) and, on the resource grain, the rows its apply leases.
200
209
  lock = await resolvePlannedLock(capability, input, planned, at.lockGrain);
201
210
  } catch (err) {
@@ -458,7 +467,7 @@ export async function opWaveStepFacts(
458
467
  }
459
468
 
460
469
  /** Apply one run through the spec's step (#4545), with the plan the gate decided. */
461
- export async function applyOpWaveStep(at: { spec: OpWavesSpec; wave: OpWave; registry: CapabilityRegistry; processEnv?: Record<string, string>; cwd?: string }, run: OpWaveRun, plan: OpWaveStepPlan): Promise<OpWaveStepApply> {
470
+ export async function applyOpWaveStep(at: { spec: OpWavesSpec; wave: OpWave; registry: CapabilityRegistry; processEnv?: Record<string, string>; cwd?: string; serial?: DeployContext["serial"] }, run: OpWaveRun, plan: OpWaveStepPlan): Promise<OpWaveStepApply> {
462
471
  const { spec, wave, registry } = at;
463
472
  let capability: Capability<never, unknown>;
464
473
  try {
@@ -466,7 +475,7 @@ export async function applyOpWaveStep(at: { spec: OpWavesSpec; wave: OpWave; reg
466
475
  } catch (err) {
467
476
  return { outcome: "failed", reason: err instanceof Error ? err.message : String(err) };
468
477
  }
469
- const ctx: DeployContext = { ...opWaveStepContext(spec, wave, run, at.cwd), ...(plan.artifact !== undefined ? { plans: { [plan.member]: plan.artifact } } : {}), ...(at.processEnv ? { processEnv: at.processEnv } : {}) };
478
+ const ctx: DeployContext = { ...opWaveStepContext(spec, wave, run, at.cwd), ...(plan.artifact !== undefined ? { plans: { [plan.member]: plan.artifact } } : {}), ...(at.processEnv ? { processEnv: at.processEnv } : {}), ...(at.serial ? { serial: at.serial } : {}) };
470
479
  const input = opWaveStepInput(spec, wave, run);
471
480
  try {
472
481
  const output = await capability.run(ctx, input as never);
@@ -224,9 +224,8 @@ describe("approvals a product settled before the gate, in the wave answer (terra
224
224
  ]);
225
225
  });
226
226
 
227
- test("the floor is the next release", async () => {
227
+ test("the floor is the release it shipped in", async () => {
228
228
  const entry = await import("./index");
229
- const { NEXT_RELEASE } = await import("../release-floors");
230
- expect(entry.OP_WAVE_ANSWER_APPROVALS_FLOOR).toBe(NEXT_RELEASE);
229
+ expect(entry.OP_WAVE_ANSWER_APPROVALS_FLOOR).toBe("0.124.80");
231
230
  });
232
231
  });
@@ -0,0 +1,185 @@
1
+ /**
2
+ * A step wave's runs planned `parallelism` at once (#4864), as they apply,
3
+ * with the wave's serial section on each run's context
4
+ * (`DeployContext.serial`). A fake capability with a delay per root records
5
+ * how many plans, applies and serial sections are in flight; the wave's
6
+ * digest, members, plan files, run hooks and after-plan runs must come out
7
+ * as one at a time makes them.
8
+ */
9
+
10
+ import { describe, expect, test } from "vitest";
11
+ import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
12
+ import { join } from "node:path";
13
+ import { tmpdir } from "node:os";
14
+ import { opWavePlanPath, runOpWave, type OpWaveShowAtBase } from "./op-waves-run";
15
+ import type { OpWavesSpec } from "./op-waves";
16
+ import type { OpWaveRunHooks } from "./op-waves-run-hooks";
17
+ import type { OpWaveAfterPlanHook } from "./op-waves-after-plan";
18
+ import { memoryGateLedgerPort } from "./gate";
19
+ import { CapabilityRegistry, type Capability, type CapabilityPlan, type DeployContext } from "../components/capability";
20
+ import { keyedSerial } from "../parallelism";
21
+
22
+ const ROOTS = ["a", "b", "c", "d"];
23
+ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
24
+
25
+ /** A gauge of how many of something are in flight, and the most there ever were. */
26
+ function gauge() {
27
+ const g = { now: 0, most: 0 };
28
+ return {
29
+ g,
30
+ async in<T>(fn: () => Promise<T>): Promise<T> {
31
+ g.now++;
32
+ g.most = Math.max(g.most, g.now);
33
+ try {
34
+ return await fn();
35
+ } finally {
36
+ g.now--;
37
+ }
38
+ },
39
+ };
40
+ }
41
+
42
+ /**
43
+ * A capability whose plan takes an "init" through `ctx.serial` under one key,
44
+ * then plans; the first root plans slowest, so plans at once finish in the
45
+ * other order. `fail` roots throw from their plan after `failAfterMs`.
46
+ */
47
+ function capability(opts: { fail?: Record<string, number> } = {}) {
48
+ const plans = gauge();
49
+ const inits = gauge();
50
+ const applies = gauge();
51
+ const seen = { serial: [] as boolean[], planned: [] as string[] };
52
+ const delay = (root: string) => (ROOTS.length - ROOTS.indexOf(root)) * 15;
53
+ const cap: Capability<Record<string, unknown>, unknown> = {
54
+ kind: "fake-apply",
55
+ async plan(ctx: DeployContext, input) {
56
+ const root = String(input.root);
57
+ seen.serial.push(typeof ctx.serial === "function");
58
+ const init = () => inits.in(() => sleep(10));
59
+ await (ctx.serial ? ctx.serial("plugin-cache", init) : init());
60
+ return plans.in(async (): Promise<CapabilityPlan> => {
61
+ const failAfter = opts.fail?.[root];
62
+ if (failAfter !== undefined) {
63
+ await sleep(failAfter);
64
+ throw new Error(`${root} broke`);
65
+ }
66
+ await sleep(delay(root));
67
+ seen.planned.push(root);
68
+ const planDigest = `sha256:${String(ROOTS.indexOf(root) + 1).repeat(64)}`;
69
+ return { member: root, planDigest, artifact: { root }, changeSet: { member: { member: root, lexicon: "terraform", planner: "chant", status: "planned", planDigest, holes: [] }, entries: [{ member: root, lexicon: "terraform", planner: "chant", address: `x.${root}`, type: "x", action: "create", attributes: [] }] } };
70
+ });
71
+ },
72
+ async run(ctx: DeployContext, input) {
73
+ const root = String(input.root);
74
+ const init = () => inits.in(() => sleep(5));
75
+ await (ctx.serial ? ctx.serial("plugin-cache", init) : init());
76
+ return applies.in(async () => {
77
+ await sleep(delay(root));
78
+ return { applied: root };
79
+ });
80
+ },
81
+ };
82
+ return { registry: new CapabilityRegistry().register(cap), plans: plans.g, inits: inits.g, applies: applies.g, seen };
83
+ }
84
+
85
+ const spec = (parallelism: number | undefined): OpWavesSpec => ({
86
+ name: "roots",
87
+ op: "tf-apply",
88
+ ...(parallelism !== undefined ? { parallelism } : {}),
89
+ step: { kind: "fake-apply", root: "{target}" },
90
+ waves: [{ name: "dev", runs: ROOTS.map((target) => ({ target })), gate: "never" }],
91
+ });
92
+
93
+ async function wave(s: OpWavesSpec, registry: CapabilityRegistry, o: { runHooks?: OpWaveRunHooks; afterPlan?: OpWaveAfterPlanHook } = {}) {
94
+ const cwd = mkdtempSync(join(tmpdir(), "op-waves-par-plans-"));
95
+ const show: OpWaveShowAtBase = () => ({ sha: "b".repeat(40), text: JSON.stringify(s) });
96
+ const log: string[] = [];
97
+ try {
98
+ const result = await runOpWave({ spec: s, specFile: "waves.json", wave: 1, cwd, show, gates: memoryGateLedgerPort(), capabilities: registry, log: (l) => log.push(l), ...o });
99
+ const files = ROOTS.map((r) => {
100
+ const f = join(cwd, opWavePlanPath(s, 1, r));
101
+ return existsSync(f) ? readFileSync(f, "utf-8") : undefined;
102
+ });
103
+ return { result, files, log };
104
+ } finally {
105
+ rmSync(cwd, { recursive: true, force: true });
106
+ }
107
+ }
108
+
109
+ describe("a step wave's runs planned at once (#4864)", () => {
110
+ test("up to parallelism plans run at once, and the serial section keeps the inits to one at a time", async () => {
111
+ const c = capability();
112
+ const { result } = await wave(spec(3), c.registry);
113
+ expect(result).toMatchObject({ exitCode: 0, applied: ROOTS });
114
+ expect(c.plans.most).toBe(3);
115
+ expect(c.applies.most).toBe(3);
116
+ expect(c.inits.most).toBe(1);
117
+ expect(c.seen.serial).toEqual([true, true, true, true]);
118
+ // The first root plans slowest, so it is not the first to finish.
119
+ expect(c.seen.planned[0]).not.toBe("a");
120
+ });
121
+
122
+ test("the digest, members, plan files, run hooks and after-plan runs come out as one at a time makes them", async () => {
123
+ const take = async (parallelism: number | undefined) => {
124
+ const c = capability();
125
+ const before: string[] = [];
126
+ const after: string[] = [];
127
+ const runHooks: OpWaveRunHooks = { name: "product", beforePlan: (r) => void before.push(r.target) };
128
+ const afterPlan: OpWaveAfterPlanHook = { name: "report", run: (p) => void after.push(`${p.target} ${p.planDigest}`) };
129
+ const got = await wave(spec(parallelism), c.registry, { runHooks, afterPlan });
130
+ return { c, got, before, after };
131
+ };
132
+ const one = await take(undefined);
133
+ const many = await take(4);
134
+ expect(one.c.plans.most).toBe(1);
135
+ expect(many.c.plans.most).toBe(4);
136
+ expect(many.got.result.decision?.digest).toBe(one.got.result.decision?.digest);
137
+ expect(many.got.result.decision?.members).toEqual(one.got.result.decision?.members);
138
+ expect(many.got.result.decision?.members.map((m) => m.member)).toEqual(ROOTS);
139
+ expect(many.got.files).toEqual(one.got.files);
140
+ expect(many.got.files.every((f) => f !== undefined)).toBe(true);
141
+ expect(many.after).toEqual(one.after);
142
+ expect([...many.before].sort()).toEqual(one.before);
143
+ expect(many.got.result.applied).toEqual(one.got.result.applied);
144
+ // One at a time, a run's context has no serial section.
145
+ expect(one.c.seen.serial).toEqual([false, false, false, false]);
146
+ });
147
+
148
+ test("a failing plan fails the wave with the first failing run in the wave's order, though a later one failed first", async () => {
149
+ const c = capability({ fail: { b: 40, c: 1 } });
150
+ // c's init takes its turn after b's and fails at once; b fails later, and it is the one one at a time stops at.
151
+ await expect(wave(spec(4), c.registry)).rejects.toThrow(/^planning b through fake-apply failed: b broke$/);
152
+ });
153
+
154
+ test("a plan that fails starts no more plans", async () => {
155
+ const c = capability({ fail: { a: 1 } });
156
+ await expect(wave(spec(2), c.registry)).rejects.toThrow(/planning a through fake-apply failed/);
157
+ // a fails beside b, which finishes; c and d never start.
158
+ expect(c.seen.planned).toEqual(["b"]);
159
+ });
160
+ });
161
+
162
+ describe("keyedSerial (#4864)", () => {
163
+ test("one call per key at a time, other keys beside it, and a failure is the call's own", async () => {
164
+ const serial = keyedSerial();
165
+ const same = gauge();
166
+ const order: string[] = [];
167
+ const call = (key: string, name: string, ms: number, fail = false) =>
168
+ serial(key, () =>
169
+ same.in(async () => {
170
+ await sleep(ms);
171
+ order.push(name);
172
+ if (fail) throw new Error(name);
173
+ return name;
174
+ }),
175
+ );
176
+ const other = serial("other", async () => {
177
+ order.push("other");
178
+ return "other";
179
+ });
180
+ const got = await Promise.allSettled([call("k", "k1", 30, true), call("k", "k2", 5), other]);
181
+ expect(got.map((g) => g.status)).toEqual(["rejected", "fulfilled", "fulfilled"]);
182
+ expect(order).toEqual(["other", "k1", "k2"]);
183
+ expect(same.g.most).toBe(1);
184
+ });
185
+ });
@@ -81,6 +81,11 @@ describe("runs applied at once (#4625)", () => {
81
81
  expect(seen.most).toBe(1);
82
82
  });
83
83
 
84
+ test("the spec refuses together with a parallelism over 1", () => {
85
+ const s = spec(2, [{ target: "a" }, { target: "b" }]);
86
+ expect(() => assertOpWavesSpec({ ...s, waves: [{ ...s.waves[0]!, together: true, parallelism: 2 }] })).toThrow(/no parallelism over 1/);
87
+ });
88
+
84
89
  test("the spec refuses a parallelism over 1 without a step, and one that is not a whole number", () => {
85
90
  const { step: _s, ...noStep } = spec(2, [{ target: "a" }]);
86
91
  expect(() => assertOpWavesSpec({ ...noStep, apply: ["./apply", "{target}"], plan: ["./plan", "{target}", "{plan}"] })).toThrow(/apply at once only through `step`/);
@@ -106,7 +106,7 @@ import type { LockGrain } from "../lifecycle/lock-rows";
106
106
  import type { LockLiveness } from "../lifecycle/resource-lock";
107
107
  import type { GatePolicyExec } from "../gate-policy";
108
108
  import { gitOpWaveNewerCommit, opWaveStandDownReason, tipAppliesTargets, type OpWaveNewerCommitPort } from "./op-waves-stand-down";
109
- import { mapLimited } from "../parallelism";
109
+ import { keyedSerial, mapLimited, type SerialSection } from "../parallelism";
110
110
  import {
111
111
  applyOpWaveStep,
112
112
  opWaveCosts,
@@ -971,6 +971,8 @@ interface WaveContext {
971
971
  steps?: Map<string, OpWaveStepPlan>;
972
972
  /** Each run's environment from the run hooks (#4842), by target, which its plan and its apply run with. */
973
973
  runEnv?: Map<string, Record<string, string>>;
974
+ /** The wave's serial section (#4864), made when its runs plan or apply several at once and shared by its plans and applies. */
975
+ serial?: SerialSection;
974
976
  /** The plans file's change set (#4551), which a `pull-request` gate checks the pull request's approval against (#4550). */
975
977
  changeSetDoc?: ChangeSetDocument;
976
978
  /** An after-plan hook's report of the wave's plans (#4842), stored beside the pending fact. */
@@ -1160,15 +1162,16 @@ function runHookRun(at: { spec: OpWavesSpec; k: number; wave: OpWave; cwd: strin
1160
1162
  * with the run's environment, then `afterApply`, whose files go on the
1161
1163
  * result and whose failure fails a run that applied.
1162
1164
  */
1163
- async function applyStepRun(ctx: WaveContext, registry: CapabilityRegistry, run: OpWaveRun, plan: OpWaveStepPlan): Promise<OpWaveStepApply & { files?: string[] }> {
1165
+ async function applyStepRun(ctx: WaveContext, registry: CapabilityRegistry, run: OpWaveRun, plan: OpWaveStepPlan, serial?: SerialSection): Promise<OpWaveStepApply & { files?: string[] }> {
1164
1166
  const { spec, wave, k, options } = ctx;
1165
1167
  const hooks = opWaveRunHookList(options.runHooks);
1166
1168
  const env = ctx.runEnv?.get(run.target);
1167
- if (hooks.length === 0) return applyOpWaveStep({ spec, wave, registry, cwd: ctx.cwd, ...(env ? { processEnv: env } : {}) }, run, plan);
1169
+ const at = { spec, wave, registry, cwd: ctx.cwd, ...(env ? { processEnv: env } : {}), ...(serial ? { serial } : {}) };
1170
+ if (hooks.length === 0) return applyOpWaveStep(at, run, plan);
1168
1171
  const planned = { ...runHookRun(ctx, run), planDigest: plan.planDigest, plan: opWavePlanPath(spec, k, run.target), ...(plan.artifact !== undefined ? { artifact: plan.artifact } : {}) };
1169
1172
  const stop = await runOpWaveBeforeApply(hooks, planned);
1170
1173
  if (stop !== undefined) return { outcome: "failed", reason: stop };
1171
- const got = await applyOpWaveStep({ spec, wave, registry, cwd: ctx.cwd, ...(env ? { processEnv: env } : {}) }, run, plan);
1174
+ const got = await applyOpWaveStep(at, run, plan);
1172
1175
  const after = await runOpWaveAfterApply(hooks, { ...planned, outcome: got.outcome, ...(got.outcome === "applied" ? { output: got.output } : { reason: got.reason }) });
1173
1176
  const files = after.files.length > 0 ? { files: after.files } : {};
1174
1177
  if (after.failed && got.outcome === "applied") return { outcome: "failed", reason: `applied, and then ${after.failed}`, ...(got.facts ? { facts: got.facts } : {}), ...files };
@@ -1179,10 +1182,13 @@ async function applyStepRun(ctx: WaveContext, registry: CapabilityRegistry, run:
1179
1182
  /**
1180
1183
  * Plan `runs` of wave `k` (#4545): through the spec's `step` when it names
1181
1184
  * one, each run's plan kept on `at.steps` for its apply, else with the
1182
- * commands ({@link planRuns}).
1185
+ * commands ({@link planRuns}). With the wave's `parallelism` over 1 the
1186
+ * step plans up to that many runs at once (#4864), each with the wave's
1187
+ * serial section on its context, and the members, parts and plan files come
1188
+ * out as one at a time makes them, in the wave's order.
1183
1189
  */
1184
1190
  async function planWaveRuns(
1185
- at: { spec: OpWavesSpec; k: number; wave: OpWave; cwd: string; exec: OpWaveExec; options: Pick<RunOpWaveOptions, "capabilities" | "lockGrain" | "base" | "waveExec" | "runHooks" | "decide" | "share" | "group">; steps?: Map<string, OpWaveStepPlan>; runEnv?: Map<string, Record<string, string>> },
1191
+ at: { spec: OpWavesSpec; k: number; wave: OpWave; cwd: string; exec: OpWaveExec; options: Pick<RunOpWaveOptions, "capabilities" | "lockGrain" | "base" | "waveExec" | "runHooks" | "decide" | "share" | "group">; steps?: Map<string, OpWaveStepPlan>; runEnv?: Map<string, Record<string, string>>; serial?: SerialSection },
1186
1192
  runs: readonly OpWaveRun[],
1187
1193
  discover = false,
1188
1194
  ): Promise<PlannedRuns> {
@@ -1223,11 +1229,39 @@ async function planWaveRuns(
1223
1229
  }
1224
1230
  together = plans[0]?.group?.digest;
1225
1231
  } else {
1226
- for (const run of runs) {
1232
+ const limit = wave.parallelism ?? spec.parallelism ?? 1;
1233
+ // #4864: runs that plan beside each other share the wave's serial section, such as a turn at the provider cache for init.
1234
+ const serial = limit > 1 && runs.length > 1 ? (at.serial ??= keyedSerial()) : undefined;
1235
+ const planOne = async (run: OpWaveRun): Promise<OpWaveStepPlan> => {
1227
1236
  const before = await beforePlan(run);
1228
- const plan = await planOpWaveStep({ spec, wave, cwd, registry, planPath: opWavePlanPath(spec, k, run.target), lockGrain, ...(policyBase ? { policyBase } : {}), ...(before.env ? { processEnv: before.env } : {}), ...(before.afterInit ? { afterInit: before.afterInit } : {}) }, run);
1229
- at.steps?.set(run.target, plan);
1230
- plans.push(plan);
1237
+ return planOpWaveStep({ spec, wave, cwd, registry, planPath: opWavePlanPath(spec, k, run.target), lockGrain, ...(policyBase ? { policyBase } : {}), ...(before.env ? { processEnv: before.env } : {}), ...(before.afterInit ? { afterInit: before.afterInit } : {}), ...(serial ? { serial } : {}) }, run);
1238
+ };
1239
+ if (!serial) {
1240
+ for (const run of runs) {
1241
+ const plan = await planOne(run);
1242
+ at.steps?.set(run.target, plan);
1243
+ plans.push(plan);
1244
+ }
1245
+ } else {
1246
+ // #4864: up to `limit` plans at once, started in the wave's order. A failure starts no more, and the
1247
+ // first failing run in the wave's order fails the wave, the one a plan one at a time would have stopped at.
1248
+ let stopped = false;
1249
+ const got = await mapLimited(runs, limit, async (run): Promise<{ plan: OpWaveStepPlan } | { error: unknown } | undefined> => {
1250
+ if (stopped) return undefined;
1251
+ try {
1252
+ return { plan: await planOne(run) };
1253
+ } catch (error) {
1254
+ stopped = true;
1255
+ return { error };
1256
+ }
1257
+ });
1258
+ const failure = got.find((g) => g !== undefined && "error" in g);
1259
+ if (failure && "error" in failure) throw failure.error;
1260
+ for (const g of got) {
1261
+ if (!g || !("plan" in g)) continue;
1262
+ at.steps?.set(g.plan.target, g.plan);
1263
+ plans.push(g.plan);
1264
+ }
1231
1265
  }
1232
1266
  }
1233
1267
  const members = plans.map((p) => ({
@@ -3990,6 +4024,8 @@ async function applyStepRunsAtOnce(options: RunOpWaveOptions, ctx: WaveContext,
3990
4024
  type Outcome = { applied: true } | { failed: true } | { moved: true } | { blockedBy: string };
3991
4025
  const order = new Map(runs.map((r, i) => [r.target, i]));
3992
4026
  const registry = await opWaveRegistry(options, ctx.cwd, options.capabilities);
4027
+ // #4864: the section the wave's plans took turns in, so an apply that inits again waits its turn too.
4028
+ const serial = runs.length > 1 ? (ctx.serial ??= keyedSerial()) : undefined;
3993
4029
  const settled = new Map<string, Promise<Outcome>>();
3994
4030
  const outcomes = await mapLimited(runs, limit, (run, i) => {
3995
4031
  const going = (async (): Promise<Outcome> => {
@@ -4008,7 +4044,7 @@ async function applyStepRunsAtOnce(options: RunOpWaveOptions, ctx: WaveContext,
4008
4044
  return { failed: true };
4009
4045
  }
4010
4046
  const startedMs = targetStarted(options, k, run.target);
4011
- const got = await applyStepRun(ctx, registry, run, plan);
4047
+ const got = await applyStepRun(ctx, registry, run, plan, serial);
4012
4048
  targetSettled(options, result, k, run.target, startedMs, got.outcome === "applied" ? "ok" : got.outcome === "moved" ? "skipped" : "failed", {
4013
4049
  ...(got.outcome !== "applied" ? { error: got.reason } : {}),
4014
4050
  ...(got.facts ? { facts: got.facts } : {}),
@@ -412,7 +412,7 @@ export interface OpWave {
412
412
  hooks?: OpWaveHooks;
413
413
  /** Where this wave's jobs run (#3914). Default: the spec's `runner`. */
414
414
  runner?: OpWaveRunner;
415
- /** How many of the wave's runs apply at once (#4625), over the spec's `parallelism`. */
415
+ /** How many of the wave's runs plan and apply at once (#4625, #4864), over the spec's `parallelism`. */
416
416
  parallelism?: number;
417
417
  /**
418
418
  * The OIDC tokens this wave's jobs get (#3914), as `{ <variable>: <audience> }`;
@@ -513,10 +513,14 @@ export interface OpWavesSpec {
513
513
  */
514
514
  step?: OpWaveStep;
515
515
  /**
516
- * How many runs of a wave apply at once (#4625), a wave's own
517
- * `parallelism` winning. Default 1, one at a time. Over 1 it needs `step`:
518
- * a run whose wave lists one of its `dependsOn` before it waits for that
519
- * one, and is blocked when it failed, as one at a time.
516
+ * How many runs of a wave plan and apply at once (#4625, #4864), a wave's
517
+ * own `parallelism` winning. Default 1, one at a time. Over 1 it needs
518
+ * `step`: the plans start in the wave's order and the digest lists them as
519
+ * one at a time does, and a run whose wave lists one of its `dependsOn`
520
+ * before it waits for that one to apply, and is blocked when it failed, as
521
+ * one at a time. Each run's context carries the wave's serial section
522
+ * (`DeployContext.serial`), which `terraform-apply` and `terragrunt-apply`
523
+ * take their inits in when the runs share a provider cache.
520
524
  */
521
525
  parallelism?: number;
522
526
  /**
@@ -31,3 +31,29 @@ export async function mapLimited<T, R>(items: readonly T[], limit: number | unde
31
31
  await Promise.all(Array.from({ length: Math.max(1, limit) }, worker));
32
32
  return out;
33
33
  }
34
+
35
+ /**
36
+ * Run `fn` alone among the calls with the same `key` (#4864): a call waits
37
+ * for every earlier call with its key to settle, then runs, while calls
38
+ * with other keys go on beside it. A call's failure is its own; the next
39
+ * call with the key runs anyway.
40
+ */
41
+ export type SerialSection = <T>(key: string, fn: () => Promise<T>) => Promise<T>;
42
+
43
+ /** A fresh {@link SerialSection}, whose keys are its own: an op wave makes one per wave job. */
44
+ export function keyedSerial(): SerialSection {
45
+ const tails = new Map<string, Promise<unknown>>();
46
+ return <T>(key: string, fn: () => Promise<T>): Promise<T> => {
47
+ const before = tails.get(key) ?? Promise.resolve();
48
+ const mine = before.then(() => fn());
49
+ const tail = mine.then(
50
+ () => undefined,
51
+ () => undefined,
52
+ );
53
+ tails.set(key, tail);
54
+ void tail.then(() => {
55
+ if (tails.get(key) === tail) tails.delete(key);
56
+ });
57
+ return mine;
58
+ };
59
+ }
@@ -19,19 +19,19 @@
19
19
  * this constant to the next patch (#4501). The check fails while a floor
20
20
  * still names this constant after its version shipped.
21
21
  */
22
- export const NEXT_RELEASE = "0.124.80";
22
+ export const NEXT_RELEASE = "0.124.81";
23
23
 
24
24
  /** The first chant whose terraform lexicon's `terraform-apply` reads take `dir` and `workspace`, and read an upstream with `resolveRoot`'s binary and `readEnv` (terragucci issue 1303). */
25
- export const TERRAFORM_APPLY_INSTANCE_READS_FLOOR = NEXT_RELEASE;
25
+ export const TERRAFORM_APPLY_INSTANCE_READS_FLOOR = "0.124.80";
26
26
 
27
27
  /** The first chant whose terraform lexicon's `terraform-apply` takes `binary`, and `terraformApplyCapabilityWith({ resolveRoot })` gives each root its own binary (terragucci issue 1303). */
28
- export const TERRAFORM_APPLY_ROOT_BINARY_FLOOR = NEXT_RELEASE;
28
+ export const TERRAFORM_APPLY_ROOT_BINARY_FLOOR = "0.124.80";
29
29
 
30
30
  /** The first chant whose op wave hands a step's capability the wave's `cwd` (`DeployContext.cwd`), where `terraform-apply` looks for `chant.config.*` and `terragrunt-apply` runs when the step names neither (#4854). */
31
- export const OP_WAVE_STEP_CWD_FLOOR = NEXT_RELEASE;
31
+ export const OP_WAVE_STEP_CWD_FLOOR = "0.124.80";
32
32
 
33
33
  /** The first chant whose terraform lexicon's `terraform-apply` takes `workspace`: init in `default`, then `workspace select -or-create=true`, and plan, apply and output in it (#4854). */
34
- export const TERRAFORM_APPLY_WORKSPACE_FLOOR = NEXT_RELEASE;
34
+ export const TERRAFORM_APPLY_WORKSPACE_FLOOR = "0.124.80";
35
35
 
36
36
  /** The first chant whose change-set member can carry `outputs`, the outputs its plan changes, and counts a member with any as a change (#4849). */
37
37
  export const CHANGE_SET_OUTPUTS_FLOOR = "0.124.79";
@@ -46,13 +46,13 @@ export const YODEL_NEW_ENVS_FLOOR = "0.124.78";
46
46
  export const YODEL_CONFLICT_REPLY_FLOOR = "0.124.78";
47
47
 
48
48
  /** The first chant with `@intentius/chant/ci-note-commands`, `chant components pr-notes` and the GitLab pipeline's `note-commands` job (`--note-commands`), and whose forge reads answer `branchRuns` and `pullRequestsUpdated` (#4785). */
49
- export const NOTE_COMMANDS_FLOOR = NEXT_RELEASE;
49
+ export const NOTE_COMMANDS_FLOOR = "0.124.80";
50
50
 
51
51
  /** The first chant whose forge reads answer `compare`, `job` and, on GitLab, `artifactFile`, and whose `RunJob` names its `run`, `ref`, `tag` and `url` (#4862). */
52
- export const PR_FORGE_COMPARE_READS_FLOOR = NEXT_RELEASE;
52
+ export const PR_FORGE_COMPARE_READS_FLOOR = "0.124.80";
53
53
 
54
54
  /** The first chant whose forge reads answer `headRuns`, `run`, `runJobs` and `artifacts`: a head's workflow runs (GitLab's pipelines), a run, its jobs and its artifacts by name (#4846). */
55
- export const PR_FORGE_RUN_READS_FLOOR = NEXT_RELEASE;
55
+ export const PR_FORGE_RUN_READS_FLOOR = "0.124.80";
56
56
 
57
57
  /** The first chant whose `@intentius/chant/pr-forge-reads` exports the forge reads beside the clients (#4809). */
58
58
  export const PR_FORGE_READS_FLOOR = "0.124.78";