@intentius/chant-lexicon-terraform 0.124.79 → 0.124.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/components/capability-plugin.d.ts +74 -1
  2. package/dist/components/capability-plugin.d.ts.map +1 -1
  3. package/dist/components/capability-plugin.js +239 -138
  4. package/dist/components/compose-group.d.ts +2 -1
  5. package/dist/components/compose-group.d.ts.map +1 -1
  6. package/dist/components/compose-group.js +6 -6
  7. package/dist/index.d.ts +1 -1
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +3 -1
  10. package/dist/init-turn.d.ts +21 -0
  11. package/dist/init-turn.d.ts.map +1 -0
  12. package/dist/init-turn.js +30 -0
  13. package/dist/integrity.json +2 -2
  14. package/dist/manifest.json +1 -1
  15. package/dist/op/activities/terraform.d.ts +22 -0
  16. package/dist/op/activities/terraform.d.ts.map +1 -1
  17. package/dist/op/activities/terraform.js +29 -5
  18. package/dist/state-reads.d.ts +20 -3
  19. package/dist/state-reads.d.ts.map +1 -1
  20. package/dist/state-reads.js +20 -4
  21. package/dist/terragrunt/capability.d.ts +5 -0
  22. package/dist/terragrunt/capability.d.ts.map +1 -1
  23. package/dist/terragrunt/capability.js +21 -7
  24. package/dist/terragrunt/run.d.ts +8 -0
  25. package/dist/terragrunt/run.d.ts.map +1 -1
  26. package/dist/terragrunt/run.js +12 -6
  27. package/dist/terragrunt/wave.d.ts +9 -0
  28. package/dist/terragrunt/wave.d.ts.map +1 -1
  29. package/dist/terragrunt/wave.js +22 -0
  30. package/package.json +2 -2
  31. package/src/components/capability-plugin.ts +279 -135
  32. package/src/components/compose-group.ts +8 -7
  33. package/src/components/terraform-apply-op-wave.test.ts +295 -0
  34. package/src/components/terraform-apply-parallel.test.ts +158 -0
  35. package/src/index.ts +5 -0
  36. package/src/init-turn.ts +36 -0
  37. package/src/op/activities/terraform.ts +45 -4
  38. package/src/state-reads.ts +23 -5
  39. package/src/terragrunt/capability.test.ts +94 -0
  40. package/src/terragrunt/capability.ts +22 -7
  41. package/src/terragrunt/run.ts +18 -4
  42. package/src/terragrunt/wave.ts +22 -0
@@ -12,6 +12,14 @@
12
12
  * as `-var`, under the step's own `vars`, which win. A read whose upstream
13
13
  * has no state, or no such output yet, waits: the root is not planned on a
14
14
  * stand-in, and the wave says which upstream to apply first.
15
+ *
16
+ * An Atmos instance is read the way terragucci's `upstreamOutputs` reads it
17
+ * (terragucci issue 1303): `dir` names its directory when it has no
18
+ * `terraform.roots` entry, and `workspace` the workspace its state is in, so
19
+ * the read inits in `default`, selects or creates the workspace with
20
+ * `TF_WORKSPACE` unset, then runs `output -json` in it. A product's
21
+ * `resolveRoot` answers the binary and the environment (its role) the read
22
+ * runs with, never the reader's.
15
23
  */
16
24
 
17
25
  /** One var a root reads from another root's state. */
@@ -22,6 +30,10 @@ export interface TerraformStateRead {
22
30
  root: string;
23
31
  /** The output, then the keys into its value. */
24
32
  output: string[];
33
+ /** The upstream's directory, from the project root, when it has no `terraform.roots` entry (terragucci issue 1303). */
34
+ dir?: string;
35
+ /** The workspace the upstream's state is in (terragucci issue 1303), selected or created after an init in `default`. */
36
+ workspace?: string;
25
37
  }
26
38
 
27
39
  /** What filling a root's reads came to. */
@@ -51,6 +63,8 @@ export function stateReadsProblems(reads: unknown): string[] {
51
63
  else vars.add(read.var);
52
64
  if (typeof read.root !== "string" || read.root === "") problems.push(`reads[${i}].root must name a terraform.roots entry`);
53
65
  if (!Array.isArray(read.output) || read.output.length === 0 || !read.output.every((k) => typeof k === "string" && k !== "")) problems.push(`reads[${i}].output must be the output's name, then any keys into its value`);
66
+ if (read.dir !== undefined && (typeof read.dir !== "string" || read.dir === "")) problems.push(`reads[${i}].dir must name the upstream's directory`);
67
+ if (read.workspace !== undefined && (typeof read.workspace !== "string" || read.workspace.trim() === "")) problems.push(`reads[${i}].workspace must name a workspace`);
54
68
  }
55
69
  return problems;
56
70
  }
@@ -71,11 +85,11 @@ export function readOutputValue(outputs: Record<string, unknown>, output: readon
71
85
  * Fill `reads` from each upstream's outputs, read once per upstream through
72
86
  * `outputsOf`, which throws when the state cannot be read.
73
87
  */
74
- export async function fillStateReads(reads: readonly TerraformStateRead[], outputsOf: (root: string) => Promise<Record<string, unknown>>): Promise<FilledStateReads> {
88
+ export async function fillStateReads(reads: readonly TerraformStateRead[], outputsOf: (root: string, read: TerraformStateRead) => Promise<Record<string, unknown>>): Promise<FilledStateReads> {
75
89
  const result: FilledStateReads = { vars: {}, waiting: [], errors: [] };
76
90
  const cache = new Map<string, Promise<Record<string, unknown>>>();
77
91
  for (const read of reads) {
78
- if (!cache.has(read.root)) cache.set(read.root, outputsOf(read.root));
92
+ if (!cache.has(read.root)) cache.set(read.root, outputsOf(read.root, read));
79
93
  let outputs: Record<string, unknown>;
80
94
  try {
81
95
  outputs = await cache.get(read.root)!;
@@ -103,14 +117,18 @@ export function waitingReadsReason(waiting: FilledStateReads["waiting"]): string
103
117
  * The reads an Atmos instance's edges file lists (`{ reads: [{ var,
104
118
  * upstream, output }] }`, as terragucci's `atmos write` leaves beside each
105
119
  * instance), with each upstream mapped to its `terraform.roots` entry by
106
- * `rootOf` (by default the upstream's own path).
120
+ * `rootOf` (by default the upstream's own path). `at` answers, per
121
+ * upstream, its directory and the workspace its state is in (terragucci
122
+ * issue 1303), which the read inits and selects.
107
123
  */
108
- export function atmosStateReads(edges: unknown, rootOf: (upstream: string) => string = (u) => u): TerraformStateRead[] {
124
+ export function atmosStateReads(edges: unknown, rootOf: (upstream: string) => string = (u) => u, at?: (upstream: string) => { dir?: string; workspace?: string }): TerraformStateRead[] {
109
125
  const reads = (edges as { reads?: unknown } | null)?.reads;
110
126
  if (!Array.isArray(reads)) return [];
111
127
  return reads.flatMap((r) => {
112
128
  const read = r as { var?: unknown; upstream?: unknown; output?: unknown } | null;
113
129
  if (!read || typeof read.var !== "string" || typeof read.upstream !== "string" || !Array.isArray(read.output) || !read.output.every((k) => typeof k === "string")) return [];
114
- return [{ var: read.var, root: rootOf(read.upstream), output: read.output as string[] }];
130
+ // terragucci issue 1303: where the upstream instance is and which workspace holds its state, when the caller knows.
131
+ const where = at?.(read.upstream);
132
+ return [{ var: read.var, root: rootOf(read.upstream), output: read.output as string[], ...(where?.dir ? { dir: where.dir } : {}), ...(where?.workspace ? { workspace: where.workspace } : {}) }];
115
133
  });
116
134
  }
@@ -325,3 +325,97 @@ describe("a unit whose plan changes only outputs (#4849)", () => {
325
325
  expect(changing.digest).toBe((await decisionOf("all")).digest);
326
326
  });
327
327
  });
328
+
329
+ describe("terragrunt-apply runs where the wave runs (chant#4854)", () => {
330
+ it("with no dir in the step, a wave run in process for another directory finds, plans and applies there", async () => {
331
+ const dir = tmp();
332
+ const calls: Array<Call & { cwd: string }> = [];
333
+ const inner = stub(calls);
334
+ const exec: TerragruntExec = (f, args, o) => {
335
+ const at = calls.length;
336
+ return inner(f, args, o).then((r) => {
337
+ calls[at]!.cwd = o.cwd;
338
+ return r;
339
+ });
340
+ };
341
+ const s = spec(dir);
342
+ const { dir: _dir, ...step } = s.step as Record<string, unknown>;
343
+ const result = await wave({ ...s, step: step as OpWavesSpec["step"] }, dir, exec);
344
+ expect(result.exitCode).toBe(0);
345
+ expect(calls.length).toBeGreaterThan(0);
346
+ expect(new Set(calls.map((c) => c.cwd))).toEqual(new Set([dir]));
347
+ });
348
+ });
349
+
350
+ describe("a wave's units planned at once on one provider cache (chant#4864)", () => {
351
+ /** The recorded stub with a delay per call (a plan's longest), `init` answered here, and how many inits and plans were ever in flight together. */
352
+ const slow = (calls: Call[]) => {
353
+ const inner = stub(calls);
354
+ const most = { init: 0, plan: 0 };
355
+ const now = { init: 0, plan: 0 };
356
+ const exec: TerragruntExec = async (f, args, o) => {
357
+ const cmd = args[0] === "run" ? args[args.indexOf("--") + 1] : undefined;
358
+ const kind = cmd === "init" ? "init" : cmd === "plan" ? "plan" : undefined;
359
+ if (kind) most[kind] = Math.max(most[kind], ++now[kind]);
360
+ try {
361
+ await new Promise((r) => setTimeout(r, kind === "plan" ? 120 : 20));
362
+ if (kind === "init") {
363
+ calls.push({ file: f, args: [...args], env: o.env });
364
+ return { code: 0, stdout: "", stderr: "" };
365
+ }
366
+ return await inner(f, args, o);
367
+ } finally {
368
+ if (kind) now[kind]--;
369
+ }
370
+ };
371
+ return { exec, most };
372
+ };
373
+ const parallel = (dir: string): OpWavesSpec => ({ ...spec(dir, false), parallelism: 2 });
374
+ const env = process.env.TF_PLUGIN_CACHE_DIR;
375
+ const withCache = async <T>(cache: string | undefined, fn: () => Promise<T>): Promise<T> => {
376
+ if (cache === undefined) delete process.env.TF_PLUGIN_CACHE_DIR;
377
+ else process.env.TF_PLUGIN_CACHE_DIR = cache;
378
+ try {
379
+ return await fn();
380
+ } finally {
381
+ if (env === undefined) delete process.env.TF_PLUGIN_CACHE_DIR;
382
+ else process.env.TF_PLUGIN_CACHE_DIR = env;
383
+ }
384
+ };
385
+
386
+ it("with a cache, each unit inits in its turn before its plan, and the plans run at once", async () => {
387
+ const dir = tmp();
388
+ const calls: Call[] = [];
389
+ const { exec, most } = slow(calls);
390
+ const result = await withCache(join(dir, "cache"), () => wave(parallel(dir), dir, exec));
391
+ expect(result.exitCode).toBe(0);
392
+ expect(most).toEqual({ init: 1, plan: 2 });
393
+ const ran = runAll(calls);
394
+ expect(ran.filter((r) => r[0] === "init").map((r) => r[1]).sort()).toEqual(["{./live/dev/vpc}", "{./live/prod/vpc}"]);
395
+ for (const unit of ["{./live/dev/vpc}", "{./live/prod/vpc}"]) {
396
+ const at = (cmd: string) => ran.findIndex((r) => r[0] === cmd && r[1] === unit);
397
+ expect(at("init")).toBeLessThan(at("plan"));
398
+ }
399
+ expect(calls.find((c) => c.args.includes("init"))!.args).toEqual(["run", "--all", "--non-interactive", "--no-color", "--no-filters-file", "--filter", "{./live/dev/vpc}", "--", "init", "-input=false"]);
400
+ });
401
+
402
+ it("the digest is the one a wave planned one unit at a time decides", async () => {
403
+ const decided = async (s: OpWavesSpec, dir: string) => {
404
+ const { exec } = slow([]);
405
+ return (await withCache(join(dir, "cache"), () => wave(s, dir, exec))).decision?.digest;
406
+ };
407
+ const one = tmp();
408
+ const many = tmp();
409
+ expect(await decided(parallel(many), many)).toBe(await decided(spec(one, false), one));
410
+ });
411
+
412
+ it("with no cache, the units plan at once with no init of their own", async () => {
413
+ const dir = tmp();
414
+ const calls: Call[] = [];
415
+ const { exec, most } = slow(calls);
416
+ const result = await withCache(undefined, () => wave(parallel(dir), dir, exec));
417
+ expect(result.exitCode).toBe(0);
418
+ expect(most.plan).toBe(2);
419
+ expect(runAll(calls).map((r) => r[0])).not.toContain("init");
420
+ });
421
+ });
@@ -27,6 +27,11 @@
27
27
  * A run hook's `processEnv` is laid over the environment Terragrunt runs
28
28
  * with; the units of one `run --all` must agree on it. Terragrunt has no
29
29
  * init of its own before the plan, so `afterInit` is called before it plans.
30
+ * A wave that plans its units run by run several at once (chant#4864) gives
31
+ * each its serial section (`DeployContext.serial`): with a provider cache
32
+ * (`TF_PLUGIN_CACHE_DIR`) a unit's mock check and an explicit `run --all --
33
+ * init` take their turn there, one unit at a time, and its plan runs beside
34
+ * the others'.
30
35
  * A product passes its own `exec` (`terragruntApplyCapability`) to wrap
31
36
  * Terragrunt, as terragucci adds its lock timeout and its backend bootstrap.
32
37
  *
@@ -50,6 +55,7 @@ import type { ChangeSetPart } from "@intentius/chant/change-set";
50
55
  import { computePlanDigest } from "@intentius/chant/lifecycle/plan-digest";
51
56
  import { applyTerragruntWave, defaultTerragruntExec, discoverTerragruntUnits, planTerragruntWave, TerragruntMockRefusal, type TerragruntExec } from "./run";
52
57
  import { terragruntListedWaves, terragruntWaves } from "./units";
58
+ import { initTurnKey, type InitSection } from "../init-turn";
53
59
 
54
60
  /** The step's input. Each string has `{target}`, `{op}` and `{wave}` filled per run in an op wave. */
55
61
  export interface TerragruntApplyStepInput {
@@ -217,6 +223,9 @@ const DEFAULT_WORK_DIR = ".chant/terragrunt-apply";
217
223
 
218
224
  const dirOf = (input: TerragruntApplyStepInput): string => input.dir ?? process.cwd();
219
225
 
226
+ /** The step's input with the wave's directory as its `dir` when it names none (chant#4854). */
227
+ const atDir = <T extends TerragruntApplyStepInput | undefined>(input: T, cwd: string | undefined): T => (input && input.dir === undefined && cwd !== undefined ? { ...input, dir: cwd } : input);
228
+
220
229
  /** The work directory of a `run --all` over `units`: one per set of units, so a wave and a unit of it never share one. */
221
230
  function workDirOf(input: TerragruntApplyStepInput, units: readonly string[]): string {
222
231
  const key = createHash("sha256").update(JSON.stringify([...units].sort())).digest("hex").slice(0, 16);
@@ -232,7 +241,7 @@ function withEnv(exec: TerragruntExec, env: Record<string, string> | undefined):
232
241
  /** The settings every unit of one `run --all` shares; throws when members disagree. */
233
242
  function shared(members: readonly GroupMember<TerragruntApplyStepInput>[]): { input: TerragruntApplyStepInput; units: string[]; env?: Record<string, string> } {
234
243
  if (members.length === 0) throw new Error("terragrunt-apply: a wave needs at least one unit");
235
- const inputs = members.map((m) => m.input ?? (m.step as unknown as TerragruntApplyStepInput));
244
+ const inputs = members.map((m) => atDir(m.input ?? (m.step as unknown as TerragruntApplyStepInput), m.ctx.cwd));
236
245
  for (const i of inputs) if (typeof i.unit !== "string" || i.unit === "") throw new Error("terragrunt-apply: each run names its unit");
237
246
  const first = inputs[0]!;
238
247
  const key = (i: TerragruntApplyStepInput) => JSON.stringify([i.dir, i.terragrunt, i.binary, i.parallelism, i.workDir, i.destroy === true]);
@@ -258,10 +267,13 @@ function unitPlans(parts: readonly ChangeSetPart[], units: readonly string[], wo
258
267
  return out;
259
268
  }
260
269
 
261
- async function planUnits(exec: TerragruntExec, input: TerragruntApplyStepInput, units: readonly string[], env: Record<string, string> | undefined): Promise<{ workDir: string; plans: Map<string, CapabilityPlan> }> {
270
+ async function planUnits(exec: TerragruntExec, input: TerragruntApplyStepInput, units: readonly string[], env: Record<string, string> | undefined, serial?: InitSection): Promise<{ workDir: string; plans: Map<string, CapabilityPlan> }> {
262
271
  const dir = dirOf(input);
263
272
  const workDir = workDirOf(input, units);
273
+ // chant#4864: units planned at once that share a provider cache take turns at their inits.
274
+ const cache = serial ? initTurnKey(env) : undefined;
264
275
  const planned = await planTerragruntWave({
276
+ ...(cache ? { initTurn: <T>(fn: () => Promise<T>) => serial!(cache, fn) } : {}),
265
277
  dir,
266
278
  units,
267
279
  workDir,
@@ -350,7 +362,8 @@ export function terragruntApplyCapability(deps: TerragruntApplyDeps = {}): Capab
350
362
  // A unit plans and applies as a whole, and Terragrunt takes each unit's own backend lock.
351
363
  defaultLockGrain: () => "target",
352
364
 
353
- async discoverRuns(ctx: RunDiscoveryContext, input: TerragruntApplyStepInput): Promise<DiscoveredRuns> {
365
+ async discoverRuns(ctx: RunDiscoveryContext, given: TerragruntApplyStepInput): Promise<DiscoveredRuns> {
366
+ const input = atDir(given, ctx.cwd);
354
367
  const found = await discoverTerragruntUnits({
355
368
  dir: dirOf(input),
356
369
  exec,
@@ -362,18 +375,20 @@ export function terragruntApplyCapability(deps: TerragruntApplyDeps = {}): Capab
362
375
  return { targets: waves[ctx.wave.number - 1] ?? [], waves: waves.length };
363
376
  },
364
377
 
365
- async plan(ctx: DeployContext, input: TerragruntApplyStepInput): Promise<CapabilityPlan> {
378
+ async plan(ctx: DeployContext, given: TerragruntApplyStepInput): Promise<CapabilityPlan> {
379
+ const input = atDir(given, ctx.cwd);
366
380
  if (typeof input.unit !== "string" || input.unit === "") throw new Error("terragrunt-apply: the step names no unit");
367
381
  await ctx.afterInit?.();
368
- const { plans } = await planUnits(exec, await unitInput(resolveUnit, input, input.unit), [input.unit], ctx.processEnv);
382
+ const { plans } = await planUnits(exec, await unitInput(resolveUnit, input, input.unit), [input.unit], ctx.processEnv, ctx.serial);
369
383
  return plans.get(input.unit)!;
370
384
  },
371
385
 
372
- async run(ctx: DeployContext, input: TerragruntApplyStepInput): Promise<TerragruntApplyStepOutput> {
386
+ async run(ctx: DeployContext, step: TerragruntApplyStepInput): Promise<TerragruntApplyStepOutput> {
387
+ const input = atDir(step, ctx.cwd);
373
388
  const given = ctx.plans?.[input.unit];
374
389
  const own = await unitInput(resolveUnit, input, input.unit);
375
390
  // A unit with no plan from the wave plans now and applies that plan.
376
- const workDir = isUnitArtifact(given) ? given.workDir : (await planUnits(exec, own, [input.unit], ctx.processEnv)).workDir;
391
+ const workDir = isUnitArtifact(given) ? given.workDir : (await planUnits(exec, own, [input.unit], ctx.processEnv, ctx.serial)).workDir;
377
392
  const applied = await applyUnits(exec, own, [input.unit], workDir, ctx.processEnv);
378
393
  const result = applied.results.find((r) => r.unit === input.unit);
379
394
  if (result?.status !== "succeeded") throw new Error(`terragrunt-apply: ${input.unit}: ${result?.error ?? "Terragrunt did not apply it"}`);
@@ -38,6 +38,7 @@ import {
38
38
  import {
39
39
  parseTerragruntReport,
40
40
  terragruntEnv,
41
+ terragruntInitArgs,
41
42
  terragruntUnitResults,
42
43
  terragruntWaveArgs,
43
44
  terragruntWaveParts,
@@ -171,6 +172,14 @@ export interface TerragruntWaveRunInput extends TerragruntRunOptions {
171
172
  * are never applied. Unset, a wave that would read a mock is refused.
172
173
  */
173
174
  provisional?: boolean;
175
+ /**
176
+ * The plan's turn at the provider cache (chant#4864), when units planned
177
+ * by other calls at once share it. Given, the mock check and an explicit
178
+ * `run --all -- init` of the units run inside it, and the plan after it,
179
+ * so its auto-init finds them initialised. An init that fails is left to
180
+ * the plan, which fails the unit with its report row as before.
181
+ */
182
+ initTurn?: <T>(fn: () => Promise<T>) => Promise<T>;
174
183
  }
175
184
 
176
185
  /**
@@ -315,10 +324,15 @@ export async function planTerragruntWave(input: TerragruntWaveRunInput): Promise
315
324
  const marker = join(work, PROVISIONAL_MARKER);
316
325
  // A plan or report left by an earlier attempt must not stand in for a unit that did not plan this time.
317
326
  for (const stale of [outDir, jsonOutDir, reportFile, marker]) rmSync(stale, { recursive: true, force: true });
318
- if (!input.provisional) {
319
- const reads = await checkTerragruntWaveMocks({ ...input, exec });
320
- if (reads.length > 0) throw new TerragruntMockRefusal(reads);
321
- }
327
+ const prepare = async (): Promise<void> => {
328
+ if (!input.provisional) {
329
+ const reads = await checkTerragruntWaveMocks({ ...input, exec });
330
+ if (reads.length > 0) throw new TerragruntMockRefusal(reads);
331
+ }
332
+ // chant#4864: the init the plan would run itself, in the turn; a failure is the plan's to report.
333
+ if (input.initTurn) await exec(tg, terragruntInitArgs({ units: input.units, ...(input.destroy ? { destroy: true } : {}) }), { cwd: input.dir, env: terragruntEnv(input.binary) });
334
+ };
335
+ await (input.initTurn ? input.initTurn(prepare) : prepare());
322
336
  mkdirSync(work, { recursive: true });
323
337
  // Written before planning, so a plan cut short still never applies.
324
338
  if (input.provisional) writeFileSync(marker, "planned before its upstream applied; never applied\n");
@@ -72,6 +72,28 @@ export function terragruntWaveArgs(input: TerragruntWaveArgsInput): string[] {
72
72
  return args;
73
73
  }
74
74
 
75
+ /**
76
+ * Arguments to `terragrunt` that init a wave's units with the same filters
77
+ * as its plan (chant#4864), for an init that takes its turn at a shared
78
+ * provider cache before the plan. Throws on an empty wave, as the plan does.
79
+ */
80
+ export function terragruntInitArgs(input: { units: readonly string[]; destroy?: boolean }): string[] {
81
+ if (input.units.length === 0) throw new Error("a Terragrunt wave needs at least one unit; with no path filter run --all would run every unit");
82
+ const units = [...new Set(input.units)].sort();
83
+ return [
84
+ "run",
85
+ "--all",
86
+ "--non-interactive",
87
+ "--no-color",
88
+ "--no-filters-file",
89
+ ...units.flatMap((u) => ["--filter", unitPathFilter(u)]),
90
+ ...(input.destroy ? ["--filter-allow-destroy"] : []),
91
+ "--",
92
+ "init",
93
+ "-input=false",
94
+ ];
95
+ }
96
+
75
97
  /**
76
98
  * The environment a wave adds: `TG_TF_PATH` when the project names a binary,
77
99
  * nothing otherwise, so Terragrunt's own choice (tofu, then terraform, or a