@intentius/chant-lexicon-terraform 0.124.51 → 0.124.54

package/src/components/capability-plugin.ts

@@ -38,9 +38,27 @@
  * when both roots are in the change.
  *
  * A root whose `vars` read an output not known yet (`DeployContext.unresolvedInputs`)
- * is not planned: terraform takes no unknown variable. Its member gets a hole
- * naming the roots it waits on and no artifact, and the run that applies it
- * plans it once their outputs are known.
+ * cannot be planned as it stands: terraform takes no unknown variable. It is
+ * previewed instead (#4463, `./placeholder-preview.ts`): planned with a
+ * stand-in per variable that holds no value, typed from the root's
+ * `variable` block, with every attribute and output a stand-in drives marked
+ * unknown after apply. The member lists the variables under `placeholders`,
+ * and its artifact is the preview's shape, never a plan file. The apply plans
+ * the root again once the upstream applied, and core covers that round with
+ * the first approval when its shape is within the preview's; a `run` handed
+ * the shape itself holds the fresh plan to it. When no preview can be made
+ * (an undeclared variable, a `count` or `for_each` over a stand-in, a
+ * placeholder plan that fails) the member stays a hole naming the roots it
+ * waits on, and the hole's reason says why there is no preview.
+ *
+ * ## The plan's shape (#4436)
+ *
+ * `plan` also answers `shape`, `terraformPlanShape` over `show -json`: each
+ * resource the plan changes with its action, type, replacement, the
+ * attributes that differ and the ones unknown after apply. Core's
+ * `coveredRound` compares it with the shape of a later plan of the root. A
+ * previewed root's shape lists the upstreams it read as unknown inputs; a
+ * root that stays a hole has no plan, so no shape, and stays uncovered.
  *
  * ## choudoufu roots in a gated wave (#3459)
  *
@@ -63,13 +81,44 @@
  *
  * chant's waves stay the order. Each root is its own one-root set, so
  * choudoufu's wave over it is always wave 1.
+ *
+ * ## Composed runs (#4447)
+ *
+ * With `terraform.compose.enabled`, a group core forms of `terraform-apply`
+ * components that read each other (ws-107) plans as one plan and applies as
+ * one apply of a composed root, and a member's read of another member's new
+ * output is not a hole: the apply passes the value on. `groupReads`,
+ * `groupFallback`, `planGroup` and `runGroup` are in `./compose-group.ts`.
+ *
+ * ## A choudoufu member's outputs, for another member (#4455)
+ *
+ * The plugin also contributes {@link choudoufuOutputSource} to core's
+ * `member-outputs` verb (`CapabilityPlugin.engineOutputSources`), which is
+ * how a chant member's parameters read a choudoufu member of the same
+ * workspace: `choudoufu output -json` in the member's directory for the
+ * applied values, and `live-plan-set -json` over it for the planned ones,
+ * read the way `plan` reads a root's (`terraformPlannedOutputs`). The member
+ * is not a `terraform.roots` entry of the reading project; its directory
+ * comes from the workspace declaration.
  */
 
+import { rmSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import type { Capability, DeployContext, CapabilityPlan, PlannedOutputs } from "@intentius/chant/components/capability";
+import { loadChantConfigUpward } from "@intentius/chant/config";
+import {
+  PlanShapeMovedError,
+  planShapeDifferences,
+  planShapeDigest,
+  type PlanShapeArtifact,
+} from "@intentius/chant/components/covered-round";
 import type { ChangeSetPart } from "@intentius/chant/change-set";
 import { computePlanDigest } from "@intentius/chant/lifecycle/plan-digest";
 import { ownPackageVersion, type CapabilityPlugin } from "@intentius/chant/components/capability-plugin";
+import type { EngineMemberRef, EngineOutputSource, EnginePlannedOutputs } from "@intentius/chant/components/verbs/member-outputs";
 import {
+  choudoufuMemberOutputs,
+  choudoufuMemberPlanSet,
   choudoufuPlanSet,
   choudoufuWaveApply,
   choudoufuWaveRoot,
@@ -78,6 +127,13 @@ import {
   terraformOutputs,
   terraformPlan,
 } from "../op/activities/terraform";
+import { terraformPlanShape } from "../plan-shape";
+import { terraformChangeSetPart } from "../change-set";
+import type { TerraformConfig } from "../config";
+import { rootTexts } from "../compose/hcl";
+import { declaredVariables } from "../compose/root";
+import { terraformComposeGroup } from "./compose-group";
+import { markPlaceholderDriven, placeholderVars, waitingHole } from "./placeholder-preview";
 
 /** The `terraform-apply` step's input. */
 export interface TerraformApplyStepInput {
@@ -116,10 +172,14 @@ function requireRoot(input: TerraformApplyStepInput): void {
 async function planRoot(input: TerraformApplyStepInput, targets?: string[]): Promise<PlannedRoot & { changeSet?: ChangeSetPart; json?: unknown }> {
   requireRoot(input);
   await terraformInit(rootArgs(input));
+  // One `show -json` per plan (#4479): its document is the digest, the change
+  // set, the planned outputs and the shape. Nothing here reads the
+  // `-no-color` render, so the plan skips it.
   const planned = await terraformPlan({
     ...rootArgs(input),
     ...(input.vars ? { vars: input.vars } : {}),
     ...(targets && targets.length > 0 ? { targets } : {}),
+    text: false,
   });
   return {
     planFile: planned.planFile,
@@ -164,12 +224,39 @@ export function terraformPlannedOutputs(json: unknown): PlannedOutputs {
 }
 
 /**
- * The member of a root whose `vars` read outputs not known yet (#3999): no
- * plan, a hole naming whose outputs it waits on, and a digest over the root,
- * the vars as resolved and those names.
+ * What a choudoufu member's set plan says about its outputs (#4455), for
+ * `member-outputs`: {@link terraformPlannedOutputs} of the embedded plan,
+ * chant's digest of that plan, and whether it has changes to apply.
+ */
+export function choudoufuMemberPlannedOutputs(set: { planDigest: string; changed: boolean; json: unknown }): EnginePlannedOutputs {
+  return { ...terraformPlannedOutputs(set.json), planDigest: set.planDigest, changes: set.changed };
+}
+
+const memberArgs = (ref: EngineMemberRef) => ({ member: ref.member, dir: ref.dir, projectRoot: ref.workspaceRoot });
+
+/**
+ * The choudoufu engine's member outputs (#4455): what core's
+ * `member-outputs` reads a workspace member of kind `choudoufu` through.
+ * Applied values from the member's root outputs (the record store, for a
+ * live estate); planned values from its one-root set plan.
+ */
+export const choudoufuOutputSource: EngineOutputSource = {
+  kinds: ["choudoufu"],
+  applied: (_ctx, ref) => choudoufuMemberOutputs(memberArgs(ref)),
+  planned: async (_ctx, ref) => choudoufuMemberPlannedOutputs(await choudoufuMemberPlanSet(memberArgs(ref))),
+};
+
+const resolvedVars = (input: TerraformApplyStepInput): Record<string, unknown> =>
+  Object.fromEntries(Object.entries(input.vars ?? {}).map(([k, v]) => [k, v === undefined ? null : v]));
+
+/**
+ * The member of a root whose `vars` read outputs not known yet and that
+ * could not be previewed (#3999, #4463): no plan, a hole naming whose outputs
+ * it waits on and, when a preview was tried, why it was not possible, and a
+ * digest over the root, the vars as resolved and those names.
  */
-function waitingRoot(ctx: DeployContext, input: TerraformApplyStepInput, unresolved: string[]): CapabilityPlan {
-  const vars = Object.fromEntries(Object.entries(input.vars ?? {}).map(([k, v]) => [k, v === undefined ? null : v]));
+function waitingRoot(ctx: DeployContext, input: TerraformApplyStepInput, unresolved: string[], noPreview?: string): CapabilityPlan {
+  const vars = resolvedVars(input);
   const planDigest = computePlanDigest("terraform-apply-unresolved", { env: ctx.env, root: input.root, vars, unresolved });
   const changeSet: ChangeSetPart = {
     member: {
@@ -178,13 +265,104 @@ function waitingRoot(ctx: DeployContext, input: TerraformApplyStepInput, unresol
       planner: "chant",
       status: "planned",
       planDigest,
-      holes: [{ address: input.root, type: "terraform_root", reason: `inputs from ${unresolved.join(", ")}; planned again once they apply` }],
+      holes: [waitingHole(input.root, unresolved, noPreview)],
     },
     entries: [],
   };
   return { member: input.root, planDigest, changeSet, ...(ctx.targets?.length ? { targets: ctx.targets } : {}) };
 }
 
+/** The plan file a preview writes, removed once it is read: a plan made with stand-ins must never apply. */
+export const PREVIEW_PLAN_FILE = ".chant-preview.tfplan";
+
+/** What a previewed root hands `run` (#4463): the shape its plan had with stand-in values. Never a plan file. */
+export interface TerraformShapeArtifact extends PlanShapeArtifact {
+  kind: "terraform-apply-shape";
+}
+
+function isShapeArtifact(value: unknown): value is TerraformShapeArtifact {
+  const v = value as Partial<TerraformShapeArtifact> | undefined;
+  return v?.kind === "terraform-apply-shape" && typeof v.digest === "string" && typeof v.shape === "object" && v.shape !== null;
+}
+
+/** The root's directory, as `terraform.roots` names it from the nearest `chant.config.*`, and the configured binary. */
+async function rootDir(input: TerraformApplyStepInput): Promise<{ dir: string; binary?: string } | undefined> {
+  const start = resolve(input.cwd ?? process.cwd());
+  const { config, configPath } = await loadChantConfigUpward(start);
+  const terraform = (config as { terraform?: TerraformConfig }).terraform;
+  const dir = terraform?.roots?.[input.root]?.dir;
+  if (dir === undefined) return undefined;
+  return { dir: resolve(configPath ? dirname(configPath) : start, dir), ...(terraform?.binary ? { binary: terraform.binary } : {}) };
+}
+
+const firstLines = (err: unknown): string => {
+  const text = (err instanceof Error ? err.message : String(err)).trim();
+  const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
+  return lines.slice(0, 4).join(" ").slice(0, 600);
+};
+
+/**
+ * Preview a root whose `vars` read outputs not known yet (#4463): plan it
+ * with a typed stand-in per such variable, mark what the stand-ins drive
+ * unknown, and answer the change set, the planned outputs and the shape of
+ * that plan, with the shape as the artifact. The plan file is removed. When
+ * no preview can be made, the member stays a hole saying why.
+ */
+async function previewRoot(ctx: DeployContext, input: TerraformApplyStepInput, unresolved: string[], targets?: string[]): Promise<CapabilityPlan> {
+  const root = await rootDir(input);
+  if (root === undefined) return waitingRoot(ctx, input, unresolved, `no root named "${input.root}" in terraform.roots to read its variables from`);
+  const { dir } = root;
+  // #4457: the same files the binary reads (`.tofu` only for tofu and choudoufu).
+  const stand = placeholderVars(input.vars, declaredVariables(rootTexts(dir, root.binary)));
+  if (typeof stand === "string") return waitingRoot(ctx, input, unresolved, stand);
+  let planned: Awaited<ReturnType<typeof terraformPlan>>;
+  try {
+    await terraformInit(rootArgs(input));
+    planned = await terraformPlan({
+      ...rootArgs(input),
+      vars: stand.vars,
+      planFile: PREVIEW_PLAN_FILE,
+      ...(targets ? { targets } : {}),
+      text: false,
+    });
+  } catch (err) {
+    return waitingRoot(ctx, input, unresolved, `the plan with stand-in values failed: ${firstLines(err)}`);
+  } finally {
+    rmSync(join(dir, PREVIEW_PLAN_FILE), { force: true });
+  }
+  const marked = markPlaceholderDriven(planned.json, stand.placeholders);
+  if ("refused" in marked) return waitingRoot(ctx, input, unresolved, marked.refused);
+  const placeholders = stand.placeholders.map((name) => `var.${name}`);
+  const shape = terraformPlanShape(marked.plan, input.root, { unknownInputs: unresolved });
+  const artifact: TerraformShapeArtifact = { kind: "terraform-apply-shape", member: input.root, shape, digest: planShapeDigest(shape) };
+  const part = terraformChangeSetPart({
+    member: input.root,
+    plan: marked.plan,
+    ...(planned.changeSet?.member.planner ? { planner: planned.changeSet.member.planner } : {}),
+    ...(planned.changeSet?.member.scope ? { scope: planned.changeSet.member.scope } : {}),
+  });
+  // The stand-ins' values are not real, so the digest is over the preview's shape and entries, never the raw plan.
+  const planDigest = computePlanDigest("terraform-apply-preview", {
+    env: ctx.env,
+    root: input.root,
+    vars: resolvedVars(input),
+    unresolved,
+    placeholders,
+    shape: artifact.digest,
+    entries: part.entries,
+  });
+  const changeSet: ChangeSetPart = { ...part, member: { ...part.member, planDigest, placeholders } };
+  return {
+    member: input.root,
+    planDigest,
+    artifact,
+    changeSet,
+    outputs: terraformPlannedOutputs(marked.plan),
+    shape,
+    ...(targets ? { targets } : {}),
+  };
+}
+
 /**
  * choudoufu refused the plan `run` was handed by its wave (its exit 3):
  * the live system moved since the plan, or the plan names another estate.
@@ -255,8 +433,8 @@ export const terraformApplyCapability: Capability<TerraformApplyStepInput, Terra
   async plan(ctx: DeployContext, input: TerraformApplyStepInput): Promise<CapabilityPlan> {
     requireRoot(input);
     const unresolved = [...(ctx.unresolvedInputs ?? [])].sort();
-    if (unresolved.length > 0) return waitingRoot(ctx, input, unresolved);
     const targets = ctx.targets && ctx.targets.length > 0 ? ctx.targets : undefined;
+    if (unresolved.length > 0) return previewRoot(ctx, input, unresolved, targets);
     // A targeted root plans with `plan -target`, never as a choudoufu one-root set.
     const wave = targets ? { wave: false } : await choudoufuWaveRoot({ ...rootArgs(input), ...(input.vars ? { vars: input.vars } : {}) });
     if (wave.wave) {
@@ -265,7 +443,7 @@ export const terraformApplyCapability: Capability<TerraformApplyStepInput, Terra
         planDigest: set.planDigest,
         choudoufu: { planSetFile: set.planSetFile, setDigest: set.setDigest },
       };
-      return { member: input.root, planDigest: set.planDigest, artifact, changeSet: set.changeSet, outputs: terraformPlannedOutputs(set.json) };
+      return { member: input.root, planDigest: set.planDigest, artifact, changeSet: set.changeSet, outputs: terraformPlannedOutputs(set.json), shape: terraformPlanShape(set.json, input.root) };
     }
     const { changeSet, json, ...planned } = await planRoot(input, targets);
     return {
@@ -274,6 +452,7 @@ export const terraformApplyCapability: Capability<TerraformApplyStepInput, Terra
       artifact: planned,
       ...(changeSet ? { changeSet } : {}),
       outputs: terraformPlannedOutputs(json),
+      shape: terraformPlanShape(json, input.root),
       ...(targets ? { targets } : {}),
     };
   },
@@ -284,9 +463,28 @@ export const terraformApplyCapability: Capability<TerraformApplyStepInput, Terra
     await terraformInit(rootArgs(input));
     return terraformOutputs(rootArgs(input));
   },
+  // #4447 (ws-107): a group core forms of components that read each other plans and applies as one composed run
+  // when `terraform.compose.enabled` is set (./compose-group.ts). Delegates, so the methods are read when called.
+  groupReads: (ctx, members) => terraformComposeGroup.groupReads(ctx, members),
+  groupFallback: (ctx, members) => terraformComposeGroup.groupFallback(ctx, members),
+  planGroup: (ctx, members) => terraformComposeGroup.planGroup(ctx, members),
+  runGroup: (ctx, run, members) => terraformComposeGroup.runGroup(ctx, run, members),
   async run(ctx: DeployContext, input: TerraformApplyStepInput): Promise<TerraformApplyStepOutput> {
     const fromWave = ctx.plans?.[input.root];
     if (isChoudoufuWavePlan(fromWave)) return runChoudoufuWave(ctx, input, fromWave);
+    if (isShapeArtifact(fromWave)) {
+      // A root previewed with stand-ins (#4463): plan it now with the real
+      // values, and apply that plan only when it is within the preview's shape.
+      const fresh = await planRoot(input, ctx.targets);
+      const differences = planShapeDifferences(fromWave.shape, terraformPlanShape(fresh.json, input.root));
+      if (differences.length > 0) throw new PlanShapeMovedError(input.root, differences);
+      const applied = await terraformApply({ ...rootArgs(input), planFile: fresh.planFile });
+      if (!applied.applied) {
+        throw new Error(`terraform-apply: root "${input.root}" refused the plan it was given (${applied.refused ?? "refused"}). ${applied.refusal ?? ""}`);
+      }
+      const outputs = await terraformOutputs(rootArgs(input));
+      return { root: input.root, planDigest: fresh.planDigest, fromWavePlan: false, outputs };
+    }
     const planned: PlannedRoot = isPlannedRoot(fromWave)
       ? fromWave
       : await planRoot(input, ctx.targets).then(({ planFile, planDigest }) => ({ planFile, planDigest }));
@@ -320,4 +518,7 @@ export const terraformCapabilityPlugin: CapabilityPlugin = {
   families(): Record<string, readonly string[]> {
     return TERRAFORM_VERB_FAMILIES;
   },
+  engineOutputSources(): EngineOutputSource[] {
+    return [choudoufuOutputSource];
+  },
 };

package/src/components/change-set-graph.e2e.test.ts

@@ -16,7 +16,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, test } from "vitest";
 import { composeChangeSet } from "@intentius/chant/change-set";
-import { buildGraphIr } from "@intentius/chant/graph-ir";
+import { buildGraphIr, withInstanceNodes } from "@intentius/chant/graph-ir";
 import { readGraphIds, storeGateChangeSet } from "@intentius/chant/lifecycle/gate-change-set";
 import { readStoredChangeSet } from "@intentius/chant/lifecycle/change-set-store";
 import { terraformChangeSetPart } from "../change-set";
@@ -59,10 +59,11 @@ describe("a terraform change set mapped onto the spawned chant graph", () => {
     const spawned = await readGraphIds(dir);
     if ("error" in spawned) throw new Error(spawned.error);
     const rendered = await renderTerraformRoots({ projectRoot: dir, roots: { estate: { dir: "estate" } } });
-    expect(buildGraphIr(rendered.entities).nodes.map((n) => n.id).sort()).toEqual([...spawned.ids].sort());
+    // `chant graph` adds the instance nodes of `count = 2` (#4454).
+    expect(withInstanceNodes(buildGraphIr(rendered.entities), rendered.entities).nodes.map((n) => n.id).sort()).toEqual([...spawned.ids].sort());
   }, TIMEOUT);
 
-  test("each entry is stored on the block the graph names, and a deleted block has no node", async () => {
+  test("each entry is stored on the node the graph names, an instance on its instance node, and a deleted block has no node", async () => {
     const plan = JSON.parse(readFileSync(join(FIXTURES, "terraform.plan.json"), "utf8"));
     const doc = composeChangeSet([terraformChangeSetPart({ member: "estate", plan })]);
     const head = git("rev-parse", "HEAD");
@@ -72,13 +73,18 @@ describe("a terraform change set mapped onto the spawned chant graph", () => {
     const read = await readStoredChangeSet(doc.digest, { cwd: dir });
     if (read.status !== "found") throw new Error(read.status);
     expect(read.document.entries.map((e) => [e.address, e.node])).toEqual([
-      ["module.app.terraform_data.instance[0]", "estate/module.app/terraform_data.instance"],
-      ["module.app.terraform_data.instance[1]", "estate/module.app/terraform_data.instance"],
+      ["module.app.terraform_data.instance[0]", "estate/module.app/terraform_data.instance[0]"],
+      ["module.app.terraform_data.instance[1]", "estate/module.app/terraform_data.instance[1]"],
       ["terraform_data.config", "estate/terraform_data.config"],
       ["terraform_data.keep", "estate/terraform_data.keep"],
       ["terraform_data.old", null],
       ["terraform_data.secret", "estate/terraform_data.secret"],
       ["terraform_data.worker", "estate/terraform_data.worker"],
     ]);
+    // The instances name their block node, the rollup (#4454).
+    expect(read.document.entries.filter((e) => e.instanceOf).map((e) => e.instanceOf)).toEqual([
+      "estate/module.app/terraform_data.instance",
+      "estate/module.app/terraform_data.instance",
+    ]);
   }, TIMEOUT);
 });