@intentius/chant-lexicon-terraform 0.57.0 → 0.59.0

package/src/composites/terraform-adopt-op.ts

@@ -0,0 +1,202 @@
+/**
+ * TerraformAdoptOp composite (#2105): claim the live resources a choudoufu
+ * estate's configuration already describes.
+ *
+ * The reconcile position on the lifecycle dial, for the one backend that has
+ * a typed path from live back to the configuration. chant #2089 deferred
+ * reconcile for the terraform lexicon because stock Terraform has none: a live
+ * resource nobody has imported is knowable only as HCL somebody writes, and
+ * generating that is a different problem. choudoufu changes the question
+ * rather than answering the old one. Ownership there is two tags on the
+ * resource (`live/MARKERS.md`), `live-plan` already names every live resource
+ * that exactly matches a declared block and carries no marker, and it prints
+ * the two values that would claim it. Cloud-to-code here is "claim what the
+ * configuration already describes" — a tag write, not a regeneration.
+ *
+ * Phases: Check, Ledger, Gate, Adopt.
+ *
+ *   - **Check** runs `live-check`, which makes no cloud calls and says whether
+ *     this configuration can move under markers at all. It runs first because
+ *     a root that `live-check` refuses will produce an adoption ledger that
+ *     reads as authoritative and is not, and the cheapest place to find that
+ *     out is before the estate-wide sweep.
+ *   - **Ledger** runs `live-plan` with `adoptionOnly`, so the run's human half
+ *     is choudoufu's own adoption ledger (GitHub issue #587) and its machine
+ *     half is the bound/omissions/unowned document (#788). The ledger is the
+ *     approval subject; the document is what the Adopt step acts on.
+ *   - **Gate** always. Adoption writes tags onto resources this estate does
+ *     not yet own, which is the moment the estate's boundary moves, and no
+ *     reading of that is routine enough to skip. There is no `gate: "never"`
+ *     here, unlike `TerraformApplyOp`, so every run stops at the gate until
+ *     someone has approved it. A gate is a fact on the gate ledger (#2119):
+ *     `chant run` records that the run is waiting, ends with status `gated`
+ *     and exits 3, `chant approve <op> <gate>` writes the resolution, and the
+ *     next run reads it and walks through to the Adopt phase.
+ *   - **Adopt** writes the markers. See `choudoufuAdopt`'s own doc for the
+ *     mechanism and why it is the tag write rather than choudoufu's `adopt`
+ *     policy verb.
+ *
+ * ## The adopt mechanism, and why it is not the policy verb
+ *
+ * choudoufu's `policy` block has an `adopt` verb for the declared-but-unmarked
+ * quadrant, and its migration guide names `policy { declared_untagged =
+ * "adopt" }` as the way to claim a whole estate at once. That is the smaller
+ * write, and #2105 tried to reach it from outside the configuration. It is not
+ * reachable: a `live` block in a terraform override file is dropped in silence
+ * (choudoufu's `Module.mergeFile` has no `Lives` case), a second primary file
+ * carrying one is refused as a duplicate, and a sidecar beside an in-block
+ * form is refused as two sources of truth. What is left is editing the root's
+ * own checked-in live configuration around an apply, which leaves an estate
+ * declaring "adopt everything declared and unmarked" if the run dies between
+ * the two edits.
+ *
+ * So the mechanism is the tag write, which choudoufu's migration guide calls
+ * the whole contract: "There is no `choudoufu adopt` command and no need for
+ * one. Two tags is the whole contract, so any tool that writes two tags can
+ * adopt a resource." Each write runs the paste-ready command choudoufu itself
+ * printed in the ledger, carrying its own provider configuration's region and
+ * endpoint. `choudoufuAdopt` has the measurements behind that ruling.
+ *
+ * ## Ambiguity is reported, never resolved
+ *
+ * Two live resources at one declared identity are two `unowned` entries with
+ * the same address, and no single tag write claims that address. Those
+ * candidates never enter the adoptable set; the Adopt step is handed them
+ * separately so the Op's own result names what it declined and why, beside
+ * what it wrote.
+ *
+ * ## Compensation
+ *
+ * Refused without a command, the same stance `TerraformApplyOp` takes.
+ * Un-adopting is `untag`: dropping this estate's marker from a live resource,
+ * which is a decision about the estate rather than something chant can
+ * synthesize. It is also not always the right undo — a resource that was
+ * already carrying the markers before this run is not one this run adopted.
+ *
+ * @example
+ * ```typescript
+ * import { TerraformAdoptOp } from "@intentius/chant-lexicon-terraform";
+ *
+ * export const { op } = TerraformAdoptOp({ name: "estate-adopt", root: "estate" });
+ * ```
+ */
+
+import { Op, phase, gate, activity, OpResource } from "@intentius/chant/op";
+import {
+  choudoufuLiveCheck as checkStep,
+  choudoufuLivePlan as ledgerStep,
+  choudoufuAdopt as adoptStep,
+} from "../op/builders";
+
+export interface TerraformAdoptOpConfig {
+  /** Op name (kebab-case). `gateName` defaults to `approve-<name>`. */
+  name: string;
+  /** Key into the project's `terraform.roots`. Must be a live root: choudoufu, with a declared estate. */
+  root: string;
+  /**
+   * The estate whose markers this run looks for. Omitted, `choudoufuLivePlan`
+   * auto-detects it from the root's own `live` block or `estate.chdf.hcl`
+   * sidecar, which is the usual case.
+   */
+  estate?: string;
+  /** The gate's name. Default: `approve-<name>`, as `TerraformApplyOp` does. */
+  gateName?: string;
+  /**
+   * @deprecated The gate's name was `signalName` through 0.58.0 and is
+   * `gateName` since #2202, matching `TerraformApplyOp`. Removed in 0.60.0.
+   */
+  signalName?: string;
+  /** How long a recorded pending gate stays valid, as a duration string. Default: core's own (48h). */
+  gateTimeout?: string;
+  /** Override the gate description shown to the approver. */
+  gateDescription?: string;
+  /**
+   * Directory each step starts the `chant.config.*` search from, which is what
+   * `terraform.roots` and the root's relative `dir` resolve against. Default:
+   * the running process's cwd.
+   */
+  cwd?: string;
+  /**
+   * Saga-style rollback on a failed adoption, run as an `onFailure` phase.
+   *
+   * Refused without a command. Undoing an adoption is `untag` — removing this
+   * estate's marker from a live resource — which is a decision about the
+   * estate, and one this Op cannot even scope correctly: a resource that
+   * already carried the markers is not one this run claimed. So `true`, or an
+   * object with no `command`, throws when the Op is built, naming the Op,
+   * rather than warning at the moment a rollback is already wanted.
+   */
+  compensate?: boolean | { command?: string };
+}
+
+export interface TerraformAdoptOpResources {
+  /** Op resource. Generates the Check/Ledger/Gate/Adopt workflow. */
+  op: InstanceType<typeof OpResource>;
+}
+
+export function TerraformAdoptOp(config: TerraformAdoptOpConfig): TerraformAdoptOpResources {
+  const compensateCommand = typeof config.compensate === "object" ? config.compensate.command : undefined;
+  if (config.compensate !== undefined && config.compensate !== false && compensateCommand === undefined) {
+    throw new Error(
+      `TerraformAdoptOp "${config.name}": compensate is enabled, but there is no automatic undo for an ` +
+        `adoption — un-adopting is untag, removing this estate's ownership marker from a live resource, ` +
+        `which is a decision about the estate rather than something chant can synthesize, and which this ` +
+        `Op cannot scope for you (a resource that already carried the markers is not one this run ` +
+        `claimed). Either supply compensate: { command: "..." } with an undo of your own, or set ` +
+        `compensate: false.`,
+    );
+  }
+
+  const where = config.cwd ? { cwd: config.cwd } : {};
+  const estate = config.estate ? { estate: config.estate } : {};
+
+  const check = checkStep(config.root, { ...where });
+  check.outcomeAttribute = { name: "Refused", from: "refused" };
+
+  // `id` is what makes `.out` legal. The Adopt step below reads two fields of
+  // this one: the adoptable matches, and the contested candidates it must
+  // report and never write.
+  const ledger = ledgerStep(config.root, { ...where, ...estate, adoptionOnly: true, id: "ledger" });
+  ledger.outcomeAttribute = [
+    { name: "Adoptable", from: "adoptable" },
+    { name: "Ambiguous", from: "ambiguous" },
+  ];
+
+  const adopt = adoptStep(config.root, {
+    ...where,
+    adoptions: ledger.out.adoptions,
+    contested: ledger.out.contested,
+  });
+  adopt.outcomeAttribute = { name: "Adopted", from: "adoptedCount" };
+
+  const op = Op({
+    name: config.name,
+    overview: `Adopt the live resources the "${config.root}" choudoufu estate already declares`,
+    labels: {
+      Adopt: "true",
+      TerraformRoot: config.root,
+      TerraformMode: "live",
+    },
+    phases: [
+      phase("Check", [check]),
+      phase("Ledger", [ledger]),
+      phase("Gate", [
+        gate(config.gateName ?? config.signalName ?? `approve-${config.name}`, {
+          ...(config.gateTimeout ? { timeout: config.gateTimeout } : {}),
+          description:
+            config.gateDescription ??
+            `Approve adopting live resources into the "${config.root}" estate. The Ledger phase's ` +
+              `adoption ledger is what is being approved: every resource it lists as adoptable gets this ` +
+              `estate's tofu-estate and tofu-address tags written onto it. The Adoptable and Ambiguous ` +
+              `run outcomes from that phase are its counts; an ambiguous address is never adopted.`,
+        }),
+      ]),
+      phase("Adopt", [adopt]),
+    ],
+    ...(compensateCommand
+      ? { onFailure: [phase("Rollback", [activity("shellCmd", { cmd: compensateCommand })])] }
+      : {}),
+  });
+
+  return { op };
+}

package/src/composites/terraform-apply-op.acceptance.test.ts

@@ -3,7 +3,8 @@
  *
  * Everything else in this lexicon's Op suite proves the shape of what chant
  * emits. This proves the shape runs: build a `TerraformApplyOp` with
- * `gate: "never"` (the local executor refuses any Op containing a gate), hand
+ * `gate: "never"` (a gate ends a local run as `gated` since #2119, so an
+ * acceptance run that must reach Apply declares none), hand
  * it to `runOpLocally` with the activities the registry resolves by
  * convention, and let a real `terraform` — or `tofu` — init, plan and apply
  * the #2083 `with-backend` fixture root into a temp directory. Two
@@ -19,17 +20,29 @@
  *
  * Gating copied from `lexicons/k3s/src/serializer.acceptance.test.ts`, which
  * skips the same way when Docker is absent.
+ *
+ * `../__fixtures__/ACCEPTANCE.md` records what both blocks in this file have
+ * last passed against, with the binary version and the date (#2220). Update
+ * it in the same change as a run.
  */
 
 import { execSync } from "node:child_process";
 import { lookup } from "node:dns/promises";
-import { cpSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, describe, expect, it } from "vitest";
 import { loadActivities, loadProfiles, runOpLocally, type OpConfig } from "@intentius/chant/op";
 import { TerraformApplyOp } from "./terraform-apply-op";
-import { terraformShow } from "../op/activities/terraform";
+import {
+  MIN_CHOUDOUFU_VERSION,
+  isOlderVersion,
+  parseChoudoufuVersion,
+  terraformApply,
+  terraformInit,
+  terraformPlan,
+  terraformShow,
+} from "../op/activities/terraform";
 
 function onPath(cmd: string): boolean {
   try {
@@ -113,7 +126,7 @@ describe.skipIf(skipReason !== "")(
           await loadProfiles(),
         );
 
-        expect(result.ok).toBe(true);
+        expect(result.status).toBe("ok");
         expect(result.records.map((r) => `${r.phase}:${r.status}`)).toEqual([
           "Init:ok",
           "Plan:ok",
@@ -121,7 +134,7 @@ describe.skipIf(skipReason !== "")(
         ]);
 
         // The plan proposed two creates, and said so through the outcome the
-        // Plan step surfaces as the `Changed` search attribute.
+        // Plan step surfaces as the `Changed` run outcome.
         const plan = result.records.find((r) => r.phase === "Plan")!;
         expect(plan.outcome).toEqual({ name: "Changed", value: true });
 
@@ -136,3 +149,173 @@ describe.skipIf(skipReason !== "")(
     );
   },
 );
+
+/**
+ * `TerraformApplyOp` on a live root, against choudoufu's own pinned emulator
+ * (#2106 follow-up). Two suites, gated the same way
+ * `../op/activities/choudoufu.acceptance.test.ts` is, plus a version floor:
+ *
+ *   - no `choudoufu` on PATH,
+ *   - a `choudoufu` older than {@link MIN_CHOUDOUFU_VERSION}: v0.13.0 shipped
+ *     the approval artifact
+ *     ([choudoufu #878](https://github.com/INTENTIUS/choudoufu/issues/878),
+ *     PR 889), before which `plan -out` was refused under a live block and
+ *     this Op could not be built the way it is built now, and v0.14.0 moved
+ *     the floor again for the document the other two Ops read
+ *     ([choudoufu #894](https://github.com/INTENTIUS/choudoufu/issues/894)),
+ *   - `CHOUDOUFU_EMULATOR_ENDPOINT` unset (bring up choudoufu's `just smoke`
+ *     docker compose stack and export `http://localhost:<mapped port>`).
+ *
+ * The #894 clause the first version of this suite carried is gone, and stayed
+ * gone: that issue was about `live-plan -json`, which the apply Op does not
+ * run at all, since the plan half is the stock `plan -out` path.
+ *
+ * The first test is the happy path end to end through `runOpLocally`. The
+ * second is the refusal, and it runs the activities directly rather than
+ * through the Op, because the world has to move between the Plan step and the
+ * Apply step and there is no seam inside a running Op to do that from.
+ */
+
+function versionOf(binary: string): string | undefined {
+  try {
+    return parseChoudoufuVersion(execSync(`${binary} version`, { encoding: "utf8" }));
+  } catch {
+    return undefined;
+  }
+}
+
+const LIVE_FIXTURE = join(import.meta.dirname, "..", "__fixtures__", "live");
+
+/** A throwaway project holding a copy of the live fixture as its one root. */
+function liveProject(): string {
+  const dir = mkdtempSync(join(tmpdir(), "chant-choudoufu-apply-accept-"));
+  workspaces.push(dir);
+  cpSync(LIVE_FIXTURE, join(dir, "root"), { recursive: true });
+  writeFileSync(
+    join(dir, "chant.config.json"),
+    JSON.stringify(
+      { lexicons: ["terraform"], terraform: { binary: "choudoufu", roots: { estate: { dir: "./root" } } } },
+      null,
+      2,
+    ),
+  );
+  return dir;
+}
+
+/** The estate-wide marker sweep configures the AWS provider whatever the root declares. */
+function pointAtTheEmulator(endpoint: string): void {
+  process.env.AWS_ENDPOINT_URL ??= endpoint;
+  process.env.AWS_ACCESS_KEY_ID ??= "choudoufu-emulator";
+  process.env.AWS_SECRET_ACCESS_KEY ??= "choudoufu-emulator";
+  process.env.AWS_DEFAULT_REGION ??= "us-east-1";
+}
+
+const choudoufuVersion = onPath("choudoufu") ? versionOf("choudoufu") : undefined;
+const emulatorEndpoint = process.env.CHOUDOUFU_EMULATOR_ENDPOINT;
+
+const liveSkipReason: string = !onPath("choudoufu")
+  ? "no choudoufu binary on PATH"
+  : choudoufuVersion === undefined
+    ? "the choudoufu on PATH reports no release version (a dev build), so the approval artifact cannot be assumed"
+    : isOlderVersion(choudoufuVersion, MIN_CHOUDOUFU_VERSION)
+      ? `choudoufu ${choudoufuVersion} is older than v${MIN_CHOUDOUFU_VERSION}, the lexicon's floor (choudoufu #878's approval artifact in v0.13.0, #894's -json document in v0.14.0)`
+      : !emulatorEndpoint
+        ? "CHOUDOUFU_EMULATOR_ENDPOINT is not set (bring up choudoufu's `just smoke` emulator stack and export it)"
+        : "";
+
+describe.skipIf(liveSkipReason !== "")(
+  `TerraformApplyOp applies a live root against choudoufu's emulator${liveSkipReason ? ` (skipped: ${liveSkipReason})` : ""}`,
+  () => {
+    it(
+      'inits, plans to a file, applies that file, gate: "never" — two null_resources applied',
+      { timeout: 600_000 },
+      async () => {
+        pointAtTheEmulator(emulatorEndpoint!);
+        const project = liveProject();
+
+        const { op } = TerraformApplyOp({
+          name: "choudoufu-acceptance",
+          root: "estate",
+          gate: "never",
+          cwd: project,
+        });
+
+        // The plan file is what crosses the (skipped) gate, so the built Op
+        // has to name one even here, where nothing waits on it.
+        const applyStep = (op as unknown as { props: OpConfig }).props.phases
+          .find((p) => p.name === "Apply")!
+          .steps[0] as { args?: Record<string, unknown> };
+        expect(applyStep.args?.planFile).toBeDefined();
+
+        const activities = await loadActivities(["terraform"]);
+        const result = await runOpLocally(
+          (op as unknown as { props: OpConfig }).props,
+          activities,
+          await loadProfiles(),
+        );
+
+        expect(result.status).toBe("ok");
+        expect(result.records.map((r) => `${r.phase}:${r.status}`)).toEqual(["Init:ok", "Plan:ok", "Apply:ok"]);
+
+        // A live root has no state file to read back: the state cache is not
+        // stock's `terraform.tfstate` and `show` over it renders nothing. The
+        // estate's own answer is the next plan, which is built from the live
+        // system, so what was applied is read back the way choudoufu means it
+        // to be. Empty, and its prior state holds exactly the two resources.
+        const after = await terraformPlan({ root: "estate", cwd: project });
+        expect(after.changed).toBe(false);
+        expect({ adds: after.adds, changes: after.changes, destroys: after.destroys }).toEqual({
+          adds: 0,
+          changes: 0,
+          destroys: 0,
+        });
+        const applied = (
+          (after.json as { prior_state?: { values?: { root_module?: { resources?: Array<{ address?: string }> } } } })
+            .prior_state?.values?.root_module?.resources ?? []
+        ).map((r) => r.address);
+        expect(applied.sort()).toEqual(["null_resource.first", "null_resource.second"]);
+      },
+    );
+
+    it(
+      "a plan file the live system has moved past comes back as a named refusal, not a throw",
+      { timeout: 600_000 },
+      async () => {
+        pointAtTheEmulator(emulatorEndpoint!);
+        const project = liveProject();
+        const mainTf = join(project, "root", "main.tf");
+        const original = readFileSync(mainTf, "utf8");
+
+        await terraformInit({ root: "estate", cwd: project });
+        const plan = await terraformPlan({ root: "estate", cwd: project });
+        expect(plan.changed).toBe(true);
+        expect(plan.adds).toBe(2);
+
+        // The configuration moves after the approval: one of the two resources
+        // the approver read is gone, so the fresh plan the apply builds has a
+        // change the file does not, and the file has one the fresh plan does not.
+        writeFileSync(mainTf, original.replace(/resource "null_resource" "second" \{[\s\S]*?\n\}\n/, ""));
+
+        const refusedRun = await terraformApply({ root: "estate", cwd: project, planFile: plan.planFile });
+        expect(refusedRun.applied).toBe(false);
+        expect(refusedRun.refused).toBe("approval-mismatch");
+        expect(refusedRun.refusal).toContain("The approved plan no longer matches the live system");
+        expect(refusedRun.refusal).toContain("null_resource.second");
+
+        // The sibling refusal, from the same exit status: the file was produced
+        // against another estate than the one this directory now declares.
+        writeFileSync(mainTf, original.replace('estate = "fixture-estate"', 'estate = "renamed-estate"'));
+        const wrongEstate = await terraformApply({ root: "estate", cwd: project, planFile: plan.planFile });
+        expect(wrongEstate.applied).toBe(false);
+        expect(wrongEstate.refused).toBe("wrong-estate");
+        expect(wrongEstate.refusal).toContain("The approved plan belongs to a different estate");
+
+        // And with the world back where the approval found it, the same file applies.
+        writeFileSync(mainTf, original);
+        const applied = await terraformApply({ root: "estate", cwd: project, planFile: plan.planFile });
+        expect(applied.applied).toBe(true);
+        expect(applied.refused).toBeUndefined();
+      },
+    );
+  },
+);