@intentius/chant 0.124.44 → 0.124.45

package/dist/workspace/declaration.schema.json

@@ -289,6 +289,33 @@
               "type": "string",
               "pattern": "^[1-9][0-9]*(m|h|d)$",
               "description": "How long a pull request's environment lives after its last apply, such as 72h, 90m or 7d. chant components pr-env sweep tears down one whose TTL passed. 72h when absent."
+            },
+            "teardown": {
+              "type": "object",
+              "description": "A command that tears down what no lexicon does, such as a Terraform root's copy of the environment (#4330). pr-env down and pr-env sweep run it after the ownership check, twice: CHANT_PR_ENV_STAGE=owned writes the candidates with their read-back markers to CHANT_PR_ENV_RESULT, and CHANT_PR_ENV_STAGE=teardown deletes the candidates in CHANT_PR_ENV_CANDIDATES and writes their outcomes. It runs without the forge token variables. Added in schema 1 by chant 0.124.45, so a declaration that uses it sets minReader to 0.124.45 or newer.",
+              "required": ["command"],
+              "additionalProperties": false,
+              "patternProperties": {
+                "^x-": true
+              },
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "pattern": "^[a-z0-9][a-z0-9-]*$",
+                  "description": "The name the teardown's outcomes are attributed to, as a lexicon's are. teardown when absent."
+                },
+                "command": {
+                  "type": "array",
+                  "minItems": 1,
+                  "items": { "type": "string", "minLength": 1 },
+                  "description": "Argv, run in the member's directory at the commit the job runs at. {env}, {pr} and {stage} are filled in."
+                },
+                "lexicons": {
+                  "type": "array",
+                  "items": { "type": "string", "minLength": 1 },
+                  "description": "The lexicons the command tears down in place of, such as terraform. Their own teardown is left out, so one with no teardown execution does not leave the environment incomplete."
+                }
+              }
             }
           }
         }
@@ -363,7 +390,7 @@
     "ciCloudIdentityRules": {
       "type": "array",
       "minItems": 1,
-      "description": "Rules, first match wins: a rule matches a pipeline when its environments match the pipeline's environment and its members match the pipeline's member. A rule with no environments matches every environment, and one with no members every member and a pipeline outside a member.",
+      "description": "Rules, first match wins: a rule matches a pipeline when its environments match the pipeline's environment and its members match the pipeline's member, and a target inside the member when its targets match the target too. A rule with no environments matches every environment, one with no members every member and a pipeline outside a member, and one with no targets every target.",
       "items": {
         "type": "object",
         "additionalProperties": false,
@@ -378,6 +405,14 @@
             "description": "Environment-name globs, such as prod or staging-*."
           },
           "members": { "$ref": "#/$defs/ciMemberGlobs" },
+          "targets": {
+            "type": "array",
+            "minItems": 1,
+            "uniqueItems": true,
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Target globs inside the member (#4332), such as Terraform roots or Op wave run targets: * within one / segment, ** across segments, ? one character. The plan job trades for every plan identity the member's targets take, and an apply splits into one job per apply identity. Added in schema 1 by chant 0.124.45, so a declaration that uses it sets minReader to 0.124.45 or newer."
+          },
+          "name": { "type": "string", "pattern": "^[a-z0-9_-]+$", "description": "The name of the apply job group the rule's targets form (#4332). Default: from its first target glob. Added by chant 0.124.45." },
           "plan": { "type": "string", "minLength": 1, "description": "The identity the plan and record jobs take: a role ARN, a service account email or a client id." },
           "apply": { "type": "string", "minLength": 1, "description": "The identity the apply jobs take." }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@intentius/chant",
-  "version": "0.124.44",
+  "version": "0.124.45",
   "description": "Declarative infrastructure-as-code toolkit — TypeScript on Node.js",
   "license": "Apache-2.0",
   "homepage": "https://intentius.io/chant",

package/src/ci-identity-targets.test.ts

@@ -0,0 +1,275 @@
+/**
+ * #4332: per-target cloud identities. A rule with `targets` covers only the
+ * targets inside a member those globs match. The plan trade holds every
+ * plan identity a member's targets take, the trade leaves each identity's
+ * credentials in the CHANT_CLOUD_TARGETS file, targetCloudEnv hands a
+ * target its own, and cloudIdentityGroups splits an apply into one job per
+ * apply identity. The trade tests run the generated `node -e` line against
+ * the local emulator (./__fixtures__/oidc-emulator.ts). Test values only.
+ */
+
+import { execFile } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { build } from "esbuild";
+// @ts-ignore: picomatch has no types declaration
+import picomatch from "picomatch";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import {
+  CLOUD_TARGETS_ENV,
+  cloudIdentityGroups,
+  cloudIdentityProblems,
+  cloudIdentityWave,
+  nameGlobMatch,
+  oneApplyIdentityProblem,
+  readTargetsFile,
+  resolveCloudIdentity,
+  resolveTargetCloudIdentity,
+  targetCloudEnv,
+  targetGlobMatch,
+  TRADE_SPEC_ENV,
+  tradeIdentities,
+  tradeLine,
+  type CloudIdentityDeclaration,
+  type CloudTradeSpec,
+} from "./ci-identity";
+import { assertOpWavesSpec, type OpWavesSpec } from "./op/op-waves";
+import { emulatedCredentials, EMULATED_JWT_PREFIX, startOidcEmulator, type OidcEmulator } from "./__fixtures__/oidc-emulator";
+import { REPO } from "./workspace/__fixtures__/contract-repo";
+
+const run = promisify(execFile);
+
+const role = (n: string) => `arn:aws:iam::111111111111:role/${n}`;
+
+/** terragucci's shape: prod and staging roots plan and apply as their environment's roles, the rest as the member's own. */
+const DECL: CloudIdentityDeclaration = {
+  aws: {
+    region: "eu-west-1",
+    rules: [
+      { environments: ["prod"], members: ["infra"], targets: ["envs/prod/**"], plan: role("prod-plan"), apply: role("prod-apply") },
+      { environments: ["prod"], members: ["infra"], targets: ["envs/staging/**"], name: "staging", plan: role("staging-plan"), apply: role("staging-apply") },
+      { environments: ["prod"], plan: role("own-plan"), apply: role("own-apply") },
+      { targets: ["unreachable/**"], plan: role("late-plan"), apply: role("late-apply") },
+    ],
+  },
+  gcp: {
+    provider: "projects/123/locations/global/workloadIdentityPools/ci/providers/forge",
+    rules: [{ targets: ["envs/prod/**"], plan: "prod-plan@p.iam.gserviceaccount.com", apply: "prod-apply@p.iam.gserviceaccount.com" }],
+  },
+};
+
+const ROOTS = ["envs/prod/net", "envs/staging/net", "shared/dns", "envs/prod/app"];
+
+describe("which identity a target takes (#4332)", () => {
+  test("the first rule that covers the environment, the member and the target wins, per cloud", () => {
+    const prod = resolveTargetCloudIdentity(DECL, "prod", "infra", "envs/prod/net")!;
+    expect(prod.plan.aws).toEqual({ role: role("prod-plan"), region: "eu-west-1", audience: "sts.amazonaws.com" });
+    expect(prod.apply.gcp!.serviceAccount).toBe("prod-apply@p.iam.gserviceaccount.com");
+    const staging = resolveTargetCloudIdentity(DECL, "prod", "infra", "envs/staging/net")!;
+    expect(staging.apply.aws!.role).toBe(role("staging-apply"));
+    expect(staging.apply.gcp).toBeUndefined();
+    expect(resolveTargetCloudIdentity(DECL, "prod", "infra", "shared/dns")!.plan.aws!.role).toBe(role("own-plan"));
+  });
+
+  test("a rule with targets covers no target of another member, and a rule after one without targets is never reached", () => {
+    expect(resolveTargetCloudIdentity(DECL, "prod", "web", "envs/prod/net")!.plan.aws!.role).toBe(role("own-plan"));
+    expect(resolveTargetCloudIdentity(DECL, "prod", "infra", "unreachable/x")!.plan.aws!.role).toBe(role("own-plan"));
+    expect(resolveTargetCloudIdentity(DECL, "dev", "infra", "unreachable/x")!.plan.aws!.role).toBe(role("late-plan"));
+    expect(resolveTargetCloudIdentity({ aws: DECL.aws }, "dev", "infra", "shared/dns")).toBeUndefined();
+  });
+
+  test("the pipeline's trade names its own identity and, before it, the targets that take another", () => {
+    const p = resolveCloudIdentity(DECL, "prod", "infra")!;
+    expect(p.plan.aws).toEqual({
+      role: role("own-plan"),
+      targets: [
+        { targets: ["envs/prod/**"], role: role("prod-plan") },
+        { targets: ["envs/staging/**"], role: role("staging-plan") },
+      ],
+      region: "eu-west-1",
+      audience: "sts.amazonaws.com",
+    });
+    // GCP has no rule for the member as a whole: targets alone.
+    expect(p.plan.gcp).toMatchObject({ targets: [{ targets: ["envs/prod/**"], serviceAccount: "prod-plan@p.iam.gserviceaccount.com" }] });
+    expect(p.plan.gcp!.serviceAccount).toBeUndefined();
+    expect(tradeIdentities(p.plan)).toEqual([role("own-plan"), role("prod-plan"), role("staging-plan"), "prod-plan@p.iam.gserviceaccount.com"]);
+    // The plan trade names no apply identity.
+    for (const id of tradeIdentities(p.apply)) expect(JSON.stringify(p.plan)).not.toContain(id);
+    // Without targets rules, as #4296 resolved it.
+    expect(resolveCloudIdentity(DECL, "prod", "web")!.plan.aws).toEqual({ role: role("own-plan"), region: "eu-west-1", audience: "sts.amazonaws.com" });
+  });
+
+  test("a plan identity that any rule applies as is still refused, targets or not", () => {
+    expect(cloudIdentityProblems(DECL)).toEqual([]);
+    const crossed = cloudIdentityProblems({ aws: { rules: [{ targets: ["a/**"], plan: "x", apply: "y" }, { plan: "y", apply: "z" }] } });
+    expect(crossed).toEqual([{ pointer: "/ci/identity/cloud/aws/rules/1", message: expect.stringContaining("plans as y, which another rule applies as") }]);
+  });
+});
+
+describe("one apply job per identity (#4332)", () => {
+  test("targets that take the same identities in every cloud are one group, named from the rule, default last", () => {
+    const groups = cloudIdentityGroups(DECL, "prod", "infra", ROOTS);
+    expect(groups.map((g) => [g.name, g.targets])).toEqual([
+      ["envs-prod", ["envs/prod/net", "envs/prod/app"]],
+      ["staging", ["envs/staging/net"]],
+      ["default", ["shared/dns"]],
+    ]);
+    expect(tradeIdentities(groups[0]!.identity!.apply)).toEqual([role("prod-apply"), "prod-apply@p.iam.gserviceaccount.com"]);
+    expect(tradeIdentities(groups[2]!.identity!.apply)).toEqual([role("own-apply")]);
+  });
+
+  test("two rules whose names collide get -2, and targets no rule covers form a group with no identity", () => {
+    const decl: CloudIdentityDeclaration = { aws: { rules: [{ targets: ["a/*"], name: "x", plan: "p1", apply: "a1" }, { targets: ["b/*"], name: "x", plan: "p2", apply: "a2" }] } };
+    expect(cloudIdentityGroups(decl, "prod", undefined, ["a/1", "b/1", "c/1"])).toEqual([
+      expect.objectContaining({ name: "x", targets: ["a/1"] }),
+      expect.objectContaining({ name: "x-2", targets: ["b/1"] }),
+      { name: "default", targets: ["c/1"] },
+    ]);
+  });
+
+  test("an Op wave's deciding job trades every plan identity and each group's job its apply identities alone", () => {
+    const wave = cloudIdentityWave(DECL, "prod", "infra", ROOTS);
+    expect(wave.cloud).toEqual(resolveCloudIdentity(DECL, "prod", "infra")!.plan);
+    expect(wave.jobs!.map((j) => [j.name, j.runs, tradeIdentities(j.cloud!)])).toEqual([
+      ["envs-prod", ["envs/prod/net", "envs/prod/app"], [role("prod-apply"), "prod-apply@p.iam.gserviceaccount.com"]],
+      ["staging", ["envs/staging/net"], [role("staging-apply")]],
+      ["default", ["shared/dns"], [role("own-apply")]],
+    ]);
+    const spec: OpWavesSpec = { name: "tf-apply", op: "tf-apply", plan: ["plan", "{target}", "{plan}"], apply: ["apply", "{target}"], waves: [{ name: "wave-1", runs: ROOTS.map((target) => ({ target })), ...wave }] } as OpWavesSpec;
+    expect(() => assertOpWavesSpec(spec)).not.toThrow();
+  });
+
+  test("with one group the wave's one job trades its apply identities and has no jobs", () => {
+    expect(cloudIdentityWave(DECL, "prod", "infra", ["envs/prod/net"])).toEqual({ cloud: resolveTargetCloudIdentity(DECL, "prod", "infra", "envs/prod/net")!.apply });
+    expect(cloudIdentityWave(undefined, "prod", "infra", ["x"])).toEqual({});
+  });
+
+  test("an Op wave's cloud that names no identity is refused", () => {
+    const spec = { name: "w", op: "w", plan: ["p", "{plan}"], apply: ["a"], waves: [{ name: "one", runs: [{ target: "t" }], cloud: { aws: { region: "eu-west-1", audience: "sts.amazonaws.com" } } }] } as unknown as OpWavesSpec;
+    expect(() => assertOpWavesSpec(spec)).toThrow(/cloud\.aws must give an audience and a role/);
+    expect(() => assertOpWavesSpec({ ...spec, waves: [{ ...spec.waves[0]!, cloud: { k8s: {} } as never }] })).toThrow(/k8s is not a cloud/);
+  });
+
+  test("one job may hold one apply identity per cloud", () => {
+    expect(oneApplyIdentityProblem(resolveCloudIdentity(DECL, "prod", "web")!.apply)).toBeUndefined();
+    expect(oneApplyIdentityProblem(resolveCloudIdentity(DECL, "prod", "infra")!.apply)).toMatch(/3 aws apply identities/);
+  });
+});
+
+describe("the glob grammars (#4332)", () => {
+  test("member and environment globs read as picomatch reads the ciMemberGlobs grammar", () => {
+    const names = ["api", "web", "prod", "prod-eu", "staging-eu", "staging", "a1", "b2", "app-x"];
+    const globs = ["api", "prod-*", "staging-*", "*", "a?", "[ab][0-9]", "{api,web}", "app-{x,y}", "!prod*", "{a,b}{1,2}", "*-eu"];
+    for (const g of globs) for (const n of names) expect([g, n, nameGlobMatch(g, n)]).toEqual([g, n, (picomatch(g) as (s: string) => boolean)(n)]);
+    expect(nameGlobMatch("[!a]*", "web")).toBe(true);
+    expect(nameGlobMatch("[!a]*", "api")).toBe(false);
+  });
+
+  test("target globs read as the terraform lexicon's unit globs", () => {
+    expect(targetGlobMatch("envs/prod/**", "envs/prod/net")).toBe(true);
+    expect(targetGlobMatch("envs/prod/**", "envs/prod")).toBe(false);
+    expect(targetGlobMatch("envs/*/net", "envs/prod/net")).toBe(true);
+    expect(targetGlobMatch("envs/*", "envs/prod/net")).toBe(false);
+    expect(targetGlobMatch("a/**/b", "a/b")).toBe(true);
+    expect(targetGlobMatch("a/?", "a/b")).toBe(true);
+    expect(targetGlobMatch("envs/{a,b}", "envs/a")).toBe(false);
+  });
+});
+
+const dirs: string[] = [];
+let emu: OidcEmulator;
+beforeAll(async () => {
+  emu = await startOidcEmulator();
+});
+afterAll(async () => {
+  await emu.close();
+  for (const d of dirs) rmSync(d, { recursive: true, force: true });
+});
+
+const BASE = { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: process.env.HOME ?? "/tmp" };
+
+/** Run the trade line as a GitHub step does, with RUNNER_TEMP its own directory. */
+async function trade(spec: CloudTradeSpec): Promise<{ out: string; stderr: string; stdout: string; temp: string }> {
+  const temp = mkdtempSync(join(tmpdir(), "chant-trade-targets-"));
+  dirs.push(temp);
+  const file = join(temp, "github-env");
+  writeFileSync(file, "");
+  const r = await run("sh", ["-c", tradeLine(file, "github-env")], { env: { ...BASE, ...emu.requestEnv, RUNNER_TEMP: temp, [TRADE_SPEC_ENV]: JSON.stringify(emu.pointAt(spec)) } });
+  return { out: readFileSync(file, "utf-8"), stderr: r.stderr, stdout: r.stdout, temp };
+}
+
+const envOf = (lines: string): Record<string, string> => Object.fromEntries(lines.split("\n").filter(Boolean).map((l) => [l.slice(0, l.indexOf("=")), l.slice(l.indexOf("=") + 1)]));
+
+describe("the plan trade with targets, run against the emulator (#4332)", () => {
+  test("it trades one token per cloud for every plan identity, keeps the job's own in GITHUB_ENV and each in the targets file", async () => {
+    const before = { minted: emu.log.minted.length, assumed: emu.log.assumed.length, impersonated: emu.log.impersonated.length };
+    const plan = resolveCloudIdentity(DECL, "prod", "infra")!.plan;
+    const r = await trade(plan);
+    expect(r.stderr).not.toContain("failed");
+    // One minted token per cloud, however many identities it is traded for.
+    expect(emu.log.minted.slice(before.minted)).toEqual(["sts.amazonaws.com", "https://iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/ci/providers/forge"]);
+    expect(emu.log.assumed.slice(before.assumed).map((a) => a.role)).toEqual([role("own-plan"), role("prod-plan"), role("staging-plan")]);
+    expect(emu.log.impersonated.slice(before.impersonated)).toEqual(["prod-plan@p.iam.gserviceaccount.com"]);
+    const env = envOf(r.out);
+    expect(env.AWS_ACCESS_KEY_ID).toBe(emulatedCredentials(role("own-plan")).accessKeyId);
+    // GCP names no identity of the job's own, so the job's environment has none.
+    expect(env.GOOGLE_OAUTH_ACCESS_TOKEN).toBeUndefined();
+    const path = env[CLOUD_TARGETS_ENV]!;
+    expect(path.startsWith(r.temp)).toBe(true);
+    expect(statSync(path).mode & 0o777).toBe(0o600);
+    const file = readTargetsFile(path);
+    expect(file.clouds.aws!.default).toBe(role("own-plan"));
+    expect(Object.keys(file.clouds.aws!.vars)).toEqual([role("own-plan"), role("prod-plan"), role("staging-plan")]);
+    // Every secret is masked, and the minted token is written nowhere.
+    expect(r.stdout).toContain(`::add-mask::${emulatedCredentials(role("staging-plan")).token}`);
+    expect(r.out + readFileSync(path, "utf-8") + r.stdout + r.stderr).not.toContain(EMULATED_JWT_PREFIX);
+    // No apply identity was asked for.
+    for (const id of tradeIdentities(resolveCloudIdentity(DECL, "prod", "infra")!.apply)) expect(emu.log.assumed.map((a) => a.role)).not.toContain(id);
+
+    // targetCloudEnv hands each target its own identity, as terragucci's rootRoleEnv did.
+    const job = { ...env, PATH: "/bin", AWS_ROLE_ARN: undefined };
+    const prod = targetCloudEnv(job, "envs/prod/net");
+    expect(prod.AWS_ACCESS_KEY_ID).toBe(emulatedCredentials(role("prod-plan")).accessKeyId);
+    expect(prod.AWS_SESSION_TOKEN).toBe(emulatedCredentials(role("prod-plan")).token);
+    expect(prod.AWS_REGION).toBe("eu-west-1");
+    expect(prod.GOOGLE_OAUTH_ACCESS_TOKEN).toBe(`ya29.${emulatedCredentials("prod-plan@p.iam.gserviceaccount.com").token}`);
+    expect(prod.PATH).toBe("/bin");
+    expect(targetCloudEnv(job, "envs/staging/net").AWS_ACCESS_KEY_ID).toBe(emulatedCredentials(role("staging-plan")).accessKeyId);
+    const shared = targetCloudEnv(job, "shared/dns");
+    expect(shared.AWS_ACCESS_KEY_ID).toBe(emulatedCredentials(role("own-plan")).accessKeyId);
+    expect(shared.GOOGLE_OAUTH_ACCESS_TOKEN).toBeUndefined();
+  });
+
+  test("an apply group's trade names one identity per cloud and leaves no targets file", async () => {
+    const before = emu.log.assumed.length;
+    const group = cloudIdentityGroups(DECL, "prod", "infra", ROOTS)[1]!;
+    const r = await trade(group.identity!.apply);
+    expect(emu.log.assumed.slice(before).map((a) => a.role)).toEqual([role("staging-apply")]);
+    expect(envOf(r.out)[CLOUD_TARGETS_ENV]).toBeUndefined();
+    expect(targetCloudEnv(envOf(r.out), "envs/staging/net")).toEqual(envOf(r.out));
+  });
+
+  test("a targets file that is not one is refused by name", () => {
+    expect(() => targetCloudEnv({ [CLOUD_TARGETS_ENV]: "/nowhere.json" }, "x", () => "{}")).toThrow(/not a version 1 targets file/);
+    expect(() => targetCloudEnv({ [CLOUD_TARGETS_ENV]: "/nowhere.json" }, "x", () => "nope")).toThrow(/cannot be read/);
+  });
+});
+
+describe("the ci-identity bundle (#4332)", () => {
+  test("holds ci-identity.ts alone, with no glob library, and stays small", async () => {
+    const result = await build({
+      entryPoints: [join(REPO, "packages", "core", "src", "ci-identity.ts")],
+      bundle: true,
+      platform: "node",
+      format: "esm",
+      write: false,
+      metafile: true,
+      outdir: "out",
+      logLevel: "silent",
+    });
+    expect(Object.keys(result.metafile.inputs).map((i) => i.replace(/^.*?packages\//, "packages/"))).toEqual(["packages/core/src/ci-identity.ts"]);
+    expect(result.outputFiles.reduce((n, f) => n + f.contents.byteLength, 0)).toBeLessThan(24 * 1024);
+  });
+});