@intentius/chant-lexicon-terraform 0.124.54 → 0.124.56

package/src/state-reads.ts

@@ -0,0 +1,116 @@
+/**
+ * Variables read from another root's state (#4545, Atmos reads moved from
+ * terragucci's `atmos.ts`): an Atmos stack that sets a var with
+ * `!terraform.state <component> [<stack>] <output>` plans on the upstream
+ * instance's output, which only the job that holds a credential can read.
+ *
+ * In the `terraform-apply` step a read is `{ var, root, output }`: the var it
+ * sets, the `terraform.roots` entry whose state holds the value (its
+ * directory and workspace come from that entry), and the output followed by
+ * the keys into its value (`.subnets.private` is `["subnets", "private"]`).
+ * The step reads each upstream's `output -json` once and passes the values
+ * as `-var`, under the step's own `vars`, which win. A read whose upstream
+ * has no state, or no such output yet, waits: the root is not planned on a
+ * stand-in, and the wave says which upstream to apply first.
+ */
+
+/** One var a root reads from another root's state. */
+export interface TerraformStateRead {
+  /** The variable it sets. */
+  var: string;
+  /** The `terraform.roots` entry whose state holds the value. */
+  root: string;
+  /** The output, then the keys into its value. */
+  output: string[];
+}
+
+/** What filling a root's reads came to. */
+export interface FilledStateReads {
+  /** The values, by var. Complete only when `waiting` and `errors` are empty. */
+  vars: Record<string, unknown>;
+  /** Reads whose upstream has no such value yet: the root waits for these upstreams to apply. */
+  waiting: Array<{ read: TerraformStateRead; why: string }>;
+  /** Upstreams whose state could not be read at all. */
+  errors: string[];
+}
+
+/** What is wrong with a step's `reads`, empty when nothing is. */
+export function stateReadsProblems(reads: unknown): string[] {
+  if (reads === undefined) return [];
+  if (!Array.isArray(reads)) return ["reads must be a list of { var, root, output }"];
+  const problems: string[] = [];
+  const vars = new Set<string>();
+  for (const [i, r] of reads.entries()) {
+    const read = r as Partial<TerraformStateRead> | null;
+    if (!read || typeof read !== "object") {
+      problems.push(`reads[${i}] must be { var, root, output }`);
+      continue;
+    }
+    if (typeof read.var !== "string" || read.var === "") problems.push(`reads[${i}].var must name a variable`);
+    else if (vars.has(read.var)) problems.push(`reads[${i}] sets ${read.var}, which an earlier read sets too`);
+    else vars.add(read.var);
+    if (typeof read.root !== "string" || read.root === "") problems.push(`reads[${i}].root must name a terraform.roots entry`);
+    if (!Array.isArray(read.output) || read.output.length === 0 || !read.output.every((k) => typeof k === "string" && k !== "")) problems.push(`reads[${i}].output must be the output's name, then any keys into its value`);
+  }
+  return problems;
+}
+
+/** The value a read names in its upstream's outputs, or undefined when the outputs do not have it. */
+export function readOutputValue(outputs: Record<string, unknown>, output: readonly string[]): { value: unknown } | undefined {
+  const [name, ...keys] = output;
+  if (name === undefined || !(name in outputs)) return undefined;
+  let v: unknown = outputs[name];
+  for (const k of keys) {
+    if (!v || typeof v !== "object" || Array.isArray(v) || !(k in (v as Record<string, unknown>))) return undefined;
+    v = (v as Record<string, unknown>)[k];
+  }
+  return { value: v };
+}
+
+/**
+ * Fill `reads` from each upstream's outputs, read once per upstream through
+ * `outputsOf`, which throws when the state cannot be read.
+ */
+export async function fillStateReads(reads: readonly TerraformStateRead[], outputsOf: (root: string) => Promise<Record<string, unknown>>): Promise<FilledStateReads> {
+  const result: FilledStateReads = { vars: {}, waiting: [], errors: [] };
+  const cache = new Map<string, Promise<Record<string, unknown>>>();
+  for (const read of reads) {
+    if (!cache.has(read.root)) cache.set(read.root, outputsOf(read.root));
+    let outputs: Record<string, unknown>;
+    try {
+      outputs = await cache.get(read.root)!;
+    } catch (err) {
+      result.errors.push(`it reads ${read.var} from ${read.root}, whose state could not be read: ${err instanceof Error ? err.message : String(err)}`);
+      continue;
+    }
+    const got = readOutputValue(outputs, read.output);
+    if (!got) {
+      result.waiting.push({ read, why: Object.keys(outputs).length === 0 ? "has no state yet" : `has no output ${read.output.join(".")} yet` });
+      continue;
+    }
+    result.vars[read.var] = got.value;
+  }
+  return result;
+}
+
+/** Why a root with waiting reads was not planned, naming what to apply first. */
+export function waitingReadsReason(waiting: FilledStateReads["waiting"]): string {
+  const upstreams = [...new Set(waiting.map((w) => w.read.root))].sort();
+  return `not planned: ${waiting.map((w) => `it reads ${w.read.var} from ${w.read.root}, which ${w.why}`).join("; ")}; apply ${upstreams.join(", ")} first`;
+}
+
+/**
+ * The reads an Atmos instance's edges file lists (`{ reads: [{ var,
+ * upstream, output }] }`, as terragucci's `atmos write` leaves beside each
+ * instance), with each upstream mapped to its `terraform.roots` entry by
+ * `rootOf` (by default the upstream's own path).
+ */
+export function atmosStateReads(edges: unknown, rootOf: (upstream: string) => string = (u) => u): TerraformStateRead[] {
+  const reads = (edges as { reads?: unknown } | null)?.reads;
+  if (!Array.isArray(reads)) return [];
+  return reads.flatMap((r) => {
+    const read = r as { var?: unknown; upstream?: unknown; output?: unknown } | null;
+    if (!read || typeof read.var !== "string" || typeof read.upstream !== "string" || !Array.isArray(read.output) || !read.output.every((k) => typeof k === "string")) return [];
+    return [{ var: read.var, root: rootOf(read.upstream), output: read.output as string[] }];
+  });
+}

package/src/used-sources.test.ts

@@ -0,0 +1,75 @@
+/**
+ * The sources a root uses (chant#3924): module calls and provider addresses
+ * down the local tree, implied providers, a Terragrunt unit's source, the
+ * normal forms, and the local calls that cannot be followed.
+ */
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { usedSources } from "./used-sources";
+
+const FIX = join(import.meta.dirname, "__fixtures__", "used-sources");
+const ROOT = join(FIX, "root");
+
+describe("usedSources", () => {
+  test("lists the root's and its local modules' sources with their block paths", async () => {
+    const { used } = await usedSources(ROOT);
+    const rows = used.map((u) => [u.kind, u.block, u.source, u.normal]);
+    expect(rows).toEqual(
+      expect.arrayContaining([
+        ["provider", "required_providers.aws", "hashicorp/aws", "registry.terraform.io/hashicorp/aws"],
+        ["module", "module.vpc", "terraform-aws-modules/vpc/aws", "registry.terraform.io/terraform-aws-modules/vpc/aws"],
+        ["module", "module.net", "./modules/net", "./modules/net"],
+        ["provider", "provider.random", "hashicorp/random", "registry.terraform.io/hashicorp/random"],
+        ["provider", "provider.google", "hashicorp/google", "registry.terraform.io/hashicorp/google"],
+        ["module", "module.net > module.sg", "git::https://git.acme.dev/sg.git?ref=v1.0.0", "git::https://git.acme.dev/sg.git"],
+        ["provider", "module.net > required_providers.thing", "Registry.Acme.dev/acme/thing", "registry.acme.dev/acme/thing"],
+        ["provider", "module.net > provider.aws", "hashicorp/aws", "registry.terraform.io/hashicorp/aws"],
+      ]),
+    );
+  });
+
+  test("names the file each source is written in, relative to the project", async () => {
+    const { used } = await usedSources(ROOT);
+    expect(used.find((u) => u.block === "module.net > module.sg")!.file).toBe("modules/net/main.tf");
+    expect(used.find((u) => u.block === "module.net > required_providers.thing")!.file).toBe("modules/net/versions.tf.json");
+    expect(used.find((u) => u.block === "provider.google")!.file).toBe("extra.tofu");
+  });
+
+  test("says what implies a provider, and never implies the binary's own", async () => {
+    const { used } = await usedSources(ROOT);
+    expect(used.find((u) => u.block === "provider.random")!.implied).toBe('resource "random_id" "x"');
+    expect(used.find((u) => u.block === "module.net > provider.aws")!.implied).toBe('resource "aws_vpc" "v"');
+    expect(used.some((u) => u.block.endsWith("provider.terraform"))).toBe(false);
+    expect(used.some((u) => u.block === "provider.aws")).toBe(false);
+  });
+
+  test("names local calls it cannot follow and stops at a cycle", async () => {
+    const { used, unread } = await usedSources(ROOT);
+    expect(unread.map((u) => [u.block, u.source])).toEqual([
+      ["module.gone", "./modules/missing"],
+      ["module.outside", "../../outside"],
+    ]);
+    expect(unread[0]!.why).toContain("does not exist");
+    expect(unread[1]!.why).toContain("outside the project");
+    expect(used.filter((u) => u.block.startsWith("module.net > module.back > "))).toEqual([]);
+  });
+
+  test("reads only the root with followLocal off", async () => {
+    const { used, unread } = await usedSources(ROOT, { followLocal: false });
+    expect(used.some((u) => u.block.startsWith("module.net > "))).toBe(false);
+    expect(unread).toEqual([]);
+  });
+
+  test("leaves .tofu files out for terraform, and uses OpenTofu's registry for tofu", async () => {
+    const tf = await usedSources(ROOT, { binary: "terraform" });
+    expect(tf.used.some((u) => u.block === "provider.google")).toBe(false);
+    const tofu = await usedSources(ROOT, { binary: "tofu" });
+    expect(tofu.used.find((u) => u.block === "provider.google")!.normal).toBe("registry.opentofu.org/hashicorp/google");
+    expect(tofu.used.find((u) => u.block === "module.vpc")!.normal).toBe("registry.opentofu.org/terraform-aws-modules/vpc/aws");
+  });
+
+  test("reads a Terragrunt unit's terraform source", async () => {
+    const { used } = await usedSources(join(FIX, "unit"), { project: FIX });
+    expect(used).toEqual([{ kind: "module", file: "unit/terragrunt.hcl", block: "terraform", source: "tfr:///acme/vpc/aws?version=1.0.0", normal: "registry.terraform.io/acme/vpc/aws" }]);
+  });
+});

package/src/used-sources.ts

@@ -0,0 +1,204 @@
+/**
+ * Every module source and provider address a root uses (chant#3924), read
+ * from its files before `init` downloads anything, so a product can check
+ * them against an allow list (terragucci's `sources:`) or a proxy map
+ * (chant#4606) in a check job.
+ *
+ * For a root and, with `followLocal`, each local module it calls (`./` and
+ * `../`, followed down to the modules they call), it reads each `module`
+ * block's `source`, each `required_providers` entry, and the providers a
+ * module uses with no entry: a `provider` block, or a resource, data or
+ * ephemeral block's `provider` argument or type prefix, which `init` installs
+ * as `hashicorp/<name>`. In the root itself it also reads a Terragrunt unit's
+ * `terraform { source }`. Files are `.tf`, `.tofu` and their `.json` forms;
+ * `binary` narrows them to the set that binary reads, with OpenTofu's rule
+ * that `x.tofu` hides `x.tf`.
+ *
+ * Each source carries its file and its block path from the root
+ * (`module.a > module.b > required_providers.x`) and its normal form
+ * (./registry/source-forms.ts). A local call that cannot be followed
+ * (outside the project, or missing) is named in `unread`, since what it
+ * uses cannot be read.
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, posix, relative, sep } from "node:path";
+import { loadHcl2json, type Hcl2Json } from "@intentius/chant/terraform/hcl2json";
+import { isJsonConfigFile, jsonConfigTree, selectConfigFiles } from "./hcl/files";
+import { impliedProvider, normalizeModuleSource, providerAddress, withoutQuery, type SourceBinary } from "./registry/source-forms";
+import type { RegistryProxy } from "./registry/proxies";
+
+/** One module source or provider address a root uses. */
+export interface UsedSource {
+  kind: "module" | "provider";
+  /** The file it is written in, relative to the project, `/`-separated. */
+  file: string;
+  /**
+   * The block that names it, after the module calls that reach it:
+   * `module.vpc`, `terraform` for a terragrunt.hcl's source,
+   * `required_providers.aws`, or `provider.aws` for an implied one.
+   */
+  block: string;
+  /** The source or address as written (`hashicorp/<name>` for an implied provider). */
+  source: string;
+  /**
+   * The normal form: for a module, the source without its query, normalised
+   * ({@link normalizeModuleSource}); for a provider, `<host>/<namespace>/<type>`
+   * ({@link providerAddress}), or the source as written when it is not an address.
+   */
+  normal: string;
+  /** For an implied provider: the block that implies it. */
+  implied?: string;
+}
+
+/** A local module call that could not be followed. */
+export interface UnreadSource {
+  file: string;
+  block: string;
+  source: string;
+  why: string;
+}
+
+export interface UsedSourcesOptions {
+  /** Follow `./` and `../` module calls down the tree. Default true. */
+  followLocal?: boolean;
+  /** The project's top: file paths are relative to it, and a call that leaves it is not followed. Default the root's directory. */
+  project?: string;
+  /** Read only the files this binary reads, and default provider hosts to its registry. Default: every `.tf` and `.tofu` file, Terraform's registry. */
+  binary?: SourceBinary;
+  /** The HCL parser. Default: core's lazily loaded hcl2json. */
+  parser?: Hcl2Json;
+  /** Registry proxies (chant#4606): a source through one gets its upstream's normal form. */
+  proxies?: readonly RegistryProxy[];
+}
+
+type Tree = Record<string, unknown>;
+
+/** A block list as hcl2json writes it (an array) or a JSON file may (one object). */
+const bodies = (v: unknown): Tree[] => (Array.isArray(v) ? (v as Tree[]) : v && typeof v === "object" ? [v as Tree] : []);
+
+const ALL_SUFFIXES = [".tf", ".tofu", ".tf.json", ".tofu.json"];
+
+/** The configuration files of one module directory, plus terragrunt.hcl when `terragrunt`. */
+function moduleFiles(dir: string, terragrunt: boolean, binary: SourceBinary | undefined): string[] {
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) return [];
+  const names = readdirSync(dir).filter((n) => {
+    try {
+      return statSync(join(dir, n)).isFile();
+    } catch {
+      return false;
+    }
+  });
+  const config = binary ? selectConfigFiles(names, binary) : names.filter((n) => ALL_SUFFIXES.some((s) => n.endsWith(s))).sort();
+  return terragrunt && names.includes("terragrunt.hcl") ? [...config, "terragrunt.hcl"] : config;
+}
+
+/** The local name of the provider a resource, data or ephemeral block uses: its `provider` argument's, else its type's prefix. */
+function providerOf(type: string, body: Tree): string {
+  const ref = typeof body.provider === "string" ? /^(?:\$\{)?([A-Za-z][A-Za-z0-9_-]*)/.exec(body.provider) : null;
+  return ref ? ref[1]! : type.split("_")[0]!;
+}
+
+const isLocal = (source: string): boolean => /^\.\.?\//.test(source);
+
+interface DirRead {
+  providers: UsedSource[];
+  calls: { file: string; block: string; source: string }[];
+}
+
+async function readDir(abs: string, shown: string, prefix: string, top: boolean, o: Required<Pick<UsedSourcesOptions, "parser">> & UsedSourcesOptions): Promise<DirRead> {
+  const providers: UsedSource[] = [];
+  const calls: DirRead["calls"] = [];
+  const declared = new Set<string>();
+  const implied = new Map<string, { file: string; what: string }>();
+  const provider = (file: string, block: string, source: string, what?: string): UsedSource => ({
+    kind: "provider",
+    file,
+    block,
+    source,
+    normal: providerAddress(source, { binary: o.binary, proxies: o.proxies }) ?? source.trim().toLowerCase(),
+    ...(what ? { implied: what } : {}),
+  });
+  for (const name of moduleFiles(abs, top, o.binary)) {
+    const file = shown === "." ? name : posix.join(shown, name);
+    const text = readFileSync(join(abs, name), "utf-8");
+    let tree: Tree;
+    try {
+      tree = (isJsonConfigFile(name) ? jsonConfigTree(text) : await o.parser.parse(file, text)) as Tree;
+    } catch (e) {
+      throw new Error(`${file} could not be read (${(e as Error).message.split("\n")[0]}), so its module and provider sources are unknown`);
+    }
+    if (name === "terragrunt.hcl") {
+      for (const b of bodies(tree.terraform)) if (typeof b.source === "string") calls.push({ file, block: `${prefix}terraform`, source: b.source });
+      continue;
+    }
+    for (const [call, list] of Object.entries((tree.module as Tree | undefined) ?? {})) {
+      for (const b of bodies(list)) if (typeof b.source === "string") calls.push({ file, block: `${prefix}module.${call}`, source: b.source });
+    }
+    for (const t of bodies(tree.terraform)) {
+      for (const req of bodies(t.required_providers)) {
+        for (const [local, v] of Object.entries(req)) {
+          declared.add(local);
+          const source = v && typeof v === "object" && typeof (v as Tree).source === "string" ? ((v as Tree).source as string) : `hashicorp/${local}`;
+          providers.push(provider(file, `${prefix}required_providers.${local}`, source));
+        }
+      }
+    }
+    for (const local of Object.keys((tree.provider as Tree | undefined) ?? {})) if (!implied.has(local)) implied.set(local, { file, what: `provider "${local}"` });
+    for (const kind of ["resource", "data", "ephemeral"]) {
+      for (const [type, named] of Object.entries((tree[kind] as Tree | undefined) ?? {})) {
+        for (const [name, list] of Object.entries((named as Tree) ?? {})) {
+          for (const b of bodies(list)) {
+            const local = providerOf(type, b);
+            if (!implied.has(local)) implied.set(local, { file, what: `${kind} "${type}" "${name}"` });
+          }
+        }
+      }
+    }
+  }
+  for (const [local, at] of implied) {
+    const source = declared.has(local) ? undefined : impliedProvider(local);
+    if (source) providers.push(provider(at.file, `${prefix}provider.${local}`, source, at.what));
+  }
+  return { providers, calls };
+}
+
+/**
+ * Every module source and provider address the root at `dir` uses, and each
+ * local call that could not be followed. Throws, naming the file, when a
+ * file cannot be parsed, so a check fails loudly instead of checking less.
+ */
+export async function usedSources(dir: string, options: UsedSourcesOptions = {}): Promise<{ used: UsedSource[]; unread: UnreadSource[] }> {
+  const project = options.project ?? dir;
+  const o = { ...options, parser: options.parser ?? (await loadHcl2json()) };
+  const follow = options.followLocal !== false;
+  const used: UsedSource[] = [];
+  const unread: UnreadSource[] = [];
+  const shownOf = (abs: string) => relative(project, abs).split(sep).join("/") || ".";
+
+  const walk = async (rel: string, prefix: string, chain: string[], top: boolean): Promise<void> => {
+    const { providers, calls } = await readDir(join(project, rel), rel, prefix, top, o);
+    used.push(...providers);
+    for (const c of calls) {
+      used.push({ kind: "module", file: c.file, block: c.block, source: c.source, normal: isLocal(c.source) ? withoutQuery(c.source) : normalizeModuleSource(withoutQuery(c.source), { binary: o.binary, proxies: o.proxies }) });
+      if (!follow || !isLocal(c.source)) continue;
+      // A Terragrunt source's `//` marks the directory it copies; the module is the path with it read as one `/`.
+      const path = withoutQuery(c.source).replace(/\/\/+/g, "/");
+      const next = posix.normalize(posix.join(rel, path)).replace(/\/+$/, "") || ".";
+      if (next === ".." || next.startsWith("../")) {
+        unread.push({ file: c.file, block: c.block, source: c.source, why: "it is outside the project, so the sources it uses cannot be read" });
+        continue;
+      }
+      if (!existsSync(join(project, next))) {
+        unread.push({ file: c.file, block: c.block, source: c.source, why: `${next} does not exist, so the sources it uses cannot be read` });
+        continue;
+      }
+      if (chain.includes(next) || moduleFiles(join(project, next), false, o.binary).length === 0) continue;
+      await walk(next, `${c.block} > `, [...chain, next], false);
+    }
+  };
+
+  const start = shownOf(dir);
+  if (start === ".." || start.startsWith("../")) throw new Error(`${dir} is outside the project ${project}`);
+  await walk(start, "", [start], true);
+  return { used, unread };
+}