@intentius/chant-lexicon-terraform 0.124.52 → 0.124.55

package/src/registry/source-forms.ts

@@ -0,0 +1,216 @@
+/**
+ * Module sources and provider addresses in a normal form (chant#3924), beside
+ * the registry address parsing of ./address.ts (chant#4565), so a product's
+ * allow list, the catalog's consumer scan and the proxy forms (chant#4606)
+ * compare sources one way. No Node imports, so an edge function can carry it.
+ *
+ * A module source is compared without its `?` query: a `ref`, `version` or
+ * `tag` pins a release and does not move where the code comes from. A
+ * registry source written without a host is read by Terraform from
+ * registry.terraform.io and by OpenTofu from registry.opentofu.org, so it
+ * stands for both; Terragrunt's `tfr:///` and `tfr://<host>/` are the same
+ * sources. Registry shapes follow `classifyModuleSource`
+ * (../lint/post-synth/module-source.ts), which mirrors
+ * hashicorp/terraform-registry-address.
+ *
+ * A provider address is `<host>/<namespace>/<type>` in lower case. A
+ * `required_providers` source without a host gets the binary's default
+ * registry, one with only a type gets the `hashicorp` namespace (Terraform's
+ * legacy reading), and a provider a module uses with no entry at all is the
+ * one `init` installs for it, `hashicorp/<name>`; the binary's own
+ * `terraform` provider is never downloaded and has no address.
+ *
+ * With `proxies` (chant#4606, ./proxies.ts), a source written through an
+ * Artifactory remote repository or a Nexus proxy host block stands for its
+ * upstream source: its forms include the upstream's, and its normal form
+ * and provider address are the upstream's, so it compares equal to the
+ * plain source.
+ */
+import { classifyModuleSource } from "../lint/post-synth/module-source";
+import { parseRegistrySource } from "./address";
+import { moduleThroughProxy, providerThroughProxy, type RegistryProxy } from "./proxies";
+
+/** The public registry's host, which a source with no host names under Terraform. */
+export const PUBLIC_REGISTRY_HOST = "registry.terraform.io";
+/** OpenTofu's public registry's host. */
+export const OPENTOFU_REGISTRY_HOST = "registry.opentofu.org";
+/** The hosts a provider or registry module address written without one is read from, Terraform's first. */
+export const DEFAULT_REGISTRIES: readonly string[] = [PUBLIC_REGISTRY_HOST, OPENTOFU_REGISTRY_HOST];
+
+/** Which binary reads an address: it decides the default registry host. */
+export type SourceBinary = "terraform" | "tofu";
+
+/** The default registry host for a binary. */
+export const defaultRegistryHost = (binary: SourceBinary = "terraform"): string => (binary === "tofu" ? OPENTOFU_REGISTRY_HOST : PUBLIC_REGISTRY_HOST);
+
+/** How a source is read: the binary (for default hosts) and the registry proxies in front of upstreams. */
+export interface SourceFormOptions {
+  binary?: SourceBinary;
+  proxies?: readonly RegistryProxy[];
+}
+
+/** A binary alone, or the options. */
+export type SourceFormArg = SourceBinary | SourceFormOptions;
+
+const optionsOf = (o: SourceFormArg | undefined): SourceFormOptions => (typeof o === "string" ? { binary: o } : (o ?? {}));
+
+/** A source without its `?` query, trimmed. */
+export const withoutQuery = (source: string): string => source.trim().split("?")[0]!;
+
+/** A hostless registry module source (`<ns>/<name>/<system>`, optionally `//<subdir>`), as the registry protocol shapes one. */
+function isHostlessModule(source: string): boolean {
+  const base = source.split("//")[0]!;
+  return base.split("/").length === 3 && classifyModuleSource(base).kind === "registry";
+}
+
+/** The source without a default registry's host, when it starts with one. */
+function atDefaultRegistry(source: string): string | undefined {
+  const host = DEFAULT_REGISTRIES.find((h) => source.toLowerCase().startsWith(`${h}/`));
+  return host ? source.slice(host.length + 1) : undefined;
+}
+
+/**
+ * Every form a module source matches as: as written without its query, then
+ * its registry forms. A hostless registry source adds itself at each default
+ * registry; one at a default registry adds its hostless form and its form at
+ * the other default registry; `tfr:///x` adds `x` at each default registry
+ * and `tfr://<host>/x` adds `<host>/x`. Any other source (local, git, OCI,
+ * an archive URL) has the one form. A source through one of `proxies` adds
+ * the forms of the upstream source it stands for.
+ */
+export function moduleSourceForms(source: string, options?: SourceFormOptions): string[] {
+  const out = plainModuleForms(source);
+  const via = moduleThroughProxy(source, options?.proxies);
+  if (via) for (const f of plainModuleForms(via.upstream)) if (!out.includes(f)) out.push(f);
+  return out;
+}
+
+function plainModuleForms(source: string): string[] {
+  const written = withoutQuery(source);
+  const out = [written];
+  const add = (...forms: string[]) => {
+    for (const f of forms) if (!out.includes(f)) out.push(f);
+  };
+  const tfr = /^tfr:\/\/([^/]*)\/(.+)$/.exec(written);
+  if (tfr) {
+    if (tfr[1]) add(`${tfr[1].toLowerCase()}/${tfr[2]}`);
+    else add(...DEFAULT_REGISTRIES.map((h) => `${h}/${tfr[2]}`));
+    if (!tfr[1]) add(tfr[2]!);
+    return out;
+  }
+  if (isHostlessModule(written)) {
+    add(...DEFAULT_REGISTRIES.map((h) => `${h}/${written}`));
+    return out;
+  }
+  const hostless = atDefaultRegistry(written);
+  if (hostless && isHostlessModule(hostless)) add(hostless, ...DEFAULT_REGISTRIES.map((h) => `${h}/${hostless}`));
+  return out;
+}
+
+/**
+ * The module a pin-free source names, written one way: a registry source
+ * with no host gets the public registry's (or the binary's), a host is
+ * lower-cased, and Terragrunt's `tfr:///ns/name/system` and
+ * `tfr://host/ns/name/system` become the plain registry form. Other sources
+ * (git, OCI, an archive URL) are kept as they are. Moved from
+ * ../pin/consumers.ts (chant#4599). A source through one of `proxies` is
+ * the upstream source it stands for.
+ */
+export function normalizeModuleSource(module: string, options?: SourceFormArg): string {
+  const o = optionsOf(options);
+  const binary = o.binary ?? "terraform";
+  const via = moduleThroughProxy(module, o.proxies);
+  let s = (via ? via.upstream : module).trim();
+  const tfr = /^tfr:\/\/([^/]*)\/(.*)$/.exec(s);
+  if (tfr) s = `${tfr[1] || defaultRegistryHost(binary)}/${tfr[2]}`;
+  const [base, ...rest] = s.split("//");
+  const parts = base!.split("/");
+  const registryShaped = (n: number) => parts.length === n && parts.every((p) => p.length > 0) && !base!.includes(":");
+  if (registryShaped(3) && !/[.]/.test(parts[0]!)) return [`${defaultRegistryHost(binary)}/${base}`, ...rest].join("//");
+  const reg = parts.length === 4 ? parseRegistrySource(base!) : undefined;
+  if (reg) return [`${reg.host}/${parts.slice(1).join("/")}`, ...rest].join("//");
+  return s;
+}
+
+/** A provider address's parts. */
+export interface ProviderAddress {
+  host: string;
+  namespace: string;
+  type: string;
+  /** `<host>/<namespace>/<type>`. */
+  address: string;
+}
+
+const PROVIDER_PART = /^[0-9a-z](?:[0-9a-z_-]{0,62}[0-9a-z])?$/;
+const HOST = /^[a-z0-9.-]+(?::\d+)?$/;
+
+/**
+ * A `required_providers` source in its full form, `<host>/<namespace>/<type>`,
+ * lower case, with the binary's default host when it names none and the
+ * `hashicorp` namespace when it names only a type. Undefined for a string
+ * that is not a provider address. A source through one of `proxies`
+ * (`<host>/<repo>__<ns>/<type>` on Artifactory) gives the upstream's address.
+ */
+export function parseProviderAddress(source: string, options?: SourceFormArg): ProviderAddress | undefined {
+  const o = optionsOf(options);
+  const binary = o.binary ?? "terraform";
+  const via = providerThroughProxy(source, o.proxies);
+  const parts = (via ? via.upstream : source).trim().toLowerCase().split("/");
+  let host: string, namespace: string, type: string;
+  if (parts.length === 1) [host, namespace, type] = [defaultRegistryHost(binary), "hashicorp", parts[0]!];
+  else if (parts.length === 2) [host, namespace, type] = [defaultRegistryHost(binary), parts[0]!, parts[1]!];
+  else if (parts.length === 3) [host, namespace, type] = parts as [string, string, string];
+  else return undefined;
+  if (!HOST.test(host) || !PROVIDER_PART.test(namespace) || !PROVIDER_PART.test(type)) return undefined;
+  return { host, namespace, type, address: `${host}/${namespace}/${type}` };
+}
+
+/** {@link parseProviderAddress}'s full form, or undefined. */
+export const providerAddress = (source: string, options?: SourceFormArg): string | undefined => parseProviderAddress(source, options)?.address;
+
+/**
+ * The provider `init` installs for a local name that has no
+ * `required_providers` entry: `hashicorp/<name>`. Undefined for the binary's
+ * own `terraform` provider (terraform_data, terraform_remote_state), which is
+ * never downloaded.
+ */
+export const impliedProvider = (local: string): string | undefined => (local === "terraform" ? undefined : `hashicorp/${local.toLowerCase()}`);
+
+/**
+ * Every form a provider address matches as: in lower case as written, and,
+ * when written without a host or at a default registry, its hostless form
+ * and its form at each default registry. A source through one of `proxies`
+ * adds the forms of its upstream address.
+ */
+export function providerAddressForms(source: string, options?: SourceFormOptions): string[] {
+  const out = plainProviderForms(source);
+  const via = providerThroughProxy(source, options?.proxies);
+  if (via) for (const f of plainProviderForms(via.upstream)) if (!out.includes(f)) out.push(f);
+  return out;
+}
+
+function plainProviderForms(source: string): string[] {
+  const a = source.trim().toLowerCase();
+  const out = [a];
+  const add = (...forms: string[]) => {
+    for (const f of forms) if (!out.includes(f)) out.push(f);
+  };
+  const hostless = a.split("/").length === 2 ? a : a.split("/").length === 1 && a ? `hashicorp/${a}` : atDefaultRegistry(a);
+  if (hostless && hostless.split("/").length === 2) add(hostless, ...DEFAULT_REGISTRIES.map((h) => `${h}/${hostless}`));
+  return out;
+}
+
+/**
+ * The provider addresses a `.terraform.lock.hcl` names, in the order
+ * written, as the lock file writes them (the full form). Read by text: the
+ * lock file is generated and every entry is a `provider "<address>" {` line.
+ */
+export function lockedProviders(lockText: string): string[] {
+  const out: string[] = [];
+  for (const m of lockText.matchAll(/^\s*provider\s+"([^"]+)"\s*\{/gm)) {
+    const a = m[1]!.toLowerCase();
+    if (!out.includes(a)) out.push(a);
+  }
+  return out;
+}
+

package/src/registry/targets/artifactory.test.ts

@@ -0,0 +1,248 @@
+// chant#4605: the Artifactory module target against the in-memory Artifactory fake. No request
+// leaves the process, and no credential appears in a log line, an error or a staged record.
+import { describe, expect, it } from "vitest";
+import { PublishTargetError } from "@intentius/chant/lifecycle/release-publish";
+import { createRedactor } from "@intentius/chant/lifecycle/publish-credentials";
+import { archiveContent, artifactoryBase, artifactoryDeployUrl, artifactoryModuleTarget, moduleZip, readModuleZipComment, readZip, type ModuleVersionContext } from "../index";
+import { createFakeArtifactory, type FakeArtifactoryOptions } from "./fake-artifactory";
+import { moduleTarball } from "./fake-http";
+
+const HOST = "acme.jfrog.io";
+const URL_ = `https://${HOST}/artifactory`;
+const USER = "ci-deployer";
+const TOKEN = "jfrog-access-0123456789abcdef";
+const READER = "ci-reader";
+const READER_TOKEN = "jfrog-reader-fedcba9876543210";
+const ENV = { JFROG_USER: USER, JFROG_ACCESS_TOKEN: TOKEN };
+
+const fake = (over: Partial<FakeArtifactoryOptions> = {}) =>
+  createFakeArtifactory({ host: HOST, repos: ["terraform-local"], users: { [USER]: TOKEN, [READER]: READER_TOKEN }, deployers: { "terraform-local": [USER] }, ...over });
+
+const clock = () => {
+  let t = 0;
+  const sleeps: number[] = [];
+  return { now: () => t, sleep: async (ms: number) => void (sleeps.push(ms), (t += ms)), sleeps };
+};
+
+const archiveOf = (text: string) => moduleTarball({ "main.tf": `# ${text}\n`, "README.md": "# vpc\n" });
+
+function context(env: NodeJS.ProcessEnv, over: Partial<ModuleVersionContext> = {}): ModuleVersionContext & { lines: string[] } {
+  const archive = over.archive ?? archiveOf("vpc 1.2.0");
+  const redactor = createRedactor(env, ["JFROG_ACCESS_TOKEN"]);
+  const lines: string[] = [];
+  return {
+    cwd: "/nowhere",
+    env,
+    redactor,
+    log: (line) => lines.push(redactor.redact(line)),
+    provenance: { oidc: false, ci: null, reason: "not in CI" },
+    tag: null,
+    address: { namespace: "acme", name: "vpc", system: "aws" },
+    version: "1.2.0",
+    revision: "3f2a9c1d0b7e3f2a9c1d0b7e3f2a9c1d0b7e3f2a",
+    content: archiveContent(archive),
+    archive,
+    ...over,
+    lines,
+  };
+}
+
+const target = (af: ReturnType<typeof fake>, opts = {}, c = clock()) =>
+  artifactoryModuleTarget({ url: URL_, repo: "terraform-local", ...opts }, { fetch: af.fetch, registryFetch: af.fetch, sleep: c.sleep, now: c.now });
+
+const errorOf = async (p: Promise<unknown>): Promise<PublishTargetError> => {
+  const e = await p.then(
+    () => undefined,
+    (err: unknown) => err,
+  );
+  expect(e).toBeInstanceOf(PublishTargetError);
+  return e as PublishTargetError;
+};
+
+const noSecret = (text: string) => {
+  expect(text).not.toContain(TOKEN);
+  expect(text).not.toContain(Buffer.from(`${USER}:${TOKEN}`).toString("base64"));
+};
+
+describe("Artifactory module target (chant#4605)", () => {
+  it("deploys the zip of the attested archive with one PUT, basic credentials and application/zip", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    const t = target(af);
+    expect(await t.plan(ctx)).toEqual([{ name: "terraform-local__acme/vpc/aws 1.2.0", destination: `${HOST}/terraform-local__acme/vpc/aws`, digest: ctx.content }]);
+    const staged = await t.stage(ctx);
+    const puts = af.requests.filter((r) => r.method === "PUT");
+    expect(puts).toHaveLength(1);
+    expect(puts[0]!.url).toBe(`${URL_}/terraform-local/acme/vpc/aws/1.2.0.zip`);
+    expect(puts[0]!.headers.authorization).toBe(`Basic ${Buffer.from(`${USER}:${TOKEN}`).toString("base64")}`);
+    expect(puts[0]!.headers["content-type"]).toBe("application/zip");
+    expect(Buffer.compare(Buffer.from(puts[0]!.body!), moduleZip(ctx.archive, ctx.content, ctx.revision))).toBe(0);
+    const zip = readZip(puts[0]!.body!);
+    expect(zip.entries.map((e) => e.path).sort()).toEqual(["README.md", "main.tf"]);
+    expect(readModuleZipComment(zip.comment)?.content).toBe(ctx.content);
+    noSecret(JSON.stringify(staged));
+
+    const published = await t.reveal(ctx, staged);
+    expect(published.destination).toBe(`${HOST}/terraform-local__acme/vpc/aws`);
+    expect(await t.versions(ctx.address, ctx)).toEqual(["1.2.0"]);
+    expect(await t.held(ctx.address, "1.2.0", ctx)).toEqual({ content: ctx.content, revision: ctx.revision });
+  });
+
+  it("reads through the registry protocol at <host>/<repo>__<ns>/<name>/<system>", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    const t = target(af);
+    await t.reveal(ctx, await t.stage(ctx));
+    const reads = af.requests.filter((r) => r.method === "GET").map((r) => new URL(r.url).pathname);
+    expect(reads).toContain("/.well-known/terraform.json");
+    expect(reads).toContain("/artifactory/api/terraform/v1/modules/terraform-local__acme/vpc/aws/versions");
+    expect(reads).toContain("/artifactory/api/terraform/v1/modules/terraform-local__acme/vpc/aws/1.2.0/download");
+  });
+
+  it("sends TF_TOKEN_<host> on the protocol reads, apart from the deploy credentials", async () => {
+    const af = fake();
+    const ctx = context({ ...ENV, TF_TOKEN_acme_jfrog_io: READER_TOKEN });
+    const t = target(af);
+    await t.reveal(ctx, await t.stage(ctx));
+    const versionsRead = af.requests.find((r) => r.url.endsWith("/versions"))!;
+    expect(versionsRead.headers.authorization).toBe(`Bearer ${READER_TOKEN}`);
+    const put = af.requests.find((r) => r.method === "PUT")!;
+    expect(put.headers.authorization).toMatch(/^Basic /);
+  });
+
+  it("waits for Artifactory to index the deploy before the reveal", async () => {
+    const af = fake({ indexAfter: 2 });
+    const c = clock();
+    const ctx = context(ENV);
+    const t = target(af, { pollMs: 1000 }, c);
+    await t.reveal(ctx, await t.stage(ctx));
+    expect(c.sleeps).toEqual([1000, 1000]);
+  });
+
+  it("stops after the wait with the index fix when the version is never listed", async () => {
+    const af = fake({ indexAfter: 100 });
+    const ctx = context(ENV);
+    const t = target(af, { waitMs: 3000, pollMs: 1000 });
+    const e = await errorOf(t.reveal(ctx, await t.stage(ctx)));
+    expect(e.code).toBe("publish-reveal-missing");
+    expect(e.fix).toContain("Recalculate Index");
+  });
+
+  it("sends nothing for a version already deployed with the same content", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    af.seed("terraform-local", "acme", "vpc", "aws", "1.2.0", moduleZip(ctx.archive, ctx.content, ctx.revision));
+    const staged = await target(af).stage(ctx);
+    expect(staged.items[0]!.present).toBe(true);
+    expect(af.requests.filter((r) => r.method === "PUT")).toHaveLength(0);
+    expect(ctx.lines.join("\n")).toContain("nothing sent");
+  });
+
+  it("refuses a version already deployed with other content", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    const other = archiveOf("something else");
+    af.seed("terraform-local", "acme", "vpc", "aws", "1.2.0", moduleZip(other, archiveContent(other)));
+    const e = await errorOf(target(af).stage(ctx));
+    expect(e.code).toBe("publish-version-held");
+    expect(af.requests.filter((r) => r.method === "PUT")).toHaveLength(0);
+  });
+
+  it("refuses a zip deployed by hand, which names no attested content", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    af.seed("terraform-local", "acme", "vpc", "aws", "1.2.0", Buffer.from("PK\x05\x06" + "\0".repeat(18), "latin1"));
+    expect((await errorOf(target(af).stage(ctx))).code).toBe("publish-version-held");
+  });
+
+  it("judges a 409 after a race by what the repository then holds", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    // The held read answers 404; another run's deploy then lands before ours, which gets 409.
+    const real = af.fetch;
+    let raced = false;
+    const racing: typeof af.fetch = async (url, init) => {
+      if (!raced && init?.method === "PUT") {
+        raced = true;
+        af.seed("terraform-local", "acme", "vpc", "aws", "1.2.0", moduleZip(ctx.archive, ctx.content, ctx.revision));
+      }
+      return real(url, init);
+    };
+    const t2 = artifactoryModuleTarget({ url: URL_, repo: "terraform-local" }, { fetch: racing, registryFetch: racing });
+    const staged = await t2.stage(ctx);
+    expect(staged.items[0]!.present).toBe(true);
+    expect(af.requests.filter((r) => r.method === "PUT")).toHaveLength(1);
+  });
+
+  it("names the variables on a refused deploy, never their values", async () => {
+    const af = fake({ deployers: { "terraform-local": [] } });
+    const ctx = context(ENV);
+    const e = await errorOf(target(af).stage(ctx));
+    expect(e.code).toBe("publish-credentials-refused");
+    expect(e.message).toContain("JFROG_USER");
+    expect(e.message).toContain("JFROG_ACCESS_TOKEN");
+    noSecret(`${e.message} ${e.fix}`);
+  });
+
+  it("gives a 401 for an unknown token as refused credentials", async () => {
+    const af = fake();
+    const ctx = context({ JFROG_USER: USER, JFROG_ACCESS_TOKEN: "jfrog-wrong-token-000" });
+    const e = await errorOf(target(af).stage(ctx));
+    expect(e.code).toBe("publish-credentials-refused");
+    expect(e.message).not.toContain("jfrog-wrong-token-000");
+  });
+
+  it("refuses to plan without a user or a token", async () => {
+    const af = fake();
+    const e1 = await errorOf(target(af).plan(context({ JFROG_ACCESS_TOKEN: TOKEN })));
+    expect(e1.code).toBe("publish-credentials-missing");
+    expect(e1.message).toContain("JFROG_USER");
+    const e2 = await errorOf(target(af).plan(context({ JFROG_USER: USER })));
+    expect(e2.message).toContain("JFROG_ACCESS_TOKEN");
+    expect(await target(af, { user: USER }).plan(context({ JFROG_ACCESS_TOKEN: TOKEN }))).toHaveLength(1);
+  });
+
+  it("reads the token and user from the declared variables", async () => {
+    const af = fake();
+    const ctx = context({ DEPLOY_USER: USER, DEPLOY_TOKEN: TOKEN });
+    const staged = await target(af, { userEnv: "DEPLOY_USER", tokenEnv: "DEPLOY_TOKEN" }).stage(ctx);
+    expect(staged.items[0]!.present).toBeUndefined();
+  });
+
+  it("fails a 5xx with the status and no credential", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    const failing: typeof af.fetch = async (url, init) => (init?.method === "PUT" ? new Response("upstream down", { status: 503 }) : af.fetch(url, init));
+    const e = await errorOf(artifactoryModuleTarget({ url: URL_, repo: "terraform-local" }, { fetch: failing, registryFetch: failing }).stage(ctx));
+    expect(e.code).toBe("publish-upload-failed");
+    expect(e.message).toContain("503");
+    noSecret(e.message);
+  });
+
+  it("answers a 401 from failNext on the first read as refused credentials", async () => {
+    const af = fake();
+    af.failNext(401);
+    const e = await errorOf(target(af).stage(context(ENV)));
+    expect(e.code).toBe("publish-credentials-refused");
+  });
+
+  it("refuses an archive whose content is not the release's", async () => {
+    const af = fake();
+    const ctx = context(ENV, { content: archiveContent(archiveOf("other")) });
+    expect((await errorOf(target(af).plan(ctx))).code).toBe("publish-artifact-digest-mismatch");
+  });
+
+  it("builds the deploy URL and the base", () => {
+    expect(artifactoryBase("https://acme.jfrog.io")).toBe("https://acme.jfrog.io/artifactory");
+    expect(artifactoryBase("https://acme.jfrog.io/artifactory/")).toBe("https://acme.jfrog.io/artifactory");
+    expect(artifactoryDeployUrl("https://h/artifactory", "terraform-local", { namespace: "acme", name: "vpc", system: "aws" }, "1.0.0")).toBe("https://h/artifactory/terraform-local/acme/vpc/aws/1.0.0.zip");
+  });
+
+  it("masks the credentials in every log line", async () => {
+    const af = fake();
+    const ctx = context(ENV);
+    const t = target(af);
+    await t.reveal(ctx, await t.stage(ctx));
+    noSecret(ctx.lines.join("\n"));
+  });
+});