@intentius/chant 0.57.0 → 0.58.0

package/src/audit/secrets.ts

@@ -173,7 +173,13 @@ export function parseSecretsConfig(content: string): SecretsScanOptions {
 
 // ── Shannon entropy ─────────────────────────────────────────────────────
 
-function shannonEntropy(s: string): number {
+/**
+ * Shannon entropy in bits per character. Exported so a lexicon's own
+ * secret-shape heuristic can score a candidate on the same scale this
+ * module's SEC010 catch-all uses (the terraform lexicon's
+ * `lint/secret-shape.ts`, #2110) instead of inventing a second one.
+ */
+export function shannonEntropy(s: string): number {
   const freq = new Map<string, number>();
   for (const ch of s) freq.set(ch, (freq.get(ch) ?? 0) + 1);
   let entropy = 0;
@@ -188,8 +194,14 @@ function shannonEntropy(s: string): number {
 
 const PLACEHOLDER_RE = /example|placeholder|dummy|changeme|change-me|redacted|sample|fixme|todo|xxxxxxxx|your[-_]?(api[-_]?)?(key|token|secret)|fake[-_]?(key|token|secret)|test[-_]?(key|token|secret)/i;
 
-/** Cheap, format-agnostic "this doesn't look like a real secret" filter. */
-function looksLikePlaceholder(value: string): boolean {
+/**
+ * Cheap, format-agnostic "this doesn't look like a real secret" filter.
+ * Exported for the same reason {@link shannonEntropy} is: the placeholder
+ * vocabulary a scanner has to ignore (`changeme`, `example`, `your-api-key`,
+ * a run of one repeated character) is the same wherever the candidate came
+ * from, and a second copy of this list would drift from this one.
+ */
+export function looksLikePlaceholder(value: string): boolean {
   if (PLACEHOLDER_RE.test(value)) return true;
   const uniq = new Set(value.toLowerCase()).size;
   if (value.length > 8 && uniq <= 2) return true; // e.g. "0000000000000000"

package/src/audit/terraform-state.test.ts

@@ -0,0 +1,144 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { describe, expect, test } from "vitest";
+import {
+  auditTerraformState,
+  gitignoreCoversTerraformState,
+  isTerraformStateFile,
+  isTerraformStatePath,
+  isTerraformWorkDir,
+  type ScannableFile,
+} from "./terraform-state";
+import { collectCandidates } from "./discover";
+import { RULE_CATALOG, RULE_CATEGORY } from "./catalog";
+
+const file = (path: string): ScannableFile => ({ path, content: "" });
+
+function tmpRepo(): string {
+  return mkdtempSync(join(tmpdir(), "chant-tf023-"));
+}
+
+describe("the TF023 path detectors", () => {
+  test.each([
+    "terraform.tfstate",
+    "terraform.tfstate.backup",
+    "infra/prod/terraform.tfstate",
+    "infra/prod.tfstate",
+    "infra/prod.tfstate.backup",
+  ])("%s is state", (path) => {
+    expect(isTerraformStateFile(path)).toBe(true);
+    expect(isTerraformStatePath(path)).toBe(true);
+  });
+
+  test.each(["main.tf", "terraform.tfvars", "state.json", "docs/tfstate.md"])("%s is not", (path) => {
+    expect(isTerraformStateFile(path)).toBe(false);
+  });
+
+  test("the working directory is matched as a path segment, at any depth", () => {
+    expect(isTerraformWorkDir(".terraform")).toBe(true);
+    expect(isTerraformWorkDir(".terraform/providers/registry.terraform.io/hashicorp/aws/5.0.0")).toBe(true);
+    expect(isTerraformWorkDir("infra/prod/.terraform")).toBe(true);
+    expect(isTerraformWorkDir("infra/terraform/main.tf")).toBe(false);
+  });
+});
+
+describe("auditTerraformState", () => {
+  test("reports a committed state file as an error, without reading it", () => {
+    const findings = auditTerraformState([file("infra/prod/terraform.tfstate")]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0].checkId).toBe("TF023");
+    expect(findings[0].severity).toBe("error");
+    expect(findings[0].lexicon).toBe("terraform");
+    expect(findings[0].file).toBe("infra/prod/terraform.tfstate");
+    expect(findings[0].message).toContain("secret store");
+  });
+
+  test("reports a backup state file too", () => {
+    expect(auditTerraformState([file("terraform.tfstate.backup")])).toHaveLength(1);
+  });
+
+  test("reports a `.terraform` directory once, not once per file inside it", () => {
+    const findings = auditTerraformState([
+      file(".terraform"),
+      file(".terraform/modules/modules.json"),
+      file(".terraform/providers/registry.terraform.io/hashicorp/aws/5.0.0/terraform-provider-aws"),
+    ]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0].file).toBe(".terraform");
+    expect(findings[0].entity).toBe(".terraform");
+  });
+
+  test("reports each root's own working directory separately", () => {
+    const findings = auditTerraformState([file("infra/prod/.terraform"), file("infra/staging/.terraform")]);
+    expect(findings.map((f) => f.file)).toEqual(["infra/prod/.terraform", "infra/staging/.terraform"]);
+  });
+
+  test("says nothing about a repository with neither", () => {
+    expect(auditTerraformState([{ path: "main.tf", content: 'resource "null_resource" "a" {}' }])).toEqual([]);
+  });
+
+  test("is catalogued in core, as a merge-worthy security rule", () => {
+    expect(RULE_CATALOG.TF023.tier).toBe("merge-worthy");
+    expect(RULE_CATALOG.TF023.category).toBe("security");
+    expect(RULE_CATEGORY.TF023).toBe("security");
+    expect(RULE_CATALOG.TF023.authority?.[0].url).toContain("#gitignore");
+  });
+});
+
+describe("gitignoreCoversTerraformState", () => {
+  test.each([
+    ["*.tfstate", "terraform.tfstate"],
+    ["*.tfstate.*", "terraform.tfstate.backup"],
+    [".terraform/", ".terraform"],
+    [".terraform*", ".terraform"],
+    ["/terraform.tfstate", "terraform.tfstate"],
+    ["**/.terraform", "infra/prod/.terraform"],
+    ["# comment\n\n*.tfstate\n", "infra/prod/terraform.tfstate"],
+  ])("%s covers %s", (gitignore, path) => {
+    expect(gitignoreCoversTerraformState(gitignore, path)).toBe(true);
+  });
+
+  test.each([
+    ["node_modules/\ndist/", "terraform.tfstate"],
+    ["*.tfvars", "terraform.tfstate"],
+    ["!terraform.tfstate", "terraform.tfstate"],
+  ])("%s does not cover %s", (gitignore, path) => {
+    expect(gitignoreCoversTerraformState(gitignore, path)).toBe(false);
+  });
+});
+
+describe("discovery hands TF023 its paths, and nothing else changes", () => {
+  test("a state file becomes a candidate with no content, so nothing reads it", () => {
+    const dir = tmpRepo();
+    writeFileSync(join(dir, "main.tf"), 'resource "null_resource" "a" {}\n');
+    writeFileSync(join(dir, "terraform.tfstate"), JSON.stringify({ version: 4, resources: [] }));
+    const candidates = collectCandidates(dir);
+    const state = candidates.find((c) => c.path === "terraform.tfstate");
+    expect(state).toBeDefined();
+    expect(state!.content).toBe("");
+    expect(auditTerraformState(candidates)).toHaveLength(1);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  test("the walk records `.terraform` without descending into it", () => {
+    const dir = tmpRepo();
+    writeFileSync(join(dir, "main.tf"), 'resource "null_resource" "a" {}\n');
+    mkdirSync(join(dir, ".terraform", "providers"), { recursive: true });
+    writeFileSync(join(dir, ".terraform", "providers", "huge.json"), "{}");
+    const candidates = collectCandidates(dir);
+    expect(candidates.map((c) => c.path).sort()).toEqual([".terraform", "main.tf"]);
+    expect(auditTerraformState(candidates)).toHaveLength(1);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  test("a gitignored state file is on disk, not in the repository, so it is dropped", () => {
+    const dir = tmpRepo();
+    writeFileSync(join(dir, ".gitignore"), ".terraform/\n*.tfstate\n*.tfstate.*\n");
+    writeFileSync(join(dir, "main.tf"), 'resource "null_resource" "a" {}\n');
+    writeFileSync(join(dir, "terraform.tfstate"), "{}");
+    mkdirSync(join(dir, ".terraform"), { recursive: true });
+    expect(auditTerraformState(collectCandidates(dir))).toEqual([]);
+    rmSync(dir, { recursive: true, force: true });
+  });
+});

package/src/audit/terraform-state.ts

@@ -0,0 +1,131 @@
+/**
+ * TF023 (#2110): Terraform state, or the `.terraform/` working directory,
+ * committed to the repository.
+ *
+ * Every other rule in the TF family is a post-synth check over a parsed root
+ * module, contributed by the terraform lexicon. This one is not about HCL at
+ * all: the finding is that a path exists in version control, so there is
+ * nothing for `chant build` to check and nothing for the lexicon's parser to
+ * read. It therefore takes the shape `secrets.ts` established and `wrangler.ts`
+ * and `nginx.ts` copied, a path detector plus a check over the discovered file
+ * list, called from the audit CLI beside `scanForSecrets`, with its catalog
+ * entry in core's own `RULE_CATALOG` (./catalog.ts) rather than a lexicon's.
+ * It runs whether or not the terraform lexicon is installed.
+ *
+ * A state file is a committed secret store: it records every attribute of every
+ * managed resource, including the ones providers mark sensitive, in plaintext
+ * JSON. `.terraform/` is milder (providers and vendored modules) but is still
+ * hundreds of megabytes of machine-generated content, and it can hold a
+ * `terraform.tfstate` of its own for the workspace-local backend.
+ *
+ * Pure: no fs, no network, no Node-only global, so it is Workers-safe (epic #350). The
+ * walk that decides which paths reach here is `discover.ts`'s
+ * `collectCandidates`, which is also where a locally ignored path is dropped:
+ * see `isTerraformStatePath` there and `gitignoreCoversTerraformState` below.
+ */
+
+import type { AuditFinding } from "./core";
+
+/** The minimal file shape the check needs (matches `discover.ts`'s `RepoFile`). */
+export interface ScannableFile {
+  path: string;
+  content: string;
+}
+
+/** `terraform.tfstate`, `terraform.tfstate.backup`, and any other `*.tfstate`. */
+export function isTerraformStateFile(path: string): boolean {
+  const name = path.split("/").pop() ?? path;
+  return /\.tfstate(\.backup)?$/i.test(name);
+}
+
+/** The `.terraform` working directory itself, or anything inside it. */
+export function isTerraformWorkDir(path: string): boolean {
+  const parts = path.split("/");
+  return parts.includes(".terraform");
+}
+
+/** Either of the two shapes TF023 reports. */
+export function isTerraformStatePath(path: string): boolean {
+  return isTerraformStateFile(path) || isTerraformWorkDir(path);
+}
+
+/**
+ * Does a `.gitignore` body already exclude Terraform state? Deliberately
+ * narrow: it recognises the handful of patterns the style guide's own
+ * `.gitignore` section and `github/gitignore`'s `Terraform.gitignore` use
+ * (`*.tfstate`, `*.tfstate.*`, `.terraform/`, `.terraform*`, with an optional
+ * leading slash or a leading double-star), not the whole gitignore grammar. A repository that
+ * writes something more exotic gets a finding it can suppress; the alternative,
+ * a partial gitignore engine, would be wrong in quieter ways.
+ *
+ * Used by the local walk only (`discover.ts`), where "on disk" and "tracked by
+ * git" are different questions and nearly every Terraform working tree has an
+ * ignored `.terraform/` in it. A fetched repository's file list is already
+ * exactly the tracked files, so nothing filters it.
+ */
+function escapeRegExp(literal: string): string {
+  return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+export function gitignoreCoversTerraformState(gitignore: string, path: string): boolean {
+  const name = path.split("/").pop() ?? path;
+  for (const raw of gitignore.split("\n")) {
+    const line = raw.trim();
+    if (line === "" || line.startsWith("#") || line.startsWith("!")) continue;
+    const pattern = line.replace(/^\*\*\//, "").replace(/^\//, "").replace(/\/$/, "");
+    if (pattern === "" || pattern.includes("/")) continue;
+    // `*` matches within one path segment, as git's own glob does. The pattern
+    // is compared against the file's own name and against every segment of its
+    // path, which is what makes `.terraform/` match `infra/prod/.terraform`.
+    const re = new RegExp(`^${pattern.split("*").map(escapeRegExp).join("[^/]*")}$`);
+    if (re.test(name) || path.split("/").some((segment) => re.test(segment))) return true;
+  }
+  return false;
+}
+
+/**
+ * Report TF023 for every committed state file and `.terraform/` directory in
+ * the file list. One finding per path, and one for the `.terraform` directory
+ * as a whole rather than per file inside it: a vendored provider binary is not
+ * a separate finding from the directory that holds it.
+ */
+export function auditTerraformState(files: ScannableFile[]): AuditFinding[] {
+  const findings: AuditFinding[] = [];
+  const workDirs = new Set<string>();
+
+  for (const file of files) {
+    if (isTerraformWorkDir(file.path)) {
+      const parts = file.path.split("/");
+      const dir = parts.slice(0, parts.indexOf(".terraform") + 1).join("/");
+      if (workDirs.has(dir)) continue;
+      workDirs.add(dir);
+      findings.push({
+        checkId: "TF023",
+        severity: "warning",
+        message:
+          `\`${dir}\` is committed to the repository. Terraform's working directory holds downloaded ` +
+          "providers and vendored child modules, and the local backend keeps its state there too. " +
+          "Delete it from version control and add `.terraform/` to .gitignore.",
+        file: dir,
+        lexicon: "terraform",
+        entity: ".terraform",
+      });
+      continue;
+    }
+    if (!isTerraformStateFile(file.path)) continue;
+    findings.push({
+      checkId: "TF023",
+      severity: "error",
+      message:
+        `\`${file.path}\` is committed to the repository. A state file records every attribute of every ` +
+        "managed resource in plaintext, including the ones providers mark sensitive, so a committed " +
+        "state file is a committed secret store. Remove it from version control, add `*.tfstate` and " +
+        "`*.tfstate.backup` to .gitignore, move the state to a remote backend, and rotate anything it held.",
+      file: file.path,
+      lexicon: "terraform",
+      entity: file.path.split("/").pop() ?? file.path,
+    });
+  }
+
+  return findings;
+}

package/src/build.ts

@@ -3,12 +3,14 @@ import type { Serializer, SerializerResult } from "./serializer";
 import type { OwnershipMarker } from "./ownership";
 import type { BuildError, DiscoveryErrorType } from "./errors";
 import type { IntrinsicDef, BuildRootContribution, BuildRootContributor } from "./lexicon";
-import type { BuildParamProvenance } from "./provenance";
+import { getProvenance, type BuildParamProvenance } from "./provenance";
+import { foldProvenanceOfEntities, type FoldProvenance } from "./fold-provenance";
 import { DiscoveryError, BuildError as BuildErrorClass } from "./errors";
 import { LexiconOutput, isLexiconOutput } from "./lexicon-output";
 import { isSecretDeclaration } from "./secret-provenance";
 import { splitReceiptEntities } from "./effect-receipt";
 import { isScenario } from "./lifecycle/scenario";
+import { isOpEntity } from "./op/resource";
 import { AttrRef } from "./attrref";
 import { isAttrRefLike } from "./utils";
 import { isChildProject, type ChildProjectInstance } from "./child-project";
@@ -282,6 +284,22 @@ export interface BuildResult {
    * supplies none.
    */
   buildParams: BuildParamProvenance[];
+
+  /**
+   * chant #2161 — emitted property path to origin, per entity: which composite
+   * parameter produced the field, which literal the composite fixes, which the
+   * author declared directly, and which this build could not attribute.
+   *
+   * Beside the output on purpose, and never inside it. Nothing here is passed
+   * to a serializer, so it cannot reach the document an applier writes; see
+   * ./fold-provenance.ts's module doc and `fold-provenance-exclusion.test.ts`.
+   *
+   * Every entity with properties appears. The origins are as fine as the build
+   * managed: a run build, or a fold build over a factory outside the
+   * interpretable subset, records `unknown` for a composite's fields rather
+   * than guessing.
+   */
+  foldProvenance: FoldProvenance;
 }
 
 /**
@@ -308,6 +326,13 @@ export function partitionByLexicon(
     // Plan scenarios (#1292) are serializer-neutral the same way: a checkable
     // expectation the CLI reads off the entity map, never output.
     if (isScenario(entity)) continue;
+    // Op declarations (#2118) too. An Op's build output is its `op.json` IR,
+    // and core writes that itself from the entity map (`./cli/commands/build.ts`)
+    // rather than routing it through a lexicon serializer — the Op model, the
+    // executor and the IR are all core's, and no lexicon renders an Op entity
+    // into its own manifest. Keeping them out of every partition means no
+    // "No serializer found" warning fires for core's own `chant` lexicon.
+    if (isOpEntity(entity)) continue;
     const lexicon = entity.lexicon;
     if (!partitions.has(lexicon)) {
       partitions.set(lexicon, new Map());
@@ -852,6 +877,11 @@ async function buildFromDiscoveryResult(
     foldDecisions: discoveryResult.foldDecisions,
     lexiconVersions: { ...(options?.lexiconVersions ?? {}) },
     buildParams: options?.buildParams ?? [],
+    // chant #2161 — read off the entities AFTER every step that can change
+    // them (the shared-props merge in `expandComposite`, the build-root merge
+    // above), and deliberately after Step 7, so nothing about computing it can
+    // reach a serializer.
+    foldProvenance: foldProvenanceOfEntities(discoveryResult.entities, getProvenance),
   };
 }
 

package/src/claimed-fields.test.ts

@@ -0,0 +1,145 @@
+import { describe, test, expect } from "vitest";
+import {
+  NO_CLAIMED_FIELDS,
+  claimedFieldPaths,
+  claimedFieldsFromPaths,
+  claimedFieldsOfProps,
+  claimedFieldsOfTree,
+  classifyLiveField,
+  heldBy,
+  isClaimed,
+} from "./claimed-fields";
+import { UNRESOLVED, type DeepNormalizationHooks } from "./deep-observation";
+
+describe("claimedFieldsOfProps", () => {
+  test("walks props to leaf paths in the diff's grammar", () => {
+    const claimed = claimedFieldsOfProps(
+      { spec: { replicas: 2, template: { spec: { containers: [{ name: "app", image: "a:1" }] } } } },
+      { entityType: "K8s::Apps::Deployment" },
+    );
+    expect(claimedFieldPaths(claimed)).toEqual([
+      "spec.replicas",
+      "spec.template.spec.containers[0].image",
+      "spec.template.spec.containers[0].name",
+    ]);
+  });
+
+  test("addresses set-like arrays by the lexicon's own order key", () => {
+    const hooks: DeepNormalizationHooks = {
+      orderKey: (el) => (el.pattern === "Tags" ? String((el.element as { Key: string }).Key) : undefined),
+    };
+    const claimed = claimedFieldsOfProps(
+      { Tags: [{ Key: "env", Value: "prod" }] },
+      { entityType: "AWS::S3::Bucket", hooks },
+    );
+    expect(claimedFieldPaths(claimed)).toEqual(["Tags[#env].Key", "Tags[#env].Value"]);
+  });
+
+  test("a pruned property is not claimed", () => {
+    // The hooks decide what is even in the tree, so the claim inherits them
+    // rather than second-guessing them: a field the lexicon subtracts on both
+    // sides is not a field chant is asserting anything about.
+    const hooks: DeepNormalizationHooks = { prune: (node) => node.pattern.startsWith("status") };
+    const claimed = claimedFieldsOfProps(
+      { spec: { replicas: 2 }, status: { readyReplicas: 2 } },
+      { entityType: "T", hooks },
+    );
+    expect(claimedFieldPaths(claimed)).toEqual(["spec.replicas"]);
+  });
+
+  test("an unevaluated intrinsic is still claimed", () => {
+    // chant set the field; it just cannot say what the reference resolves to.
+    const claimed = claimedFieldsOfProps({ BucketName: UNRESOLVED }, { entityType: "T" });
+    expect(isClaimed(claimed, "BucketName")).toBe(true);
+  });
+
+  test("an empty declaration claims nothing", () => {
+    expect(claimedFieldPaths(claimedFieldsOfProps({}, { entityType: "T" }))).toEqual([]);
+    expect(claimedFieldPaths(NO_CLAIMED_FIELDS)).toEqual([]);
+  });
+
+  test("agrees with the tree form on an already-normalized tree", () => {
+    const props = { spec: { replicas: 2 } };
+    expect(claimedFieldPaths(claimedFieldsOfProps(props, { entityType: "T" }))).toEqual(
+      claimedFieldPaths(claimedFieldsOfTree(props, { entityType: "T" })),
+    );
+  });
+});
+
+describe("isClaimed", () => {
+  const claimed = claimedFieldsFromPaths(["Tags[0].Value", "spec.replicas"]);
+
+  test("matches exactly", () => {
+    expect(isClaimed(claimed, "spec.replicas")).toBe(true);
+  });
+
+  test("does not match by index-erased pattern", () => {
+    // A pattern match would answer "declared" for a path with no declared value
+    // to compare against, which is neither of the two declared classifications.
+    expect(isClaimed(claimed, "Tags[3].Value")).toBe(false);
+  });
+
+  test("does not match an ancestor's claim", () => {
+    // Declaring `spec.replicas` says nothing about `spec.strategy.type`.
+    expect(isClaimed(claimed, "spec.strategy.type")).toBe(false);
+  });
+
+  test("an absent claim claims nothing", () => {
+    expect(isClaimed(undefined, "spec.replicas")).toBe(false);
+  });
+});
+
+describe("classifyLiveField", () => {
+  const claimed = claimedFieldsFromPaths(["spec.replicas"]);
+
+  test("declared and equal", () => {
+    expect(
+      classifyLiveField({ claimed, path: "spec.replicas", declaredValue: 2, liveValue: 2 }),
+    ).toBe("declared-equal");
+  });
+
+  test("declared and different", () => {
+    expect(
+      classifyLiveField({ claimed, path: "spec.replicas", declaredValue: 2, liveValue: 5 }),
+    ).toBe("declared-changed");
+  });
+
+  test("undeclared", () => {
+    expect(
+      classifyLiveField({ claimed, path: "metadata.labels.team", declaredValue: undefined, liveValue: "platform" }),
+    ).toBe("undeclared");
+  });
+
+  test("structural equality, not identity", () => {
+    const shaped = claimedFieldsFromPaths(["spec.selector"]);
+    expect(
+      classifyLiveField({
+        claimed: shaped,
+        path: "spec.selector",
+        declaredValue: { app: "web" },
+        liveValue: { app: "web" },
+      }),
+    ).toBe("declared-equal");
+  });
+});
+
+describe("heldBy", () => {
+  test("the substrate's manager answers where it has one", () => {
+    expect(heldBy({ "spec.replicas": "hpa-controller" }, "spec.replicas")).toEqual({
+      holder: "hpa-controller",
+      source: "field-manager",
+    });
+  });
+
+  test("the claim answers where the substrate records nothing", () => {
+    expect(heldBy(undefined, "spec.replicas")).toEqual({ source: "claimed-fields" });
+  });
+
+  test("the claim answers for a path the ownership metadata does not cover", () => {
+    // Kubernetes' own case: a keyed list segment has no `managedFields` entry
+    // under that path, so the declaration is the only witness left.
+    expect(heldBy({ "spec.replicas": "hpa-controller" }, "spec.template.spec.containers[#sidecar].image")).toEqual({
+      source: "claimed-fields",
+    });
+  });
+});