@intentius/chant 0.49.0 → 0.50.0

package/src/lifecycle/scenario-eval.test.ts

@@ -0,0 +1,199 @@
+import { describe, test, expect } from "vitest";
+import { evaluateScenario } from "./scenario-eval";
+import type { ChangeSet, ChangeSetEntry } from "./change-set";
+
+function entry(overrides: Partial<ChangeSetEntry> & Pick<ChangeSetEntry, "name" | "action">): ChangeSetEntry {
+  return {
+    type: "AWS::S3::Bucket",
+    evidence: { declared: true, inSnapshot: true, live: true, observed: true },
+    ownership: "unknown",
+    ...overrides,
+  };
+}
+
+function cs(entries: ChangeSetEntry[], env = "prod"): ChangeSet {
+  return { env, entries };
+}
+
+describe("evaluateScenario — noop", () => {
+  test("passes when the plan proposes nothing", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "a", action: "noop" })]), { noop: true });
+    expect(verdict.pass).toBe(true);
+    expect(verdict.checks).toEqual([{ clause: "noop", pass: true }]);
+  });
+
+  test("fails and names the offending resource when the plan proposes a create", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "newBucket", action: "create" })]), { noop: true });
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "noop")!;
+    expect(check.pass).toBe(false);
+    expect(check.detail).toMatch(/1 create/);
+    expect(check.detail).toMatch(/newBucket/);
+  });
+
+  test("fails on an update", () => {
+    const verdict = evaluateScenario(
+      cs([entry({ name: "web", action: "update", deltas: [{ path: "size", oldValue: 1, newValue: 2 }] })]),
+      { noop: true },
+    );
+    expect(verdict.pass).toBe(false);
+  });
+
+  test("fails on a delete", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "legacy", action: "delete", ownership: "owned" })]), {
+      noop: true,
+    });
+    expect(verdict.pass).toBe(false);
+  });
+
+  test("ignores adopt/runtime/unobserved rows", () => {
+    const verdict = evaluateScenario(
+      cs([
+        entry({ name: "orphan", action: "adopt" }),
+        entry({ name: "child", action: "runtime", runtimeOwner: "deployment" }),
+      ]),
+      { noop: true },
+    );
+    expect(verdict.pass).toBe(true);
+  });
+});
+
+describe("evaluateScenario — counts", () => {
+  test("passes on an exact match across all three", () => {
+    const verdict = evaluateScenario(
+      cs([
+        entry({ name: "a", action: "create" }),
+        entry({ name: "b", action: "update" }),
+        entry({ name: "c", action: "delete", ownership: "owned" }),
+      ]),
+      { create: 1, update: 1, delete: 1 },
+    );
+    expect(verdict.pass).toBe(true);
+  });
+
+  test("fails and reports the mismatched clause only", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "a", action: "create" }), entry({ name: "b", action: "create" })]), {
+      create: 1,
+    });
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "create")!;
+    expect(check.detail).toMatch(/expected 1 create, plan proposes 2/);
+    expect(check.detail).toMatch(/a.*b|b.*a/);
+  });
+
+  test("an omitted clause is unconstrained", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "a", action: "update" })]), { create: 0 });
+    expect(verdict.pass).toBe(true);
+    expect(verdict.checks.map((c) => c.clause)).toEqual(["create"]);
+  });
+
+  test("zero is a real assertion, not 'omitted'", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "a", action: "delete", ownership: "owned" })]), { delete: 0 });
+    expect(verdict.pass).toBe(false);
+  });
+});
+
+describe("evaluateScenario — deletes", () => {
+  test("passes when every named delete matches name and ownership exactly, with no extras", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "legacy", action: "delete", ownership: "owned" })]), {
+      deletes: [{ name: "legacy", ownership: "owned" }],
+    });
+    expect(verdict.pass).toBe(true);
+  });
+
+  test("fails and names the resource when an expected delete is not proposed", () => {
+    const verdict = evaluateScenario(cs([]), { deletes: [{ name: "legacy", ownership: "owned" }] });
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "deletes")!;
+    expect(check.detail).toMatch(/"legacy"/);
+    expect(check.detail).toMatch(/not proposed/);
+  });
+
+  test("fails and names the resource on an ownership mismatch", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "legacy", action: "delete", ownership: "foreign" })]), {
+      deletes: [{ name: "legacy", ownership: "owned" }],
+    });
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "deletes")!;
+    expect(check.detail).toMatch(/"legacy"/);
+    expect(check.detail).toMatch(/expected ownership "owned", plan proposes ownership "foreign"/);
+  });
+
+  test("fails and names the resource on an unexpected extra delete", () => {
+    const verdict = evaluateScenario(
+      cs([
+        entry({ name: "legacy", action: "delete", ownership: "owned" }),
+        entry({ name: "surprise", action: "delete", ownership: "owned" }),
+      ]),
+      { deletes: [{ name: "legacy", ownership: "owned" }] },
+    );
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "deletes")!;
+    expect(check.detail).toMatch(/"surprise"/);
+    expect(check.detail).toMatch(/not expected/);
+  });
+});
+
+describe("evaluateScenario — unobserved", () => {
+  test("\"refuse\" fails on any unobserved row", () => {
+    const verdict = evaluateScenario(
+      cs([entry({ name: "web", action: "unobserved", unobservedReason: "no-credentials" })]),
+      { unobserved: "refuse" },
+    );
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "unobserved")!;
+    expect(check.detail).toMatch(/web/);
+    expect(check.detail).toMatch(/no credentials/);
+  });
+
+  test("\"refuse\" passes when nothing is unobserved", () => {
+    const verdict = evaluateScenario(cs([entry({ name: "web", action: "noop" })]), { unobserved: "refuse" });
+    expect(verdict.pass).toBe(true);
+  });
+
+  test("{ allow } tolerates only the named entities", () => {
+    const verdict = evaluateScenario(
+      cs([
+        entry({ name: "web", action: "unobserved", unobservedReason: "no-binding" }),
+        entry({ name: "db", action: "unobserved", unobservedReason: "no-binding" }),
+      ]),
+      { unobserved: { allow: ["web"] } },
+    );
+    expect(verdict.pass).toBe(false);
+    const check = verdict.checks.find((c) => c.clause === "unobserved")!;
+    expect(check.detail).toMatch(/db/);
+    expect(check.detail).not.toMatch(/web/);
+  });
+
+  test("{ allow } passes when every unobserved entity is named", () => {
+    const verdict = evaluateScenario(
+      cs([entry({ name: "web", action: "unobserved", unobservedReason: "no-binding" })]),
+      { unobserved: { allow: ["web"] } },
+    );
+    expect(verdict.pass).toBe(true);
+  });
+
+  test("omitted is unconstrained — unobserved rows neither pass nor fail it", () => {
+    const verdict = evaluateScenario(
+      cs([entry({ name: "web", action: "unobserved", unobservedReason: "read-failed" })]),
+      { create: 0 },
+    );
+    expect(verdict.pass).toBe(true);
+    expect(verdict.checks.map((c) => c.clause)).toEqual(["create"]);
+  });
+});
+
+describe("evaluateScenario — composed clauses", () => {
+  test("every clause is evaluated and reported, even when only one fails", () => {
+    const verdict = evaluateScenario(
+      cs([
+        entry({ name: "legacy", action: "delete", ownership: "owned" }),
+        entry({ name: "web", action: "unobserved", unobservedReason: "read-failed" }),
+      ]),
+      { create: 0, deletes: [{ name: "legacy", ownership: "owned" }], unobserved: "refuse" },
+    );
+    expect(verdict.pass).toBe(false);
+    const byClause = Object.fromEntries(verdict.checks.map((c) => [c.clause, c.pass]));
+    expect(byClause).toEqual({ create: true, deletes: true, unobserved: false });
+  });
+});

package/src/lifecycle/scenario-eval.ts

@@ -0,0 +1,158 @@
+/**
+ * Pure evaluation of a plan scenario's `expect` clause against a `ChangeSet`
+ * (#1292).
+ *
+ * `evaluateScenario` reads only what the caller hands it — a `ChangeSet` and
+ * a `ScenarioExpect` — and does no I/O. Building the change set (reading a
+ * fixture, replaying it in place of a live observation) is the CLI handler's
+ * job (../cli/handlers/scenario.ts); this module is the part that stays
+ * testable without a filesystem, a build, or a fixture.
+ *
+ * Built on the same primitives `chant lifecycle plan` already computes from:
+ * `summarize` for counts, a filter over `cs.entries` for named deletes and
+ * ownership, and `evaluateUnobservedGate` (./unobserved-gate.ts, #1568) for
+ * the unobserved clause.
+ */
+
+import { summarize, type ChangeSet } from "./change-set";
+import { evaluateUnobservedGate, type UnobservedGateFinding } from "./unobserved-gate";
+import { unobservedReasonText } from "../observation";
+import type { ScenarioDeleteExpectation, ScenarioExpect, ScenarioUnobservedPolicy } from "./scenario";
+
+/**
+ * One clause's verdict — always present in {@link ScenarioVerdict.checks}, in
+ * declaration order, whether it passed or not. A scenario with no failing
+ * clauses is legible too: every check that ran, and that it held.
+ */
+export interface ScenarioCheckResult {
+  /** Which `expect` clause this is (`"noop"`, `"create"`, `"deletes"`, …). */
+  clause: string;
+  pass: boolean;
+  /** Present when `pass` is false — what was expected vs what the plan proposes, naming resources for delete/ownership failures. */
+  detail?: string;
+}
+
+/** The result of evaluating one scenario's `expect` against a change set. */
+export interface ScenarioVerdict {
+  pass: boolean;
+  /** Every clause `expect` declared, evaluated — see {@link ScenarioCheckResult}. */
+  checks: ScenarioCheckResult[];
+}
+
+/**
+ * Evaluate `expect` against `cs`. Pure: reads `cs` and `expect`, computes
+ * nothing else. Every clause present on `expect` is checked independently and
+ * every one contributes to `checks`; `pass` is true only when all of them are.
+ */
+export function evaluateScenario(cs: ChangeSet, expect: ScenarioExpect): ScenarioVerdict {
+  const counts = summarize(cs);
+  const checks: ScenarioCheckResult[] = [];
+
+  if (expect.noop) {
+    // An effect receipt's fire (#1832) is a real proposed action — a
+    // migration or seed job about to run — even though it never counts
+    // toward the create/update/delete triad `buildChangeSet` classifies
+    // ordinary resources into. `noop: true` claims the plan does nothing;
+    // a pending effect fire is something, so it offends the claim exactly
+    // like a create/update/delete would.
+    const offenders = { create: counts.create, update: counts.update, delete: counts.delete, effect: counts.effect };
+    const clean = offenders.create === 0 && offenders.update === 0 && offenders.delete === 0 && offenders.effect === 0;
+    checks.push({
+      clause: "noop",
+      pass: clean,
+      ...(clean
+        ? {}
+        : {
+            detail:
+              `expected noop (no create/update/delete/effect) but the plan proposes ` +
+              `${offenders.create} create, ${offenders.update} update, ${offenders.delete} delete, ${offenders.effect} effect — ` +
+              nameList(cs, ["create", "update", "delete", "effect"]),
+          }),
+    });
+  }
+
+  for (const clause of ["create", "update", "delete"] as const) {
+    const want = expect[clause];
+    if (want === undefined) continue;
+    const got = counts[clause];
+    const pass = got === want;
+    checks.push({
+      clause,
+      pass,
+      ...(pass ? {} : { detail: `expected ${want} ${clause}, plan proposes ${got} — ${nameList(cs, [clause])}` }),
+    });
+  }
+
+  if (expect.deletes) {
+    checks.push(evaluateDeletesClause(cs, expect.deletes));
+  }
+
+  if (expect.unobserved !== undefined) {
+    checks.push(evaluateUnobservedClause(cs, expect.unobserved));
+  }
+
+  return { pass: checks.every((c) => c.pass), checks };
+}
+
+/** Names (with type) of every entry matching one of `actions`, for a legible failure message. */
+function nameList(cs: ChangeSet, actions: readonly string[]): string {
+  const names = cs.entries
+    .filter((e) => (actions as readonly string[]).includes(e.action))
+    .map((e) => `${e.name}${e.type ? ` (${e.type})` : ""} [${e.action}]`);
+  return names.length === 0 ? "(no matching entries)" : names.join(", ");
+}
+
+function evaluateDeletesClause(
+  cs: ChangeSet,
+  wanted: readonly ScenarioDeleteExpectation[],
+): ScenarioCheckResult {
+  const actualDeletes = cs.entries.filter((e) => e.action === "delete");
+  const actualByName = new Map(actualDeletes.map((e) => [e.name, e] as const));
+  const wantedNames = new Set(wanted.map((w) => w.name));
+  const problems: string[] = [];
+
+  for (const want of wanted) {
+    const match = actualByName.get(want.name);
+    if (!match) {
+      problems.push(`"${want.name}" (ownership ${want.ownership}) — not proposed for delete`);
+    } else if (match.ownership !== want.ownership) {
+      problems.push(
+        `"${want.name}" — expected ownership "${want.ownership}", plan proposes ownership "${match.ownership}"`,
+      );
+    }
+  }
+  for (const e of actualDeletes) {
+    if (!wantedNames.has(e.name)) {
+      problems.push(`"${e.name}"${e.type ? ` (${e.type})` : ""} [${e.ownership}] — deleted but not expected`);
+    }
+  }
+
+  return {
+    clause: "deletes",
+    pass: problems.length === 0,
+    ...(problems.length > 0 ? { detail: problems.join("; ") } : {}),
+  };
+}
+
+function evaluateUnobservedClause(cs: ChangeSet, policy: ScenarioUnobservedPolicy): ScenarioCheckResult {
+  // evaluateUnobservedGate enumerates every unobserved row regardless of
+  // reason — "refuse" collects the full set to check against below. A
+  // scenario's allow list names ENTITIES, not reasons (./scenario.ts's
+  // ScenarioUnobservedPolicy doc explains why), so the gate's own reason-keyed
+  // policy isn't reused for the pass/fail decision — only for the enumeration.
+  const gate = evaluateUnobservedGate(cs, "refuse");
+  const allowedNames = typeof policy === "object" ? new Set(policy.allow) : undefined;
+  const unallowed = gate.findings.filter((f: UnobservedGateFinding) => !allowedNames?.has(f.name));
+
+  return {
+    clause: "unobserved",
+    pass: unallowed.length === 0,
+    ...(unallowed.length > 0
+      ? {
+          detail: unallowed
+            .map((f) => `${f.name}${f.type ? ` (${f.type})` : ""} — ${unobservedReasonText(f.reason)}${f.detail ? `: ${f.detail}` : ""}`)
+            .join(", "),
+        }
+      : {}),
+  };
+}

package/src/lifecycle/scenario.test.ts

@@ -0,0 +1,195 @@
+import { describe, test, expect } from "vitest";
+import {
+  Scenario,
+  snapshot,
+  isScenario,
+  collectScenarios,
+  SCENARIO_MARKER,
+  SCENARIO_ENTITY_TYPE,
+} from "./scenario";
+import { DECLARABLE_MARKER, type Declarable } from "../declarable";
+import { partitionByLexicon } from "../build";
+
+describe("snapshot()", () => {
+  test("classifies a slash-containing string as a file", () => {
+    expect(snapshot("fixtures/prod-baseline.json")).toEqual({ kind: "file", path: "fixtures/prod-baseline.json" });
+  });
+
+  test("classifies a .json-suffixed string as a file even without a slash", () => {
+    expect(snapshot("baseline.json")).toEqual({ kind: "file", path: "baseline.json" });
+  });
+
+  test("classifies a bare token as an environment name", () => {
+    expect(snapshot("prod")).toEqual({ kind: "env", env: "prod" });
+  });
+
+  test("rejects an empty string", () => {
+    expect(() => snapshot("")).toThrow(/non-empty/);
+  });
+});
+
+describe("Scenario()", () => {
+  test("builds a Declarable-shaped, marked declaration", () => {
+    const s = Scenario("plan-neutral refactor", {
+      given: snapshot("fixtures/prod-baseline.json"),
+      expect: { noop: true },
+    });
+    expect(s[DECLARABLE_MARKER]).toBe(true);
+    expect(s[SCENARIO_MARKER]).toBe(true);
+    expect(s.entityType).toBe(SCENARIO_ENTITY_TYPE);
+    expect(s.lexicon).toBe("chant");
+    expect(s.name).toBe("plan-neutral refactor");
+    expect(s.given).toEqual({ kind: "file", path: "fixtures/prod-baseline.json" });
+    expect(s.expect).toEqual({ noop: true });
+  });
+
+  test("isScenario recognizes it and rejects ordinary declarables", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { noop: true } });
+    expect(isScenario(s)).toBe(true);
+    expect(isScenario({})).toBe(false);
+    expect(isScenario(null)).toBe(false);
+    const other: Declarable = { [DECLARABLE_MARKER]: true, lexicon: "aws", entityType: "AWS::S3::Bucket" };
+    expect(isScenario(other)).toBe(false);
+  });
+
+  test("declared fields are locked but the object stays extensible", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { noop: true } });
+    expect(() => {
+      (s as { name: string }).name = "renamed";
+    }).toThrow();
+    // Discovery stamps its own symbol-keyed metadata — the object must accept it.
+    const extra = Symbol("chant.discovery.logical-name");
+    expect(() => {
+      (s as unknown as Record<symbol, unknown>)[extra] = "s";
+    }).not.toThrow();
+  });
+
+  test("nested given/expect structures are frozen", () => {
+    const s = Scenario("s", {
+      given: snapshot("fixtures/x.json"),
+      expect: { deletes: [{ name: "legacy", ownership: "owned" }] },
+    });
+    expect(Object.isFrozen(s.given)).toBe(true);
+    expect(Object.isFrozen(s.expect)).toBe(true);
+    expect(Object.isFrozen(s.expect.deletes)).toBe(true);
+    expect(Object.isFrozen(s.expect.deletes![0])).toBe(true);
+  });
+
+  test("rejects an empty name", () => {
+    expect(() => Scenario("", { given: snapshot("prod"), expect: { noop: true } })).toThrow(/non-empty/);
+  });
+
+  test("rejects a missing given", () => {
+    // @ts-expect-error deliberately omitting given
+    expect(() => Scenario("s", { expect: { noop: true } })).toThrow(/given/);
+  });
+
+  test("rejects an empty expect", () => {
+    expect(() => Scenario("s", { given: snapshot("prod"), expect: {} })).toThrow(/at least one clause/);
+  });
+
+  test("rejects an unknown expect clause", () => {
+    expect(() =>
+      // @ts-expect-error deliberately passing an unrecognized clause
+      Scenario("s", { given: snapshot("prod"), expect: { bogus: true } }),
+    ).toThrow(/unknown `expect` clause/);
+  });
+
+  test("rejects noop: false (not a valid literal)", () => {
+    expect(() =>
+      // @ts-expect-error deliberately passing noop: false
+      Scenario("s", { given: snapshot("prod"), expect: { noop: false } }),
+    ).toThrow(/must be exactly `true`/);
+  });
+
+  test("rejects a negative count", () => {
+    expect(() => Scenario("s", { given: snapshot("prod"), expect: { create: -1 } })).toThrow(/non-negative integer/);
+  });
+
+  test("rejects a non-integer count", () => {
+    expect(() => Scenario("s", { given: snapshot("prod"), expect: { update: 1.5 } })).toThrow(/non-negative integer/);
+  });
+
+  test("accepts create/update/delete counts together", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { create: 0, update: 1, delete: 2 } });
+    expect(s.expect).toEqual({ create: 0, update: 1, delete: 2 });
+  });
+
+  test("rejects a deletes entry with a bad ownership value", () => {
+    expect(() =>
+      Scenario("s", {
+        given: snapshot("prod"),
+        // @ts-expect-error deliberately passing an invalid ownership
+        expect: { deletes: [{ name: "legacy", ownership: "nope" }] },
+      }),
+    ).toThrow(/ownership/);
+  });
+
+  test("rejects a deletes entry with a missing name", () => {
+    expect(() =>
+      Scenario("s", {
+        given: snapshot("prod"),
+        // @ts-expect-error deliberately omitting name
+        expect: { deletes: [{ ownership: "owned" }] },
+      }),
+    ).toThrow(/name/);
+  });
+
+  test("accepts unobserved: \"refuse\"", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { unobserved: "refuse" } });
+    expect(s.expect.unobserved).toBe("refuse");
+  });
+
+  test("accepts unobserved: { allow: [names] }", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { unobserved: { allow: ["legacy-bucket"] } } });
+    expect(s.expect.unobserved).toEqual({ allow: ["legacy-bucket"] });
+  });
+
+  test("rejects a malformed unobserved policy", () => {
+    expect(() =>
+      // @ts-expect-error deliberately passing a bad policy shape
+      Scenario("s", { given: snapshot("prod"), expect: { unobserved: "nope" } }),
+    ).toThrow(/unobserved/);
+  });
+});
+
+describe("collectScenarios()", () => {
+  test("extracts only scenario declarations, keyed by entity name", () => {
+    const s1 = Scenario("s1", { given: snapshot("prod"), expect: { noop: true } });
+    const s2 = Scenario("s2", { given: snapshot("staging"), expect: { create: 0 } });
+    const other: Declarable = { [DECLARABLE_MARKER]: true, lexicon: "aws", entityType: "AWS::S3::Bucket" };
+    const entities = new Map<string, Declarable>([
+      ["scenarioOne", s1],
+      ["scenarioTwo", s2],
+      ["bucket", other],
+    ]);
+    const found = collectScenarios(entities);
+    expect(found.size).toBe(2);
+    expect(found.get("scenarioOne")).toBe(s1);
+    expect(found.get("scenarioTwo")).toBe(s2);
+  });
+});
+
+describe("serializer exclusion", () => {
+  test("partitionByLexicon excludes scenarios from every lexicon partition", () => {
+    const s = Scenario("s", { given: snapshot("prod"), expect: { noop: true } });
+    const bucket: Declarable = {
+      [DECLARABLE_MARKER]: true,
+      lexicon: "aws",
+      entityType: "AWS::S3::Bucket",
+      kind: "resource",
+    };
+    const entities = new Map<string, Declarable>([
+      ["myScenario", s],
+      ["bucket", bucket],
+    ]);
+    const partitions = partitionByLexicon(entities);
+    expect(partitions.has("chant")).toBe(false);
+    for (const [, group] of partitions) {
+      for (const [, entity] of group) {
+        expect(isScenario(entity)).toBe(false);
+      }
+    }
+    expect(partitions.get("aws")?.has("bucket")).toBe(true);
+  });
+});