@intentius/chant 0.33.1 → 0.34.1

package/src/codegen/publish-order.test.ts

@@ -0,0 +1,133 @@
+/**
+ * Publishing a package runs its prepack, and a prepack builds against and
+ * imports its workspace dependencies' *generated* output — output that only
+ * exists once that dependency has been published. So publish order has to be a
+ * topological order, and for a long time directory order stood in for one.
+ *
+ * It held until it did not. chant-v0.34.0 published twelve of fourteen packages
+ * and stranded lexicon-forgejo and lexicon-helm a version behind: forgejo needs
+ * github's `src/generated/index` and helm needs k8s's `dist/generated`, but
+ * `forgejo` < `github` and `helm` < `k8s`, so both ran before the thing they
+ * import existed. Half a release shipped and the failure only surfaced from
+ * npm, after the tag.
+ *
+ * publish-packages.sh now derives the order instead of assuming one. This
+ * asserts the derivation is actually topological, so the next lexicon that
+ * depends on an alphabetically earlier one fails here rather than mid-release.
+ */
+
+import { describe, expect, it } from "vitest";
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const REPO = join(dirname(fileURLToPath(import.meta.url)), "../../../..");
+
+/** Run one of publish-packages.sh's own functions and read back what it prints. */
+function ask(fn: string): string[] {
+  const script = readFileSync(join(REPO, "scripts/publish-packages.sh"), "utf8");
+  const body = script.match(new RegExp(`^${fn}\\(\\) \\{$.*?^\\}$`, "ms"));
+  if (!body) throw new Error(`${fn}() not found in scripts/publish-packages.sh`);
+  return execFileSync("bash", ["-c", `${body[0]}\n${fn}`], { cwd: REPO, encoding: "utf8" })
+    .split("\n")
+    .filter(Boolean);
+}
+
+const publishOrder = () => ask("publishable_dirs");
+
+function manifest(dir: string) {
+  return JSON.parse(readFileSync(join(REPO, dir, "package.json"), "utf8"));
+}
+
+describe("publish order", () => {
+  const order = publishOrder();
+
+  it("covers every publishable workspace package", () => {
+    // A package missing from the order never publishes at all, which is the
+    // same stranding by a different route.
+    const all = execFileSync(
+      "bash",
+      ["-c", 'for d in packages/*/ lexicons/*/; do [ -f "$d/package.json" ] && echo "${d%/}"; done'],
+      { cwd: REPO, encoding: "utf8" },
+    )
+      .split("\n")
+      .filter(Boolean)
+      .filter((d) => !manifest(d).private);
+
+    expect([...order].sort()).toEqual([...all].sort());
+  });
+
+  it("places every package after the workspace packages it depends on", () => {
+    const position = new Map(order.map((d, i) => [d, i]));
+    const owner = new Map(order.map((d) => [manifest(d).name as string, d]));
+
+    for (const dir of order) {
+      const pkg = manifest(dir);
+      const deps = Object.keys({
+        ...pkg.dependencies,
+        ...pkg.peerDependencies,
+        ...pkg.optionalDependencies,
+      });
+      for (const name of deps) {
+        const depDir = owner.get(name);
+        if (!depDir || depDir === dir) continue;
+        expect(
+          position.get(depDir)!,
+          `${pkg.name} (${dir}) publishes before its dependency ${name} (${depDir}), ` +
+            `so ${name}'s generated output will not exist when ${pkg.name} prepacks`,
+        ).toBeLessThan(position.get(dir)!);
+      }
+    }
+  });
+
+  it("puts the two packages that stranded chant-v0.34.0 after what they import", () => {
+    // The specific regression, named, so the failure says what broke rather
+    // than only that some invariant did.
+    expect(order.indexOf("lexicons/github")).toBeLessThan(order.indexOf("lexicons/forgejo"));
+    expect(order.indexOf("lexicons/k8s")).toBeLessThan(order.indexOf("lexicons/helm"));
+  });
+});
+
+/**
+ * Ordering alone did not fix the release. The rerun published nothing, found
+ * k8s already at 0.34.0, skipped it — and skipping the publish skipped the
+ * prepack that builds `dist/generated`, so helm failed on the same missing
+ * module. Anything another package compiles against has to be built whether or
+ * not it needs publishing.
+ */
+describe("packages built even when their publish is skipped", () => {
+  const built = new Set(ask("depended_on_dirs"));
+  const order = publishOrder();
+
+  it("covers every workspace dependency, transitively", () => {
+    for (const dir of order) {
+      const pkg = manifest(dir);
+      const owner = new Map(order.map((d) => [manifest(d).name as string, d]));
+      for (const name of Object.keys({
+        ...pkg.dependencies,
+        ...pkg.peerDependencies,
+        ...pkg.optionalDependencies,
+      })) {
+        const depDir = owner.get(name);
+        if (!depDir || depDir === dir) continue;
+        expect(
+          built.has(depDir),
+          `${name} (${depDir}) is a dependency of ${pkg.name} but would not be built when ` +
+            `its own publish is skipped, so ${pkg.name} compiles against nothing`,
+        ).toBe(true);
+      }
+    }
+  });
+
+  it("includes the two whose skipped build failed the chant-v0.34.0 rerun", () => {
+    expect(built.has("lexicons/k8s")).toBe(true);
+    expect(built.has("lexicons/github")).toBe(true);
+  });
+
+  it("does not build a leaf nothing depends on", () => {
+    // The set is the reason a rerun is not a full 14-package rebuild.
+    expect(built.has("lexicons/helm")).toBe(false);
+    expect(built.has("lexicons/forgejo")).toBe(false);
+  });
+});

package/src/codegen/release-wiring.test.ts

@@ -80,3 +80,95 @@ describe("release wiring: justfile tags vs publish.yml trigger", () => {
     expect(examples).toContain("lexicon-docker-v9.9.9");
   });
 });
+
+/**
+ * The body of one justfile recipe, up to the next top-level recipe header.
+ * Line-based on purpose: a recipe header sits at column 0 and every body line
+ * is indented, which a regex over the whole file gets wrong (`^` under /m
+ * matches the start of the slice too).
+ */
+const RECIPE_HEADER = /^[a-z_][a-z0-9_-]*(\s+[^:]*)?:/;
+
+function recipeBody(name: string): string {
+  const lines = readFileSync(join(repoRoot, "justfile"), "utf-8").split("\n");
+  const start = lines.findIndex((l) => new RegExp(`^${name}(\\s|:)`).test(l));
+  if (start === -1) throw new Error(`justfile: recipe "${name}" not found`);
+
+  let end = lines.length;
+  for (let i = start + 1; i < lines.length; i++) {
+    if (RECIPE_HEADER.test(lines[i])) {
+      end = i;
+      break;
+    }
+  }
+  return lines.slice(start, end).join("\n");
+}
+
+/**
+ * #1255.2 — `just release` rewrites the `@intentius/*` peer ranges alongside
+ * `.version` (added in #411, because ranges frozen at `^0.1.0` break clean
+ * installs). `release-lexicon` only set `.version`, so a single-lexicon patch
+ * needing a newer core silently shipped a stale range.
+ *
+ * The two recipes pin to different things on purpose: a whole-repo release
+ * moves everything together so `^$next` is right, while a single-lexicon
+ * patch leaves core where it is, so the range must track the CURRENT core
+ * version. Pinning it to `$next` would demand a core that does not exist —
+ * which is why this asserts the mechanism, not a shared literal.
+ */
+describe("release wiring: peer ranges stay in lockstep (#1255)", () => {
+  it("the whole-repo release rewrites both @intentius peer ranges", () => {
+    const body = recipeBody("release");
+    expect(body).toMatch(/peerDependencies\["@intentius\/chant"\]/);
+    expect(body).toMatch(/peerDependencies\["@intentius\/chant-lexicon-github"\]/);
+  });
+
+  it("the single-lexicon release rewrites them too", () => {
+    const body = recipeBody("release-lexicon");
+    expect(
+      body.includes('peerDependencies["@intentius/chant"]'),
+      "release-lexicon sets .version without touching the @intentius/chant peer range — " +
+        "a lexicon patch that needs a newer core would ship a stale range (#411, #1255)",
+    ).toBe(true);
+    expect(body).toMatch(/peerDependencies\["@intentius\/chant-lexicon-github"\]/);
+  });
+
+  it("the single-lexicon release pins peers to core, not to its own new version", () => {
+    const body = recipeBody("release-lexicon");
+    // It reads the current core version rather than reusing $next.
+    expect(body).toMatch(/jq -r \.version packages\/core\/package\.json/);
+    expect(
+      /peerDependencies\["@intentius\/chant"\]\s*=\s*"\^"\s*\+\s*\$next/.test(body),
+      "release-lexicon must not pin the core peer range to the lexicon's own new version",
+    ).toBe(false);
+  });
+
+  it("both recipes keep the committed lockfile in step", () => {
+    for (const recipe of ["release", "release-lexicon"]) {
+      expect(recipeBody(recipe)).toMatch(/npm install --package-lock-only/);
+    }
+  });
+});
+
+/**
+ * #1255.3 — both recipes pushed the bump commit straight to main, bypassing
+ * branch protection, so a release tag could point at a commit CI never ran.
+ */
+describe("release wiring: preflight gates both recipes (#1255)", () => {
+  it("every release recipe runs the preflight before tagging", () => {
+    for (const recipe of ["release", "release-lexicon"]) {
+      const body = recipeBody(recipe);
+      expect(
+        body.includes("scripts/release-preflight.sh"),
+        `justfile recipe "${recipe}" tags and pushes without running the preflight — ` +
+          "it could release a commit CI never ran (#1255)",
+      ).toBe(true);
+      // Ordering matters: a check that runs after `git tag` proves nothing.
+      expect(body.indexOf("scripts/release-preflight.sh")).toBeLessThan(body.indexOf("git tag"));
+    }
+  });
+
+  it("the whole-repo release requires main, since it pushes main", () => {
+    expect(recipeBody("release")).toMatch(/release-preflight\.sh main/);
+  });
+});

package/src/graph-ir-live.test.ts

@@ -157,6 +157,34 @@ describe("sourceOverlayGraphs (#821 source-anchored overlay)", () => {
     expect(vpc.ownership).toBe("owned");
   });
 
+  it("carries observed attributes onto a managed declared node (#1279)", () => {
+    // Before this, the overlay copied identity but dropped everything observed
+    // ABOUT the resource, so a fact that only exists in the account — an
+    // instance's VpcId — was unreachable from the overlaid graph and
+    // `search --show VpcId` printed a blank column.
+    const observed: GraphIR = {
+      ...live,
+      nodes: live.nodes.map((x) => (x.id === "web-vpc" ? { ...x, attrs: { CidrBlock: "10.0.0.0/16" } } : x)),
+    };
+    const vpc = sourceOverlayGraphs(declared, observed).nodes.find((x) => x.id === "web-vpc")!;
+    expect(vpc.attrs.CidrBlock).toBe("10.0.0.0/16");
+  });
+
+  it("lets an observed value beat the declared one of the same name (#1279)", () => {
+    // The declared side holds an unresolved reference to another entity; the
+    // account holds the id it actually resolved to.
+    const withRef: GraphIR = {
+      ...declared,
+      nodes: declared.nodes.map((x) => (x.id === "web-vpc" ? { ...x, attrs: { VpcId: "${vpc.id}" } } : x)),
+    };
+    const observed: GraphIR = {
+      ...live,
+      nodes: live.nodes.map((x) => (x.id === "web-vpc" ? { ...x, attrs: { VpcId: "vpc-0a1b" } } : x)),
+    };
+    const vpc = sourceOverlayGraphs(withRef, observed).nodes.find((x) => x.id === "web-vpc")!;
+    expect(vpc.attrs.VpcId).toBe("vpc-0a1b");
+  });
+
   it("appends foreign nodes with their live edges, and keeps declared groups", () => {
     const ir = sourceOverlayGraphs(declared, live);
     expect(ir.nodes.map((x) => x.id)).toContain("rogue-sg");
@@ -208,3 +236,58 @@ describe("collectUnobserved (#1089)", () => {
     ).toEqual({ a: { reason: "read-failed" }, b: { reason: "no-binding" } });
   });
 });
+
+// #1271 — an observation can report relationships, not just existence. Without
+// these the live side of the graph has no edges, so a topology fold has nothing
+// to traverse and a lexicon has to compute derived answers itself.
+describe("observed edges (#1271)", () => {
+  const twoNodes = {
+    lexicon: "aws",
+    resources: {
+      "app-subnet": { type: "AWS::EC2::Subnet", status: "CREATE_COMPLETE", physicalId: "subnet-0c2d" },
+      "web-vpc": { type: "AWS::EC2::VPC", status: "CREATE_COMPLETE", physicalId: "vpc-0a1b" },
+    },
+  } satisfies LiveObservation;
+
+  it("projects an observed edge between two observed nodes", () => {
+    const ir = buildLiveGraphIr([
+      { ...twoNodes, edges: [{ from: "app-subnet", to: "web-vpc", kind: "ref", viaAttr: "VpcId" }] },
+    ]);
+    expect(ir.edges).toEqual([{ from: "app-subnet", to: "web-vpc", kind: "ref", viaAttr: "VpcId" }]);
+  });
+
+  it("no edges reported → no edges, unchanged from before", () => {
+    expect(buildLiveGraphIr([twoNodes]).edges).toEqual([]);
+  });
+
+  it("drops an edge whose endpoint was never observed", () => {
+    // A dangling reference would traverse to nothing during a fold, which reads
+    // as "no such relationship" rather than "the other end was not read".
+    const ir = buildLiveGraphIr([
+      { ...twoNodes, edges: [{ from: "app-subnet", to: "never-read", kind: "ref", viaAttr: "VpcId" }] },
+    ]);
+    expect(ir.edges).toEqual([]);
+  });
+
+  it("dedupes identical edges reported by more than one observation", () => {
+    const edge = { from: "app-subnet", to: "web-vpc", kind: "ref" as const, viaAttr: "VpcId" };
+    const ir = buildLiveGraphIr([
+      { ...twoNodes, edges: [edge] },
+      { ...twoNodes, edges: [edge] },
+    ]);
+    expect(ir.edges).toHaveLength(1);
+  });
+
+  it("orders edges deterministically — the IR is compared and committed", () => {
+    const ir = buildLiveGraphIr([
+      {
+        ...twoNodes,
+        edges: [
+          { from: "web-vpc", to: "app-subnet", kind: "ref", viaAttr: "Z" },
+          { from: "app-subnet", to: "web-vpc", kind: "ref", viaAttr: "A" },
+        ],
+      },
+    ]);
+    expect(ir.edges?.map((e) => e.from)).toEqual(["app-subnet", "web-vpc"]);
+  });
+});

package/src/graph-ir.ts

@@ -489,6 +489,20 @@ export interface LiveObservation {
    * overlay must not paint them "pending". See {@link sourceOverlayGraphs}.
    */
   unobserved?: Record<string, UnobservedEntity>;
+  /**
+   * Relationships the lexicon observed between the resources it read (#1271).
+   *
+   * Without these an observation can only say what exists, never how it
+   * connects — so a fold over topology has nothing to traverse on the live side
+   * and a lexicon has to compute derived answers itself and inject them as
+   * attributes. Reporting the edges instead lets the one graph fold in
+   * {@link import("./graph-effective").enrichEffectiveTopology} do the work,
+   * for every lexicon, over live and recorded observations alike.
+   *
+   * `from`/`to` are node ids in the same space as {@link LiveObservation.resources}
+   * keys, so an edge can only reference something the observation also reported.
+   */
+  edges?: IREdge[];
 }
 
 /**
@@ -540,7 +554,31 @@ export function buildLiveGraphIr(observations: LiveObservation[]): GraphIR {
   if (Object.keys(byLexicon).length) groups.byLexicon = sortKeys(byLexicon);
   if (Object.keys(byStack).length) groups.byStack = sortKeys(byStack);
 
-  return { nodes, edges: [], groups };
+  // Observed relationships (#1271). An edge whose endpoints were not both
+  // observed is dropped rather than kept as a dangling reference: the fold
+  // resolves ids to nodes, and a half-edge would silently traverse to nothing.
+  // Deduped and sorted for the same reason nodes are — the IR is compared and
+  // committed, so it has to be stable across reads.
+  const observedIds = new Set(nodes.map((n) => n.id));
+  const seen = new Set<string>();
+  const edges: IREdge[] = [];
+  for (const observation of observations) {
+    for (const edge of observation.edges ?? []) {
+      if (!observedIds.has(edge.from) || !observedIds.has(edge.to)) continue;
+      const key = `${edge.from}${edge.to}${edge.viaAttr ?? ""}${edge.toAttr ?? ""}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      edges.push(edge);
+    }
+  }
+  edges.sort(
+    (a, b) =>
+      a.from.localeCompare(b.from) ||
+      a.to.localeCompare(b.to) ||
+      (a.viaAttr ?? "").localeCompare(b.viaAttr ?? ""),
+  );
+
+  return { nodes, edges, groups };
 }
 
 /** How an overlay learns which declared nodes were never looked at (#1089). */
@@ -647,6 +685,18 @@ export function sourceOverlayGraphs(declared: GraphIR, live: GraphIR, opts?: Ove
     const merged: IRNode = { ...n }; // managed — carry the observed identity
     if (obs.physicalId) merged.physicalId = obs.physicalId;
     if (obs.ownership) merged.ownership = obs.ownership;
+    // ...and what was observed ABOUT it (#1279). The declared canvas holds what
+    // the source says; the observation holds what the account says, and a fact
+    // like `VpcId` exists only on the second. Dropping it here made every
+    // observed property invisible to anything reading the overlaid graph —
+    // `search --show VpcId` printed six blank columns, which reads as an estate
+    // with no VPCs rather than a read that never happened.
+    //
+    // Observed wins a collision. The declared value for something like `VpcId`
+    // is an unresolved reference to another entity, not an id — the account has
+    // the id. Same precedence `enrichLiveAttrs` already uses when it folds live
+    // facts onto this graph, so both paths agree on what a name means.
+    if (obs.attrs && Object.keys(obs.attrs).length > 0) merged.attrs = { ...n.attrs, ...obs.attrs };
     return tagStatus(merged, "good");
   });
   for (const n of live.nodes) {

package/src/graph-refs.test.ts

@@ -129,3 +129,62 @@ describe("mergeCatalogs", () => {
     expect(m.refs).toHaveLength(1);
   });
 });
+
+// #1275 — `viaAttr` defaulted to `label ?? path`, which serves a renderer and
+// starves a traversal: the catalog's labels are human-facing ("sg", "via")
+// while a fold matches provider attribute names ("SecurityGroupIds").
+describe("traversal name vs rendering label (#1275)", () => {
+  const nodes: IRNode[] = [
+    { id: "web", kind: "AWS::EC2::Instance", lexicon: "aws", attrs: { SubnetId: "subnet-1", SecurityGroups: [{ GroupId: "sg-1" }] } },
+    { id: "sub", kind: "AWS::EC2::Subnet", lexicon: "aws", attrs: { SubnetId: "subnet-1" } },
+    { id: "sg", kind: "AWS::EC2::SecurityGroup", lexicon: "aws", attrs: { GroupId: "sg-1" } },
+  ];
+  const identities = [
+    { kind: "AWS::EC2::Subnet", ids: ["SubnetId"] },
+    { kind: "AWS::EC2::SecurityGroup", ids: ["GroupId"] },
+  ];
+
+  it("viaAttr wins over label, so the fold can match the attribute name", () => {
+    const { edges } = reconstructEdges(nodes, {
+      identities,
+      refs: [
+        { from: "AWS::EC2::Instance", path: "SecurityGroups[].GroupId", targetKind: "AWS::EC2::SecurityGroup", relation: "reference", label: "sg", viaAttr: "SecurityGroupIds" },
+      ],
+    });
+    expect(edges).toEqual([{ from: "web", to: "sg", kind: "ref", viaAttr: "SecurityGroupIds" }]);
+  });
+
+  it("without viaAttr the label still wins, unchanged", () => {
+    const { edges } = reconstructEdges(nodes, {
+      identities,
+      refs: [
+        { from: "AWS::EC2::Instance", path: "SecurityGroups[].GroupId", targetKind: "AWS::EC2::SecurityGroup", relation: "reference", label: "sg" },
+      ],
+    });
+    expect(edges[0].viaAttr).toBe("sg");
+  });
+
+  it("a containment rule with viaAttr yields both the boundary pair and a traversable edge", () => {
+    // An instance is *in* a subnet — a boundary for the picture, and the first
+    // hop of internetFacing. It has to be both without being drawn twice.
+    const { edges, containment } = reconstructEdges(nodes, {
+      identities,
+      refs: [
+        { from: "AWS::EC2::Instance", path: "SubnetId", targetKind: "AWS::EC2::Subnet", relation: "containment", label: "in subnet", viaAttr: "SubnetId" },
+      ],
+    });
+    expect(containment).toEqual([{ child: "web", parent: "sub", label: "in subnet" }]);
+    expect(edges).toEqual([{ from: "web", to: "sub", kind: "ref", viaAttr: "SubnetId" }]);
+  });
+
+  it("a containment rule without viaAttr stays a boundary hint only", () => {
+    const { edges, containment } = reconstructEdges(nodes, {
+      identities,
+      refs: [
+        { from: "AWS::EC2::Instance", path: "SubnetId", targetKind: "AWS::EC2::Subnet", relation: "containment", label: "in subnet" },
+      ],
+    });
+    expect(containment).toHaveLength(1);
+    expect(edges).toEqual([]);
+  });
+});

package/src/graph-refs.ts

@@ -40,6 +40,25 @@ export interface RefRule {
   relation: "reference" | "containment";
   /** Edge / containment label (e.g. "in VPC", "sg", "targets"). */
   label?: string;
+  /**
+   * What the reconstructed edge's `viaAttr` should be, when traversal needs a
+   * different string than rendering does (#1275).
+   *
+   * `viaAttr` defaulted to `label ?? path`, which serves a renderer well and a
+   * traversal badly. The labels here are human-facing — "sg", "via", "in VPC" —
+   * while a fold like `enrichEffectiveTopology` matches provider attribute
+   * names: `SecurityGroupIds`, `SubnetId`, `LaunchTemplateId`. One field could
+   * not be both, so a rule that is traversed declares the name explicitly and
+   * keeps its label for the picture.
+   *
+   * On a `containment` rule this additionally opts the relation into producing
+   * an edge, on top of the boundary pair it already produces. Containment is
+   * not an edge by default and should not become one — but a fold's first hop
+   * is sometimes exactly a containment relation (an instance is *in* a subnet),
+   * and that hop has to be traversable without duplicating the rule as a
+   * reference and drawing the line twice.
+   */
+  viaAttr?: string;
 }
 
 /** A lexicon's reference knowledge — its identity map and reference rules. */
@@ -154,17 +173,29 @@ export function reconstructEdges(nodes: IRNode[], catalog: ReferenceCatalog): Re
           continue;
         }
         if (match.id === node.id) continue; // self-reference (e.g. an SG rule to its own group)
-        if (rule.relation === "containment") {
-          const k = `${node.id}|${match.id}`;
-          if (seenCont.has(k)) continue;
-          seenCont.add(k);
-          containment.push({ child: node.id, parent: match.id, ...(rule.label ? { label: rule.label } : {}) });
-        } else {
-          const via = rule.label ?? rule.path;
+        const pushEdge = (via: string): void => {
           const k = `${node.id}|${match.id}|${via}`;
-          if (seenEdge.has(k)) continue;
+          if (seenEdge.has(k)) return;
           seenEdge.add(k);
           edges.push({ from: node.id, to: match.id, kind: "ref", viaAttr: via });
+        };
+        if (rule.relation === "containment") {
+          const k = `${node.id}|${match.id}`;
+          if (!seenCont.has(k)) {
+            seenCont.add(k);
+            containment.push({ child: node.id, parent: match.id, ...(rule.label ? { label: rule.label } : {}) });
+          }
+          // A containment relation is a boundary hint, not an edge — unless the
+          // rule declares a traversal name (#1275). A fold's first hop is
+          // sometimes exactly a containment ("an instance is in a subnet"), and
+          // it must be traversable without duplicating the rule as a reference
+          // and drawing the line twice.
+          if (rule.viaAttr) pushEdge(rule.viaAttr);
+        } else {
+          // Traversal name wins over the rendering label: the labels are
+          // human-facing ("sg", "via"), and a fold matches provider attribute
+          // names (#1275).
+          pushEdge(rule.viaAttr ?? rule.label ?? rule.path);
         }
       }
     }