@intentius/chant 0.33.0 → 0.34.0

package/src/lexicon.ts

@@ -11,6 +11,7 @@ import type { DriverComponent } from "./components/driver";
 import type { EmulatorCapability } from "./op/emulator-lifecycle";
 import type { RuleMeta } from "./audit/catalog";
 import type { ReferenceCatalog } from "./graph-refs";
+import type { IREdge } from "./graph-ir";
 import type { DescribeResourcesResult } from "./observation";
 import type { DeepNormalizationHooks, DeepObservationResult } from "./deep-observation";
 import type { OwnerChainVerdict } from "./owner-chain";
@@ -24,6 +25,10 @@ export type { CommandGroup, CommandGroupCommand, CommandGroupContext } from "./c
 // `@intentius/chant/lexicon` entry they import the plugin contract from.
 export type { ReferenceCatalog, IdentityRule, RefRule } from "./graph-refs";
 
+// An observation can report relationships (#1271/#1273), so a lexicon needs the
+// edge type from the same entry it imports the plugin contract from.
+export type { IREdge } from "./graph-ir";
+
 // The observation contract (#1089), re-exported from the same entry so a
 // lexicon's `describeResources` can report NOT-OBSERVED without a second
 // import path. Runtime helpers live in `@intentius/chant/observation`.
@@ -641,6 +646,84 @@ export interface LexiconPlugin {
    * Throwing is the whole-lexicon failure, same as the thin read: core turns it
    * into `read-failed` for every declared entity.
    */
+  /**
+   * Report the undeclared resources this estate *depends on* (#1273), as
+   * opposed to the ones it manages.
+   *
+   * `describeResources` is scoped to what the stack declares. Anything the
+   * estate references but does not declare — an account's default VPC route
+   * tables, a shared subnet, networking owned by another team — is invisible to
+   * it, so it never becomes a node, so no edge can reach it and no fold can
+   * traverse it. That is why derived facts about un-modelled topology have had
+   * to be computed inside lexicons and injected as attributes.
+   *
+   * The closure rule is depth one by reference, plus whatever chains this
+   * lexicon's {@link referenceCatalog} declares as meaningful — so AWS follows
+   * `SubnetId` → `RouteTableId` → `GatewayId` because the catalog says those
+   * references matter, not because they happen to be reachable. Without a rule
+   * a VPC transitively reaches most of an account.
+   *
+   * Every returned resource must carry {@link ResourceMetadata.referencedBy},
+   * naming the nodes that pulled it in. A dependency with no referrer is
+   * unbounded discovery, which is what the closure rule exists to prevent.
+   *
+   * Optional and additive. A lexicon that does not implement it behaves exactly
+   * as before, and a consumer that ignores dependencies sees what it always saw.
+   */
+  observeDependencies?(options: {
+    environment: string;
+    /** Declared entities, for a lexicon that resolves references from source. */
+    entities: Map<string, { entityType: string; props: Record<string, unknown> }>;
+    /** What {@link describeResources} just found — the roots of the closure. */
+    observed: Record<string, ResourceMetadata>;
+    stack?: string;
+    region?: string;
+  }): Promise<DependencyObservation>;
+
+  /**
+   * Report resources of a kind this estate manages that exist in the account
+   * without being declared or referenced (#1278).
+   *
+   * `describeResources` answers "what do I manage" and `observeDependencies`
+   * answers "what do I rely on". Neither can see a resource that is simply
+   * *there* — an unattached security group, an orphaned volume — because both
+   * resolve outward from the declared estate and an unused resource is reached
+   * by nothing.
+   *
+   * That is a real question with no other answer: "which of my security groups
+   * are unused" cannot be resolved from a state file at all, because a state
+   * file knows only what it created. A lexicon that can enumerate a kind can
+   * answer it.
+   *
+   * `kinds` bounds the scan to types the project actually declares, so a
+   * project managing security groups is not made to enumerate the account. A
+   * lexicon returns resources marked {@link ResourceMetadata.ambient}, and must
+   * exclude anything already in `observed` — those are managed, not ambient.
+   *
+   * Optional and opt-in. A lexicon that does not implement it, or a caller that
+   * does not ask, sees exactly what it saw before.
+   */
+  /**
+   * Kinds this lexicon can enumerate beyond the declared estate (#1278).
+   *
+   * Declared separately from {@link observeAmbient} so a caller can say that
+   * ambient resources of a kind are POSSIBLE without paying for a scan to find
+   * out. `chant search` uses it to point out that `--ambient` is relevant to
+   * the kind just queried — an agent asking which security groups are unused
+   * has no way to know that some are not in the answer at all.
+   */
+  ambientKinds?(): string[];
+
+  observeAmbient?(options: {
+    environment: string;
+    /** Entity types the project declares — the bound on what to enumerate. */
+    kinds: string[];
+    /** Already-observed managed resources, to exclude. */
+    observed: Record<string, ResourceMetadata>;
+    stack?: string;
+    region?: string;
+  }): Promise<Record<string, ResourceMetadata>>;
+
   observeResourcesDeep?(options: {
     environment: string;
     buildOutput: string;
@@ -648,6 +731,10 @@ export interface LexiconPlugin {
     entities: Map<string, { entityType: string; props: Record<string, unknown> }>;
     /** Deployed stack to observe, for a multi-stack project (see `stacks` in {@link ChantConfig}). */
     stack?: string;
+    /** Region the stack is deployed in (#1267), mirroring the thin path's
+     * `describeResources` (#1261). When omitted, an implementation keeps its
+     * ambient-region default. */
+    region?: string;
     /** Restrict to chant-owned resources (#119). A lexicon with no marker channel on this path says so. */
     owned?: boolean;
   }): Promise<DeepObservationResult>;
@@ -799,6 +886,31 @@ export type ExportedTemplate = TemplateIR & {
 /**
  * Metadata about a deployed resource, returned by describeResources.
  */
+/**
+ * What a lexicon saw outside its declared estate (#1273): the resources
+ * something declared references, and how they connect.
+ *
+ * Kept as its own result rather than folded into `describeResources` so every
+ * consumer downstream can tell "what I manage" from "what I depend on" without
+ * disentangling them — the same reason the change set separates `orphan` from
+ * `runtimeChildren`.
+ */
+export interface DependencyObservation {
+  /**
+   * Undeclared resources, keyed by a stable id. Physical id is the natural
+   * choice: these have no logical name, and the key has to match between a live
+   * read and a snapshot replay or an overlay double-counts them.
+   */
+  resources: Record<string, ResourceMetadata>;
+  /**
+   * Relationships among the dependencies and back to the declared nodes that
+   * reached them. Reported rather than reconstructed, because a reference
+   * catalog can only resolve what it has an identity index for, and part of the
+   * point here is expressing a hop the catalog does not model.
+   */
+  edges?: IREdge[];
+}
+
 export interface ResourceMetadata {
   /** Entity type (e.g. AWS::S3::Bucket, K8s::Apps::Deployment) */
   type: string;
@@ -832,6 +944,39 @@ export interface ResourceMetadata {
    * undeclared live resource stays `orphan`.
    */
   ownerChain?: OwnerChainVerdict;
+  /**
+   * The declared node ids that reference this resource (#1273), set on an
+   * undeclared resource observed only because something declared points at it —
+   * the account's default VPC route table an instance routes through, a shared
+   * subnet, a network someone else owns.
+   *
+   * A third kind of observed-but-undeclared resource, beside {@link ownership}
+   * and {@link ownerChain}. Not an orphan: nobody is going to adopt or delete
+   * the account's default VPC, so offering it as a delete candidate is wrong.
+   * Not a runtime child either — nothing declared created it. It gets a runtime
+   * child's drift treatment for the same reason: it is not yours, it changes on
+   * its own, and alerting on it is noise.
+   *
+   * Carries the referrers rather than a bare flag so the reason a resource was
+   * pulled in is answerable, and so a closure can be walked back to the declared
+   * entity that justified it.
+   */
+  referencedBy?: string[];
+  /**
+   * Observed in the account, of a kind this estate manages, but neither
+   * declared nor referenced by anything declared (#1278).
+   *
+   * The third and last category of observed-but-undeclared resource, after
+   * {@link ownerChain}'s runtime children and {@link referencedBy}'s
+   * dependencies. A default security group AWS creates per VPC is the type
+   * case: nothing declares it, nothing points at it, and it is exactly what
+   * "which of my security groups are unused" is asking about.
+   *
+   * Opt-in, because finding these means asking the provider what exists rather
+   * than resolving out from what is declared, which is a broader read and a
+   * different claim about what an observation is.
+   */
+  ambient?: boolean;
 }
 
 /**

package/src/lifecycle/deep-observe.ts

@@ -39,6 +39,10 @@ export interface DeepObserveOptions {
   entities: DeclaredEntities;
   /** Deployed stack for a multi-stack project (#932). */
   stack?: string;
+  /** Region that stack is deployed in (#1267). Same contract as the thin path
+   * (#1261): a multi-region estate reads each stack in its own region, not in
+   * whichever one the shell is set to. */
+  region?: string;
   /** Component projects deploy one stack per component; read them all and merge. */
   componentStacks?: string[];
   owned?: boolean;
@@ -80,6 +84,7 @@ export async function observeDeep(
     buildOutput: opts.buildOutput,
     entityNames,
     entities: opts.entities,
+    ...(opts.region ? { region: opts.region } : {}),
     ...(opts.owned !== undefined ? { owned: opts.owned } : {}),
   };
   try {

package/src/lifecycle/live-diff.test.ts

@@ -336,3 +336,41 @@ describe("diffSnapshots (#822)", () => {
     expect(d.added).toEqual(["a", "b"]);
   });
 });
+
+describe("attribute comparison is key-order insensitive (#1279)", () => {
+  const inst = (attributes: Record<string, unknown>): ResourceMetadata => ({
+    type: "AWS::EC2::Instance",
+    status: "OBSERVED",
+    physicalId: "i-1",
+    attributes,
+  });
+
+  test("does not drift when a provider reorders the keys of a nested object", () => {
+    // Exactly what AWS did between two reads of the same untouched instance.
+    const result = diffLive({
+      declared: new Set(["one"]),
+      observedNow: { one: inst({ Placement: { Tenancy: "default", AvailabilityZone: "us-east-1c" } }) },
+      observedThen: { one: inst({ Placement: { AvailabilityZone: "us-east-1c", Tenancy: "default" } }) },
+    });
+    expect(result.driftedSinceSnapshot).toEqual([]);
+    expect(result.unchanged).toEqual(["one"]);
+  });
+
+  test("still reports a real change to a nested value", () => {
+    const result = diffLive({
+      declared: new Set(["one"]),
+      observedNow: { one: inst({ Placement: { AvailabilityZone: "us-east-1d" } }) },
+      observedThen: { one: inst({ Placement: { AvailabilityZone: "us-east-1c" } }) },
+    });
+    expect(result.driftedSinceSnapshot.map((d) => d.changes[0].path)).toEqual(["attributes.Placement"]);
+  });
+
+  test("keeps array order significant — for a list, order is part of the value", () => {
+    const result = diffLive({
+      declared: new Set(["one"]),
+      observedNow: { one: inst({ Rules: [{ p: 80 }, { p: 22 }] }) },
+      observedThen: { one: inst({ Rules: [{ p: 22 }, { p: 80 }] }) },
+    });
+    expect(result.driftedSinceSnapshot).toHaveLength(1);
+  });
+});

package/src/lifecycle/live-diff.ts

@@ -133,11 +133,31 @@ function compareMetadata(
   return changes;
 }
 
+/**
+ * Value equality that does not care what order a provider listed the keys in.
+ *
+ * This compared with `JSON.stringify`, which is key-order sensitive. That held
+ * while observed attributes were flat strings, and broke the moment they carried
+ * nested objects (#1279): a provider returning `{AvailabilityZone, Tenancy}` on
+ * one read and `{Tenancy, AvailabilityZone}` on the next made an unchanged
+ * instance drift on every single run. Order is not a fact about the resource,
+ * and reporting it as drift is exactly the noise this module exists to remove.
+ *
+ * Arrays stay order-sensitive — for a list, order is part of the value.
+ */
 function shallowEqual(a: unknown, b: unknown): boolean {
   if (a === b) return true;
   if (a == null || b == null) return false;
   if (typeof a !== "object" || typeof b !== "object") return false;
-  return JSON.stringify(a) === JSON.stringify(b);
+  return canonical(a) === canonical(b);
+}
+
+/** JSON with object keys sorted at every depth, so equal values stringify equally. */
+function canonical(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  const entries = Object.entries(value as Record<string, unknown>).sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0));
+  return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
 }
 
 /** The delta between two saved snapshots (#822): a two-way observed diff. */
@@ -227,8 +247,28 @@ export function diffLive(input: DiffLiveInput): LiveDiffResult {
   // same as `unowned`/`foreign` — composing with #1168's tri-state precedent:
   // an incomplete read never earns the more confident classification.
   const runtimeChildNames = new Set<string>();
+  const dependencyNames = new Set<string>();
   for (const name of observedNowNames) {
     if (declared.has(name)) continue;
+    // A referenced dependency (#1273) is observed only because something
+    // declared points at it — an account's default VPC route table, a shared
+    // subnet. Offering it as a delete/adopt candidate is wrong: it is not
+    // yours, and it changes on its own, so counting it as drift is noise.
+    // Same treatment as a runtime child, for the same reason, arrived at from
+    // the other direction — a child is something declared created, a
+    // dependency is something declared relies on.
+    if ((observedNow[name]?.referencedBy?.length ?? 0) > 0) {
+      dependencyNames.add(name);
+      continue;
+    }
+    // Ambient (#1278): observed because it exists, not because anything points
+    // at it. Reported so a caller can ask about it — "which of these are
+    // unused" is the whole point — but never counted as drift, since chant
+    // neither created it nor tracks its changes.
+    if (observedNow[name]?.ambient) {
+      dependencyNames.add(name);
+      continue;
+    }
     const chain = observedNow[name]?.ownerChain;
     if (chain?.root === "declared") {
       runtimeChildNames.add(name);
@@ -256,7 +296,10 @@ export function diffLive(input: DiffLiveInput): LiveDiffResult {
   // happened to record the same name (e.g. a StatefulSet's stable pod
   // identity) must not turn its ordinary churn into `driftedSinceSnapshot`.
   for (const name of observedNowNames) {
-    if (runtimeChildNames.has(name)) continue;
+    // Referenced dependencies (#1273) are excluded for the same reason runtime
+    // children are: they are observed to complete the picture, not to be
+    // governed, and their ordinary churn is somebody else's.
+    if (runtimeChildNames.has(name) || dependencyNames.has(name)) continue;
     const now = observedNow[name];
     const then = observedThenMap[name];
     if (!then) {

package/src/lifecycle/observe.ts

@@ -14,7 +14,8 @@ import type { BuildResult } from "../build";
 import { build as buildProject } from "../build";
 import { resolve as resolvePath } from "node:path";
 import type { SerializerResult } from "../serializer";
-import type { LiveObservation } from "../graph-ir";
+import type { LiveObservation, IREdge } from "../graph-ir";
+import type { ResourceMetadata } from "../lexicon";
 import {
   mergeObservations,
   normalizeObservation,
@@ -73,10 +74,18 @@ export async function observeResources(
   environment: string,
   plugins: ObservationLexicon[],
   buildResult: BuildResult,
-  opts?: { owned?: boolean; stacks?: Array<string | { name: string; region?: string; src?: string }> },
+  opts?: {
+    owned?: boolean;
+    stacks?: Array<string | { name: string; region?: string; src?: string }>;
+    /** Also report resources of a managed kind that nothing declares or
+     * references (#1278). Opt-in: it asks the provider what exists rather than
+     * resolving out from what is declared. */
+    ambient?: boolean;
+  },
 ): Promise<ObserveResult> {
   const owned = opts?.owned ?? true;
   const stacks = (opts?.stacks ?? []).map((st) => (typeof st === "string" ? { name: st } : st));
+  const includeAmbient = opts?.ambient ?? false;
   // A stack's `src` (multi-stack, #1162) is built SCOPED to recover that stack's
   // BARE entity names — the names it actually deploys. Matching deployed bare
   // LogicalResourceIds against the whole-project build's DISAMBIGUATED names
@@ -177,7 +186,38 @@ export async function observeResources(
           }),
         );
       }
-      pushObservation(observations, warnings, plugin.name, observed, environment, entityNames.length);
+      // What the estate depends on but does not declare (#1273). Read after the
+      // managed resources, because the declared observation is the closure's
+      // roots — there is nothing to reference out from until it exists.
+      const dependencies = await collectDependencies(plugin, {
+        environment,
+        entities,
+        observed: observed.resources,
+        stacks,
+      });
+      for (const message of dependencies.warnings) warnings.push(message);
+      // Resources of a managed kind that nothing declares or references (#1278).
+      // Bounded by what this lexicon's declared entities actually are, so a
+      // project managing security groups is not made to enumerate the account.
+      const ambient = includeAmbient
+        ? await collectAmbient(plugin, {
+            environment,
+            kinds: [...new Set([...entities.values()].map((e) => e.entityType))],
+            observed: observed.resources,
+            stacks,
+            warnings,
+          })
+        : {};
+      for (const [id, meta] of Object.entries(ambient)) dependencies.resources[id] ??= meta;
+      pushObservation(
+        observations,
+        warnings,
+        plugin.name,
+        observed,
+        environment,
+        entityNames.length,
+        dependencies,
+      );
     } catch (err) {
       // A thrown read is the whole-lexicon failure: every declared entity is
       // NOT-OBSERVED, not absent (#1089). Emitting nothing here is what made a
@@ -201,6 +241,141 @@ export async function observeResources(
   return { observations, warnings, errors };
 }
 
+/**
+ * The subset of an observation belonging to one stack.
+ *
+ * A scoped stack's ids are qualified `${stack}::${id}` (#1162), so the prefix is
+ * the whole test. An unqualified observation — single-stack, or a bare-string
+ * stack sharing one id space — has no way to be split and is returned whole,
+ * which is what it already was.
+ */
+function scopeToStack(
+  resources: Record<string, ResourceMetadata>,
+  stack: string | undefined,
+): Record<string, ResourceMetadata> {
+  if (!stack) return resources;
+  const prefix = `${stack}::`;
+  const scoped = Object.fromEntries(
+    Object.entries(resources)
+      .filter(([id]) => id.startsWith(prefix))
+      .map(([id, meta]) => [id, meta] as const),
+  );
+  // No qualified ids at all means this observation was never stack-scoped.
+  return Object.keys(scoped).length > 0 ? scoped : resources;
+}
+
+/**
+ * Ask a lexicon what exists of the kinds it manages, beyond what is declared
+ * (#1278). Once per stack for the region, merged by physical id — the same
+ * ambient resource seen from two stacks is one resource.
+ */
+export async function collectAmbient(
+  plugin: ObservationLexicon,
+  opts: {
+    environment: string;
+    kinds: string[];
+    observed: Record<string, ResourceMetadata>;
+    stacks: Array<{ name: string; region?: string; src?: string }>;
+    warnings: string[];
+  },
+): Promise<Record<string, ResourceMetadata>> {
+  if (!plugin.observeAmbient || opts.kinds.length === 0) return {};
+  const found: Record<string, ResourceMetadata> = {};
+  const refs = opts.stacks.length > 0 ? opts.stacks : [{ name: undefined, region: undefined }];
+  for (const ref of refs) {
+    try {
+      const part = await plugin.observeAmbient({
+        environment: opts.environment,
+        kinds: opts.kinds,
+        observed: opts.observed,
+        ...(ref.name ? { stack: ref.name } : {}),
+        ...(ref.region ? { region: ref.region } : {}),
+      });
+      for (const [id, meta] of Object.entries(part)) found[id] ??= meta;
+    } catch (err) {
+      opts.warnings.push(
+        `${plugin.name}: ambient resources not read${ref.name ? ` for stack "${ref.name}"` : ""} — ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+  return found;
+}
+
+/** Dependencies collected across a lexicon's stacks, plus anything to report. */
+export interface CollectedDependencies {
+  resources: Record<string, ResourceMetadata>;
+  edges: IREdge[];
+  warnings: string[];
+}
+
+const NO_DEPENDENCIES: CollectedDependencies = { resources: {}, edges: [], warnings: [] };
+
+/**
+ * Ask a lexicon what its declared estate references but does not manage (#1273).
+ *
+ * Called once per stack, because the closure roots and the region differ per
+ * stack, and merged by key. Dependencies are keyed by physical id and are
+ * deliberately NOT stack-qualified: the account's default VPC route table is the
+ * same resource whichever stack routes through it, and qualifying it would
+ * produce one node per referrer and an edge to each.
+ *
+ * Best-effort. A lexicon that does not implement the hook, or one whose read
+ * fails, contributes nothing — the managed observation is already complete and
+ * useful on its own, and failing it because an ambient dependency could not be
+ * read would trade a whole answer for a partial one.
+ */
+export async function collectDependencies(
+  plugin: ObservationLexicon,
+  opts: {
+    environment: string;
+    entities: Map<string, { entityType: string; props: Record<string, unknown> }>;
+    observed: Record<string, ResourceMetadata>;
+    stacks: Array<{ name: string; region?: string; src?: string }>;
+  },
+): Promise<CollectedDependencies> {
+  if (!plugin.observeDependencies) return NO_DEPENDENCIES;
+
+  const resources: Record<string, ResourceMetadata> = {};
+  const edges: IREdge[] = [];
+  const warnings: string[] = [];
+  const refs = opts.stacks.length > 0 ? opts.stacks : [{ name: undefined, region: undefined }];
+
+  for (const ref of refs) {
+    // Only this stack's resources are the closure's roots. Handing a lexicon
+    // the whole estate makes it resolve out from resources that live somewhere
+    // else — for AWS that means `describe-instances` in one region with another
+    // region's instance ids, which fails outright with InvalidInstanceID and
+    // takes the whole read down with it.
+    const roots = scopeToStack(opts.observed, ref.name);
+    if (Object.keys(roots).length === 0) continue;
+    try {
+      const found = await plugin.observeDependencies({
+        environment: opts.environment,
+        entities: opts.entities,
+        observed: roots,
+        ...(ref.name ? { stack: ref.name } : {}),
+        ...(ref.region ? { region: ref.region } : {}),
+      });
+      for (const [id, meta] of Object.entries(found.resources)) {
+        // Merge referrers rather than overwrite: two stacks routing through the
+        // same table is one node reached twice, and the reason it is here is
+        // both of them.
+        const existing = resources[id];
+        resources[id] = existing
+          ? { ...existing, referencedBy: [...new Set([...(existing.referencedBy ?? []), ...(meta.referencedBy ?? [])])] }
+          : meta;
+      }
+      edges.push(...(found.edges ?? []));
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      warnings.push(
+        `${plugin.name}: dependencies not read${ref.name ? ` for stack "${ref.name}"` : ""} — ${message}`,
+      );
+    }
+  }
+  return { resources, edges, warnings };
+}
+
 /** Record one lexicon's observation, warning once per unobserved entity. */
 function pushObservation(
   observations: LiveObservation[],
@@ -209,6 +384,7 @@ function pushObservation(
   observed: NormalizedObservation,
   environment: string,
   declaredCount: number,
+  dependencies: CollectedDependencies = NO_DEPENDENCIES,
 ): void {
   const hasResources = Object.keys(observed.resources).length > 0;
   const unobservedNames = Object.keys(observed.unobserved);
@@ -225,9 +401,15 @@ function pushObservation(
     if (notice) warnings.push(notice);
     return;
   }
+  // Dependencies ride alongside the managed resources so they become nodes, and
+  // carry `referencedBy` so every consumer can still tell the two apart.
+  const hasDependencies = Object.keys(dependencies.resources).length > 0;
   observations.push({
     lexicon,
-    resources: observed.resources,
+    resources: hasDependencies
+      ? { ...observed.resources, ...dependencies.resources }
+      : observed.resources,
     ...(unobservedNames.length > 0 ? { unobserved: observed.unobserved } : {}),
+    ...(dependencies.edges.length > 0 ? { edges: dependencies.edges } : {}),
   });
 }