@intentius/chant 0.33.0 → 0.34.0

package/src/lifecycle/replay.ts

@@ -0,0 +1,141 @@
+/**
+ * Replay a recorded snapshot as if it had just been observed (#1266, #1279).
+ *
+ * Shared by `chant search --at` and `chant graph --at`. A snapshot already holds
+ * what an observation is — resources with their physical ids and attributes,
+ * and since #1266 the relationships between them — so turning it back into
+ * `LiveObservation[]` lets a replay rejoin the live path at `buildLiveGraphIr`.
+ * Every fold, overlay and edge reconstruction below that point is shared, and
+ * nothing downstream needs to know which source it got.
+ *
+ * That sharing is the reason this is a module rather than a helper inside one
+ * command: `graph` had no `--at` at all, so anyone wanting the raw IR of a
+ * recorded estate had to reach for the live endpoint, and a snapshot could
+ * answer most questions but never all of them.
+ */
+
+import { readEnvironmentSnapshots } from "./git";
+import type { LiveObservation } from "../graph-ir";
+import type { LifecycleSnapshot } from "./types";
+
+/**
+ * Rebuild observations from a recorded snapshot (#1266).
+ *
+ * A snapshot already holds what an observation is: resources with their
+ * physical ids and attributes, and — since #1266 — the relationships between
+ * them. Turning it back into `LiveObservation[]` means the replay rejoins the
+ * live path at `buildLiveGraphIr`, and every fold, overlay and query below that
+ * is shared. Nothing downstream needs to know which source it got.
+ *
+ * `latest` is the only ref for now. A specific commit is the natural extension
+ * and the storage already supports it (`readSnapshotAt`), but "answer from what
+ * is recorded" is the question worth settling first.
+ */
+/**
+ * Whether this environment has a recording, without reading one.
+ *
+ * Used to turn "the estate could not be read" into "the estate could not be
+ * read, and you already have a recording of it". Cheap and best-effort: a
+ * failure here means the caller says the plain version of the message, never
+ * that the command fails.
+ */
+export async function hasSnapshot(environment: string): Promise<boolean> {
+  try {
+    return (await readEnvironmentSnapshots(environment)).size > 0;
+  } catch {
+    return false;
+  }
+}
+
+export async function replaySnapshots(
+  environment: string,
+  ref: string,
+  scopedStacks: Set<string>,
+): Promise<{ observations: LiveObservation[]; commit: string; timestamp: string } | { error: string; hint?: string }> {
+  if (ref !== "latest" && ref !== "true") {
+    return {
+      error: `chant search --at only accepts "latest" for now, got "${ref}"`,
+      hint: "a specific snapshot commit is not wired up yet",
+    };
+  }
+  const stored = await readEnvironmentSnapshots(environment);
+  if (stored.size === 0) {
+    return {
+      error: `No snapshots found for environment "${environment}"`,
+      hint: `Record one first: chant lifecycle snapshot ${environment}`,
+    };
+  }
+  const observations: LiveObservation[] = [];
+  let commit = "";
+  let timestamp = "";
+  // Ambient and dependency resources are keyed by physical id and are
+  // account-level: the default security group three stacks each recorded is one
+  // group, not three. Managed resources are stack-qualified below and cannot
+  // collide, so only the unqualified ones need this.
+  const seenUnqualified = new Set<string>();
+  // A stack's snapshot could only exclude what THAT stack manages, so a stack
+  // declaring no security groups reported the neighbouring stack's as ambient.
+  // The union is only knowable here, with every snapshot in hand.
+  const managedPhysicalIds = new Set<string>();
+  for (const content of stored.values()) {
+    const snap = JSON.parse(content) as LifecycleSnapshot;
+    for (const meta of Object.values(snap.resources ?? {})) {
+      if (!meta.ambient && !meta.referencedBy?.length && meta.physicalId) {
+        managedPhysicalIds.add(meta.physicalId);
+      }
+    }
+  }
+  for (const [key, content] of stored) {
+    const snapshot = JSON.parse(content) as LifecycleSnapshot;
+    // The storage key is `<stack>__<lexicon>` for a multi-stack project; the
+    // snapshot carries its own lexicon, which is the one to trust.
+    const lexicon = snapshot.lexicon ?? key;
+    // A scoped stack's ids are qualified `${stack}::${id}` on the live path
+    // (#1162), because the same bare LogicalResourceId exists in every region's
+    // stack. A snapshot stores them bare, so a replay has to re-apply the same
+    // rule — otherwise `server` from us-west-1 and `server` from us-west-2
+    // collide, and none of them join the declared canvas, which qualifies.
+    const stack = snapshot.stack;
+    const qualify = stack !== undefined && scopedStacks.has(stack);
+    // Dependencies (#1273) are keyed by physical id and are account-level: the
+    // default VPC's route table is one resource however many stacks route
+    // through it. Qualifying those would split it per stack and break the
+    // edges into it.
+    const managed = (id: string, meta: { referencedBy?: string[]; ambient?: boolean }): string =>
+      qualify && !meta.ambient && !(meta.referencedBy && meta.referencedBy.length > 0)
+        ? `${stack}::${id}`
+        : id;
+    const resources: Record<string, (typeof snapshot.resources)[string]> = {};
+    for (const [id, meta] of Object.entries(snapshot.resources ?? {})) {
+      const key = managed(id, meta);
+      if (key === id) {
+        // Ambient means "nothing manages this". Another stack managing it makes
+        // that false, and reporting it twice would inflate any count over it.
+        if (meta.ambient && meta.physicalId && managedPhysicalIds.has(meta.physicalId)) continue;
+        // Unqualified: account-level, so first sighting wins and the rest are
+        // the same resource seen again from another stack's snapshot.
+        if (seenUnqualified.has(id)) continue;
+        seenUnqualified.add(id);
+      }
+      resources[key] = meta;
+    }
+    const known = new Set(Object.keys(snapshot.resources ?? {}));
+    const requalify = (id: string): string => {
+      const meta = (snapshot.resources ?? {})[id];
+      return known.has(id) && meta ? managed(id, meta) : id;
+    };
+    const edges = (snapshot.edges ?? []).map((e) => ({ ...e, from: requalify(e.from), to: requalify(e.to) }));
+    observations.push({
+      lexicon,
+      resources,
+      ...(edges.length > 0 ? { edges } : {}),
+    });
+    commit ||= snapshot.commit ?? "";
+    // Report the OLDEST timestamp across stacks: a caller asking how stale this
+    // answer is wants the weakest link, not the freshest one.
+    if (!timestamp || (snapshot.timestamp && snapshot.timestamp < timestamp)) {
+      timestamp = snapshot.timestamp ?? timestamp;
+    }
+  }
+  return { observations, commit, timestamp };
+}

package/src/lifecycle/snapshot.test.ts

@@ -1,6 +1,8 @@
 import { describe, test, expect, vi, beforeEach } from "vitest";
 import { createMockPlugin, staticDescribeResources, staticObservation, staticListArtifacts } from "@intentius/chant-test-utils";
 import type { BuildResult } from "../build";
+import type { DeepResourceObservation } from "../deep-observation";
+import type { UnobservedEntity } from "../observation";
 
 const writeSnapshotMock = vi.fn();
 const getHeadCommitMock = vi.fn();
@@ -83,6 +85,126 @@ describe("takeSnapshot", () => {
     expect(writeSnapshotMock.mock.calls[0][1]).toBe("loom-backend__aws");
   });
 
+  test("region option: the stack's own region reaches describeResources (#1261)", async () => {
+    let observedRegion: string | undefined = "unset";
+    const plugin = createMockPlugin({
+      name: "aws",
+      describeResources: async (options: { region?: string }) => {
+        observedRegion = options.region;
+        return { bucket: { type: "AWS::S3::Bucket", status: "CREATE_COMPLETE", physicalId: "b" } };
+      },
+    });
+    await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), {
+      stack: "loom-us-west-2",
+      region: "us-west-2",
+    });
+    // Without this the reader falls back to the ambient region, and every stack
+    // outside it snapshots as "no valid resources or artifacts returned".
+    expect(observedRegion).toBe("us-west-2");
+  });
+
+  test("no region declared: describeResources keeps its ambient-region default", async () => {
+    let observedRegion: string | undefined = "unset";
+    const plugin = createMockPlugin({
+      name: "aws",
+      describeResources: async (options: { region?: string }) => {
+        observedRegion = options.region;
+        return { bucket: { type: "AWS::S3::Bucket", status: "CREATE_COMPLETE", physicalId: "b" } };
+      },
+    });
+    await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }));
+    expect(observedRegion).toBeUndefined();
+  });
+
+  // #1267 — a snapshot records identity by default; --deep also records each
+  // resource's property tree, which is what a fold over topology needs.
+  describe("deep snapshots (#1267)", () => {
+    const identity = { bucket: { type: "AWS::S3::Bucket", status: "CREATE_COMPLETE", physicalId: "b" } };
+
+    function deepPlugin(
+      resources: Record<string, DeepResourceObservation>,
+      unobserved: Record<string, UnobservedEntity> = {},
+    ) {
+      return createMockPlugin({
+        name: "aws",
+        describeResources: staticDescribeResources(identity),
+        observeResourcesDeep: async () => ({ deepObservation: "v1" as const, resources, unobserved }),
+      });
+    }
+
+    test("without --deep: identity only, and depth is not written", async () => {
+      const plugin = deepPlugin({
+        bucket: { type: "AWS::S3::Bucket", physicalId: "b", properties: { versioning: "Enabled" } },
+      });
+      const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }));
+      // Absent, not "identity" — every snapshot written before #1267 is thin,
+      // and a reader must treat a missing field as thin rather than unknown.
+      expect(result.snapshots[0].depth).toBeUndefined();
+      expect(result.snapshots[0].properties).toBeUndefined();
+    });
+
+    test("with --deep: records the property trees alongside identity", async () => {
+      const plugin = deepPlugin({
+        bucket: { type: "AWS::S3::Bucket", physicalId: "b", properties: { versioning: "Enabled" } },
+      });
+      const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), { deep: true });
+      expect(result.snapshots[0].depth).toBe("deep");
+      expect(result.snapshots[0].properties?.bucket.properties).toEqual({ versioning: "Enabled" });
+      // Identity is still there — deep adds, it does not replace.
+      expect(result.snapshots[0].resources.bucket).toMatchObject({ type: "AWS::S3::Bucket" });
+    });
+
+    test("--deep against a lexicon with no deep reader: identity snapshot plus a warning", async () => {
+      const plugin = createMockPlugin({ name: "aws", describeResources: staticDescribeResources(identity) });
+      const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), { deep: true });
+      // Still a usable snapshot, but it must not claim a depth it does not have.
+      expect(result.snapshots).toHaveLength(1);
+      expect(result.snapshots[0].depth).toBeUndefined();
+      expect(result.warnings.join("\n")).toContain("no deep reader");
+    });
+
+    test("--deep returning nothing: downgrades to identity rather than discarding a good snapshot", async () => {
+      const plugin = deepPlugin({});
+      const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), { deep: true });
+      expect(result.snapshots).toHaveLength(1);
+      expect(result.snapshots[0].depth).toBeUndefined();
+      expect(result.warnings.join("\n")).toContain("deep read returned no properties");
+    });
+
+    test("--deep passes the stack's region to the deep reader (#1261 family)", async () => {
+      let seen: string | undefined = "unset";
+      const plugin = createMockPlugin({
+        name: "aws",
+        describeResources: staticDescribeResources(identity),
+        observeResourcesDeep: async (options: { region?: string }) => {
+          seen = options.region;
+          return {
+            deepObservation: "v1" as const,
+            resources: { bucket: { type: "AWS::S3::Bucket", physicalId: "b", properties: {} } },
+            unobserved: {},
+          };
+        },
+      });
+      await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), {
+        stack: "app-us-west-2",
+        region: "us-west-2",
+        deep: true,
+      });
+      // Without this the deep read targets the ambient region and comes back
+      // empty, which downgrades a multi-region snapshot to identity silently.
+      expect(seen).toBe("us-west-2");
+    });
+
+    test("--deep reports entities the deep reader could not read", async () => {
+      const plugin = deepPlugin(
+        { bucket: { type: "AWS::S3::Bucket", physicalId: "b", properties: {} } },
+        { queue: { reason: "read-failed", detail: "boom" } },
+      );
+      const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["bucket"] }), { deep: true });
+      expect(result.warnings.join("\n")).toContain("not observed deeply");
+    });
+  });
+
   test("plugin without describeResources is skipped", async () => {
     const plugin = createMockPlugin({ name: "aws" });
     const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["x"] }));
@@ -231,3 +353,60 @@ describe("takeSnapshot", () => {
     expect(result.snapshots).toEqual([]);
   });
 });
+
+// #1266 — a snapshot that records only what it manages cannot answer a fold
+// question when it is replayed: the account's default VPC routing is not in it,
+// so `internetFacing` is unanswerable and `search --at` would be quietly weaker
+// than `search --live`.
+describe("dependencies and edges in a snapshot (#1266)", () => {
+  const identity = { webServer: { type: "AWS::EC2::Instance", status: "OK", physicalId: "i-1" } };
+
+  test("records the dependencies the estate references, and the edges to them", async () => {
+    const plugin = createMockPlugin({
+      name: "aws",
+      describeResources: staticDescribeResources(identity),
+      observeDependencies: async () => ({
+        resources: {
+          "rtb-default": {
+            type: "AWS::EC2::RouteTable",
+            status: "OBSERVED",
+            physicalId: "rtb-default",
+            referencedBy: ["webServer"],
+          },
+        },
+        edges: [{ from: "webServer", to: "rtb-default", kind: "ref" as const, viaAttr: "RouteTableId" }],
+      }),
+    });
+    const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["webServer"] }));
+    // Both, in one record: what exists and how it connects.
+    expect(result.snapshots[0].resources["rtb-default"]).toMatchObject({ referencedBy: ["webServer"] });
+    expect(result.snapshots[0].edges).toEqual([
+      { from: "webServer", to: "rtb-default", kind: "ref", viaAttr: "RouteTableId" },
+    ]);
+  });
+
+  test("a lexicon with no dependency reader snapshots exactly as before", async () => {
+    const plugin = createMockPlugin({ name: "aws", describeResources: staticDescribeResources(identity) });
+    const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["webServer"] }));
+    expect(Object.keys(result.snapshots[0].resources)).toEqual(["webServer"]);
+    // Absent, not empty — "no relationships recorded", not "none existed".
+    expect(result.snapshots[0].edges).toBeUndefined();
+  });
+
+  test("a dependency read that fails warns and keeps the managed snapshot", async () => {
+    const plugin = createMockPlugin({
+      name: "aws",
+      describeResources: staticDescribeResources(identity),
+      observeDependencies: async () => {
+        throw new Error("route tables unreadable");
+      },
+    });
+    const result = await takeSnapshot("prod", [plugin], makeBuildResult({ aws: ["webServer"] }));
+    // The managed observation is complete and useful on its own; losing it
+    // because an ambient dependency could not be read trades a whole answer
+    // for none.
+    expect(result.snapshots).toHaveLength(1);
+    expect(result.snapshots[0].resources.webServer).toBeDefined();
+    expect(result.warnings.join("\n")).toContain("dependencies not read");
+  });
+});

package/src/lifecycle/snapshot.ts

@@ -5,8 +5,11 @@
 import type { ObservationLexicon, ResourceMetadata, ArtifactMetadata } from "../lexicon";
 import type { BuildResult } from "../build";
 import type { SerializerResult } from "../serializer";
-import type { LifecycleSnapshot } from "./types";
+import type { LifecycleSnapshot, ObservationDepth } from "./types";
+import { observeDeep } from "./deep-observe";
+import type { DeepResourceObservation } from "../deep-observation";
 import { computeBuildDigest } from "./digest";
+import { collectDependencies, collectAmbient } from "./observe";
 import { writeSnapshot, snapshotStorageKey, getHeadCommit, pushLifecycle } from "./git";
 import { sortedJsonReplacer } from "../utils";
 import { formatUnobserved, normalizeObservation, unobservedAll, type UnobservedEntity } from "../observation";
@@ -80,9 +83,24 @@ export async function takeSnapshot(
   environment: string,
   plugins: ObservationLexicon[],
   buildResult: BuildResult,
-  opts?: { cwd?: string; stack?: string },
+  opts?: {
+    cwd?: string;
+    stack?: string;
+    region?: string;
+    deep?: boolean;
+    ambient?: boolean;
+    /** Kinds the PROJECT manages, not just this stack — a region whose stack
+     * declares no security group still has a default one (#1278). */
+    ambientKinds?: string[];
+  },
 ): Promise<TakeSnapshotResult> {
   const stack = opts?.stack;
+  const depth: ObservationDepth = opts?.deep ? "deep" : "identity";
+  // A stack declares the region it deploys to (#1261). Passing it through is
+  // what lets a multi-region estate be observed at all: the reader targets the
+  // stack's own region rather than whichever one the shell happens to be set
+  // to, so out-of-region stacks stop coming back empty.
+  const region = opts?.region;
   const warnings: string[] = [];
   const errors: string[] = [];
   const snapshots: LifecycleSnapshot[] = [];
@@ -131,6 +149,7 @@ export async function takeSnapshot(
             entityNames,
             entities,
             stack,
+            region,
           }),
         );
         const { valid, dropped, warnings: validationWarnings } = validateResources(observed.resources);
@@ -149,6 +168,8 @@ export async function takeSnapshot(
       }
 
       if (plugin.listArtifacts) {
+        // No region: artifacts are registry/chart objects (docker, helm), not
+        // regional cloud resources, and `listArtifacts` takes no region.
         const raw = await plugin.listArtifacts({ environment, entities, stack });
         const { valid, dropped, warnings: validationWarnings } = validateResources(raw);
         warnings.push(...validationWarnings);
@@ -168,15 +189,79 @@ export async function takeSnapshot(
         continue;
       }
 
+      // The deep read is a second pass, after identity is known to be readable
+      // (#1267). Keeping it separate means a lexicon with no deep reader still
+      // snapshots exactly as before, and a deep read that comes back empty
+      // downgrades the record rather than discarding an identity snapshot that
+      // was already good.
+      let properties: Record<string, DeepResourceObservation> | undefined;
+      let recordedDepth: ObservationDepth = "identity";
+      if (depth === "deep") {
+        if (!plugin.observeResourcesDeep) {
+          warnings.push(
+            `${plugin.name}: no deep reader — recording an identity snapshot; property questions cannot be answered from it`,
+          );
+        } else {
+          const observed = await observeDeep(plugin, {
+            environment,
+            buildOutput,
+            entities,
+            ...(stack ? { stack } : {}),
+            ...(region ? { region } : {}),
+          });
+          for (const [name, entry] of Object.entries(observed.unobserved)) {
+            warnings.push(`${plugin.name}: not observed deeply — ${formatUnobserved(name, entry)}`);
+          }
+          if (Object.keys(observed.resources).length > 0) {
+            properties = observed.resources;
+            recordedDepth = "deep";
+          } else {
+            warnings.push(
+              `${plugin.name}: deep read returned no properties — recording an identity snapshot`,
+            );
+          }
+        }
+      }
+      // What this estate depends on but does not manage (#1273), recorded so a
+      // replayed snapshot can answer the same questions a live read can (#1266).
+      // Without them a snapshot holds the managed resources and no route to the
+      // account's default VPC, so `internetFacing` is unanswerable from it —
+      // which would make `search --at` quietly weaker than `search --live`.
+      const dependencies = await collectDependencies(plugin, {
+        environment,
+        entities,
+        observed: resources,
+        stacks: stack ? [{ name: stack, ...(region ? { region } : {}) }] : [],
+      });
+      for (const message of dependencies.warnings) warnings.push(message);
+      // Ambient resources (#1278) are recorded too when asked for, so a replayed
+      // snapshot can answer "which of these are unused" without a live read.
+      // Without this `search --at --ambient` filters a set that was never
+      // recorded and silently returns nothing.
+      const ambient = opts?.ambient
+        ? await collectAmbient(plugin, {
+            environment,
+            kinds: opts?.ambientKinds ?? [...new Set([...entities.values()].map((e) => e.entityType))],
+            observed: resources,
+            stacks: stack ? [{ name: stack, ...(region ? { region } : {}) }] : [],
+            warnings,
+          })
+        : {};
+      const withDependencies = { ...resources, ...dependencies.resources, ...ambient };
+
       const snapshot: LifecycleSnapshot = {
         lexicon: plugin.name,
         environment,
         ...(stack ? { stack } : {}),
         commit: headCommit,
         timestamp,
-        resources,
+        resources: withDependencies,
+        ...(dependencies.edges.length > 0 ? { edges: dependencies.edges } : {}),
         ...(Object.keys(unobserved).length > 0 && { unobserved }),
         ...(Object.keys(artifacts).length > 0 && { artifacts }),
+        // Only written when deep. An absent field means identity, which is what
+        // every snapshot taken before #1267 was.
+        ...(recordedDepth === "deep" && { depth: recordedDepth, properties }),
         digest,
       };
 

package/src/lifecycle/types.ts

@@ -1,8 +1,25 @@
 import type { ResourceMetadata, ArtifactMetadata } from "../lexicon";
 import type { UnobservedEntity } from "../observation";
+import type { DeepResourceObservation } from "../deep-observation";
+import type { IREdge } from "../graph-ir";
 
 export type { ResourceMetadata, ArtifactMetadata } from "../lexicon";
 
+/**
+ * How much of each resource an observation actually read (#1267).
+ *
+ * `identity` is the thin path: logical name, type, physical id, status. It is
+ * what a snapshot recorded before deep reads existed, and it stays the default
+ * because a deep read costs more provider calls and a larger record.
+ *
+ * `deep` additionally carries each resource's normalized property tree, which
+ * is what a fold over topology needs — a subnet's route-table association, a
+ * security group's rules. A consumer must branch on this rather than assume:
+ * asking an `identity` snapshot a property question has no answer, and
+ * silently returning nothing would read as "no such resources".
+ */
+export type ObservationDepth = "identity" | "deep";
+
 /**
  * State snapshot for a single lexicon in an environment.
  */
@@ -28,6 +45,36 @@ export interface LifecycleSnapshot {
   unobserved?: Record<string, UnobservedEntity>;
   /** Artifact metadata keyed by server-side identifier (lexicon-specific). */
   artifacts?: Record<string, ArtifactMetadata>;
+  /**
+   * How much of each resource this snapshot read (#1267). Absent means
+   * `identity` — every snapshot written before deep reads existed was thin, and
+   * treating a missing field as unknown rather than as thin would invalidate
+   * them all.
+   */
+  depth?: ObservationDepth;
+  /**
+   * Normalized per-resource property trees, present only at `deep` depth and
+   * keyed by the same logical names as `resources`.
+   *
+   * Kept beside `resources` rather than merged into it so the thin record stays
+   * exactly what it always was: a reader that only wants identity does not have
+   * to learn a new shape, and an old snapshot and a new one parse the same way.
+   */
+  properties?: Record<string, DeepResourceObservation>;
+  /**
+   * Relationships observed between the recorded resources (#1266).
+   *
+   * `resources` says what existed; without this a snapshot cannot say how any
+   * of it connected, so a fold over topology has nothing to traverse when the
+   * snapshot is replayed. That is the difference between a snapshot answering
+   * "which instances exist" and answering "which are reachable from the
+   * internet" — and the second is the whole reason the graph is worth
+   * recording.
+   *
+   * Absent on every snapshot written before this, which is read as "no
+   * relationships recorded" rather than "no relationships existed".
+   */
+  edges?: IREdge[];
   /** Build digest at snapshot time — what was declared when this snapshot was taken */
   digest?: BuildDigest;
 }