@intentius/chant 0.105.1 → 0.106.3

package/src/components/pr-run.test.ts

@@ -12,8 +12,21 @@ import { describe, expect, test } from "vitest";
 import { CapabilityRegistry, type DeployContext } from "./capability";
 import { memoryGateLedgerPort } from "../op/gate";
 import { planFanOut } from "./fan-out";
-import { applyPrSet, approvalOf, approversWithoutReview, decidePrGate, planPrSet, readOnlyLedger } from "./pr-run";
-import { changeSetDigest } from "../change-set";
+import {
+  applyPrSet,
+  approvalOf,
+  approversWithoutReview,
+  decidePrGate,
+  planCoveredBy,
+  planPrSet,
+  prReport,
+  readOnlyLedger,
+  resumePrSet,
+  type PrPlanSet,
+} from "./pr-run";
+import { changeSetDigest, changeSetDocumentDigest, composeChangeSet, type ChangeSetEntry, type ChangeSetPart } from "../change-set";
+import type { PrApplyRecord } from "./fan-out-record";
+import type { FanOutPlan } from "./fan-out";
 import { computePlanDigest } from "../lifecycle/plan-digest";
 import type { DriverComponent } from "./driver";
 
@@ -66,7 +79,17 @@ function world() {
         changeSet: {
           member: { member: input.root, lexicon: "terraform", planner: "tofu", status: "planned", planDigest, holes: [] },
           entries: changes
-            ? [{ member: input.root, lexicon: "terraform", planner: "tofu", address: "terraform_data.x", type: "terraform_data", action: "update", attributes: [] }]
+            ? [
+                {
+                  member: input.root,
+                  lexicon: "terraform",
+                  planner: "tofu",
+                  address: "terraform_data.x",
+                  type: "terraform_data",
+                  action: "update",
+                  attributes: [{ path: "version", before: "v1", after: w.versions[input.root] }],
+                },
+              ]
             : [],
         },
       };
@@ -117,7 +140,8 @@ describe("planPrSet", () => {
       ["a", "planned", 1],
       ["app", "planned", 0],
     ]);
-    expect(set.doc.digest).toBe(changeSetDigest(set.members.map((m) => ({ member: m.member, planDigest: m.planDigest }))));
+    expect(set.doc.digest).toBe(changeSetDocumentDigest(set.doc));
+    expect(set.doc.digest).not.toBe(changeSetDigest(set.members.map((m) => ({ member: m.member, planDigest: m.planDigest }))));
     expect(set.failed).toBe(false);
   });
 
@@ -203,3 +227,286 @@ describe("applyPrSet", () => {
     expect(members.find((m) => m.member === "a")?.error).toBe("a failed");
   });
 });
+
+/** The record a first attempt leaves (`pr-apply --resume`), from the set it applied. */
+function recordOf(set: PrPlanSet, approvedBy = "github:alice"): PrApplyRecord {
+  return {
+    op: "pr-12",
+    gate: "pr-apply",
+    head: "2".repeat(40),
+    digest: set.doc.digest,
+    approvedBy: [approvedBy],
+    members: set.members,
+    changeSet: set.doc,
+    planOutputs: set.outputs,
+  };
+}
+
+/** Plan and apply once, with `failing` failing to apply. */
+async function firstAttempt(w: ReturnType<typeof world>, plan: FanOutPlan, failing: string[]) {
+  for (const f of failing) w.failing.add(f);
+  const set = await planPrSet({ components: ESTATE, plan, registry: w.registry, env: "prod" });
+  const first = await applyPrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", set });
+  for (const f of failing) w.failing.delete(f);
+  return { set, first, record: recordOf(set) };
+}
+
+describe("resuming an apply that failed partway (#3464)", () => {
+  test("plans and applies only what did not apply, and the set keeps the approved digest", async () => {
+    const w = world();
+    const plan = planFanOut({ components: ESTATE, changed: ["net", "dns"] });
+    const { set, first, record } = await firstAttempt(w, plan, ["a"]);
+    expect(first.status).toBe("failed");
+    expect(first.run.completed).toEqual(["b", "dns", "net"]);
+
+    w.planned.length = 0;
+    w.applied.length = 0;
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: first.run.completed });
+    if (!resumed.ok) throw new Error(resumed.reasons.join("; "));
+
+    expect(resumed.plan.order).toEqual(["a", "app"]);
+    // Planned against the outputs the approved plans read, so each member plans as it did.
+    expect(w.planned).toEqual([
+      { root: "a", vars: { cidr: "10.0.0.0/16" } },
+      { root: "app", vars: { subnet: "a-v1" } },
+    ]);
+    expect(resumed.set.doc.digest).toBe(set.doc.digest);
+    expect(resumed.fresh.members.map((m) => m.planDigest)).toEqual(
+      set.members.filter((m) => m.member === "a" || m.member === "app").map((m) => m.planDigest),
+    );
+
+    // The gate is decided against the digest that was approved, and it still stands.
+    const ledger = memoryGateLedgerPort({ resolutions: [approval(set.doc.digest)] });
+    const check = await decidePrGate(ledger, { op: "pr-12", gate: "pr-apply", digest: resumed.set.doc.digest, now: NOW });
+    expect(check.satisfied).toBe(true);
+    expect(ledger.appended).toEqual([]);
+
+    const second = await applyPrSet({
+      components: ESTATE,
+      plan: resumed.plan,
+      registry: w.registry,
+      env: "prod",
+      set: resumed.set,
+      resumed: { completed: first.run.completed, outputs: {} },
+    });
+    expect(second.status).toBe("applied");
+    expect(w.applied).toEqual(["a", "app"]);
+    expect(w.fromPlan.a).toBe(true);
+    expect(Object.fromEntries(second.members.map((m) => [m.member, m.status]))).toEqual({
+      a: "applied",
+      app: "applied",
+      b: "applied",
+      dns: "applied",
+      net: "applied",
+    });
+  });
+
+  test("the resumed report carries the approved digest and names what the earlier attempt applied", async () => {
+    const w = world();
+    const plan = planFanOut({ components: ESTATE, changed: ["net", "dns"] });
+    const { set, first, record } = await firstAttempt(w, plan, ["a"]);
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: first.run.completed });
+    if (!resumed.ok) throw new Error(resumed.reasons.join("; "));
+    const second = await applyPrSet({
+      components: ESTATE,
+      plan: resumed.plan,
+      registry: w.registry,
+      env: "prod",
+      set: resumed.set,
+      resumed: { completed: first.run.completed, outputs: {} },
+    });
+    const report = prReport({
+      stage: "apply",
+      pr: 12,
+      env: "prod",
+      base: "1".repeat(40),
+      head: "2".repeat(40),
+      op: "pr-12",
+      gate: "pr-apply",
+      plan,
+      signal: { changed: ["net", "dns"], unclaimed: [] } as never,
+      set: resumed.set,
+      approval: { status: "approved", approvedBy: ["github:alice"] },
+      status: second.status,
+      members: second.members,
+      resumed: first.run.completed,
+    });
+    expect(report.digest).toBe(set.doc.digest);
+    expect(report.changeSet).toBe(set.doc);
+    expect(report.resumed).toEqual({ applied: ["b", "dns", "net"] });
+  });
+
+  test("a dependent whose dependency applied in the earlier attempt applies its approved plan and says its inputs moved", async () => {
+    const w = world();
+    w.versions.a = "v2";
+    const plan = planFanOut({ components: ESTATE, changed: ["a"] });
+    const { first, record } = await firstAttempt(w, plan, ["app"]);
+    expect(first.run.completed).toEqual(["a"]);
+    const outputs = { a: first.run.componentOutputs.a };
+    expect(outputs.a).toEqual({ id: "a-v2" });
+
+    w.planned.length = 0;
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: ["a"] });
+    if (!resumed.ok) throw new Error(resumed.reasons.join("; "));
+    // app is planned against the value it was approved with, not the one a wrote since.
+    expect(w.planned).toEqual([{ root: "app", vars: { subnet: "a-v1" } }]);
+
+    const second = await applyPrSet({
+      components: ESTATE,
+      plan: resumed.plan,
+      registry: w.registry,
+      env: "prod",
+      set: resumed.set,
+      resumed: { completed: ["a"], outputs },
+    });
+    expect(second.members.map((m) => [m.member, m.status, m.inputsMoved])).toEqual([
+      ["a", "applied", undefined],
+      ["app", "applied", ["a"]],
+    ]);
+  });
+
+  test("a member left over whose plan moved is not covered, and says why", async () => {
+    const w = world();
+    w.versions.a = "v2";
+    const plan = planFanOut({ components: ESTATE, changed: ["net", "a"] });
+    const { first, record } = await firstAttempt(w, plan, ["a"]);
+    expect(first.run.completed).toEqual(["b", "net"]);
+
+    w.versions.a = "v3";
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: first.run.completed });
+    expect(resumed.ok).toBe(false);
+    if (resumed.ok) return;
+    expect(resumed.reasons).toEqual(["a now writes terraform_data.x.version to a different value than the approved plan"]);
+  });
+
+  test("a member left over that fails to plan is not covered", async () => {
+    const w = world();
+    const plan = planFanOut({ components: ESTATE, changed: ["net"] });
+    const { first, record } = await firstAttempt(w, plan, ["a"]);
+    w.planFailing.add("a");
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: first.run.completed });
+    expect(resumed.ok).toBe(false);
+    if (resumed.ok) return;
+    expect(resumed.reasons[0]).toMatch(/^a failed to plan/);
+  });
+
+  test("a record whose change set is not the one its digest names does not resume (#3543)", async () => {
+    const w = world();
+    const plan = planFanOut({ components: ESTATE, changed: ["net"] });
+    const { set, first, record } = await firstAttempt(w, plan, ["a"]);
+    const other = { ...set.doc, digest: "jcs1-sha256:" + "0".repeat(64) };
+    for (const forged of [
+      { ...record, changeSet: other },
+      { ...record, digest: other.digest, changeSet: other },
+      { ...record, changeSet: { ...set.doc, members: set.doc.members.map((m) => (m.member === "a" ? { ...m, planDigest: "jcs1-sha256:" + "1".repeat(64) } : m)) } },
+      // The digest binds the entries (#3555): an entry or a hole planted in the record does not resume.
+      {
+        ...record,
+        changeSet: {
+          ...set.doc,
+          entries: [...set.doc.entries, { member: "a", lexicon: "terraform", planner: "tofu" as const, address: "aws_vpc.planted", type: "aws_vpc", action: "create" as const, attributes: [] }],
+        },
+      },
+      { ...record, changeSet: { ...set.doc, members: set.doc.members.map((m) => (m.member === "a" ? { ...m, holes: [{ address: "data.planted", reason: "unobserved" }] } : m)) } },
+    ]) {
+      w.planned.length = 0;
+      const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record: forged, completed: first.run.completed });
+      expect(resumed.ok).toBe(false);
+      if (resumed.ok) return;
+      expect(resumed.reasons[0]).toMatch(/^the record's change set is not the one its digest/);
+      expect(w.planned).toEqual([]);
+    }
+  });
+
+  test("with every component applied, nothing is left to plan or apply", async () => {
+    const w = world();
+    const plan = planFanOut({ components: ESTATE, changed: ["dns"] });
+    const { first, record } = await firstAttempt(w, plan, []);
+    expect(first.status).toBe("applied");
+    w.planned.length = 0;
+    const resumed = await resumePrSet({ components: ESTATE, plan, registry: w.registry, env: "prod", record, completed: first.run.completed });
+    if (!resumed.ok) throw new Error(resumed.reasons.join("; "));
+    expect(resumed.plan.order).toEqual([]);
+    expect(w.planned).toEqual([]);
+  });
+});
+
+describe("planCoveredBy", () => {
+  const entry = (address: string, action: ChangeSetEntry["action"], attributes: ChangeSetEntry["attributes"] = [], extra: Partial<ChangeSetEntry> = {}): ChangeSetEntry => ({
+    member: "net",
+    lexicon: "terraform",
+    planner: "tofu",
+    address,
+    type: address.split(".")[0],
+    action,
+    attributes,
+    ...extra,
+  });
+  const doc = (digest: string, entries: ChangeSetEntry[], extra: Partial<ChangeSetPart["member"]> = {}) =>
+    composeChangeSet([
+      {
+        member: { member: "net", lexicon: "terraform", planner: "tofu", status: "planned", planDigest: `jcs1-sha256:${digest.repeat(64)}`, holes: [], ...extra },
+        entries,
+      },
+    ]);
+
+  const approved = doc("1", [
+    entry("aws_vpc.main", "update", [{ path: "tags", before: { a: "1" }, after: { a: "2" } }]),
+    entry("aws_subnet.a", "create", [{ path: "vpc_id", unknown: true }, { path: "cidr_block", after: "10.0.1.0/24" }]),
+    entry("aws_instance.web", "replace", [{ path: "ami", before: "ami-1", after: "ami-2", forcesReplacement: true }]),
+    entry("aws_db_instance.db", "update", [{ path: "password", sensitive: true }]),
+  ]);
+
+  test("the same plan digest is covered", () => {
+    expect(planCoveredBy(approved, doc("1", []), "net")).toBeUndefined();
+  });
+
+  test("what is left of the approved changes is covered", () => {
+    // The VPC applied, the subnet's vpc_id is known now, and the instance's
+    // old object was destroyed before the create failed.
+    const left = doc("2", [
+      entry("aws_vpc.main", "no-op"),
+      entry("aws_subnet.a", "create", [{ path: "vpc_id", after: "vpc-123" }, { path: "cidr_block", after: "10.0.1.0/24" }]),
+      entry("aws_instance.web", "create", [{ path: "ami", after: "ami-2" }, { path: "instance_type", after: "t3.micro" }]),
+      entry("aws_db_instance.db", "update", [{ path: "password", sensitive: true }]),
+    ]);
+    expect(planCoveredBy(approved, left, "net")).toBeUndefined();
+  });
+
+  test("a create-before-destroy replace left with its deposed object to delete is covered", () => {
+    const left = doc("2", [entry("aws_instance.web", "delete", [], { deposed: "00abc" })]);
+    expect(planCoveredBy(approved, left, "net")).toBeUndefined();
+  });
+
+  test("a change the approved plan did not make is not covered", () => {
+    expect(planCoveredBy(approved, doc("2", [entry("aws_s3_bucket.logs", "create")]), "net")).toBe(
+      "net now plans to create aws_s3_bucket.logs, which the approved plan did not",
+    );
+    expect(planCoveredBy(approved, doc("2", [entry("aws_vpc.main", "delete")]), "net")).toBe(
+      "net now plans to delete aws_vpc.main, where the approved plan would update it",
+    );
+    expect(planCoveredBy(approved, doc("2", [entry("aws_vpc.main", "update", [{ path: "tags", before: { a: "1" }, after: { a: "3" } }])]), "net")).toBe(
+      "net now writes aws_vpc.main.tags to a different value than the approved plan",
+    );
+    expect(planCoveredBy(approved, doc("2", [entry("aws_vpc.main", "update", [{ path: "cidr_block", before: "a", after: "b" }])]), "net")).toBe(
+      "net now writes aws_vpc.main.cidr_block, which the approved plan did not",
+    );
+  });
+
+  test("a sensitive value is covered only where the approved plan wrote it as sensitive", () => {
+    expect(planCoveredBy(approved, doc("2", [entry("aws_vpc.main", "update", [{ path: "tags", sensitive: true }])]), "net")).toMatch(/as sensitive/);
+    expect(planCoveredBy(approved, doc("2", [entry("aws_db_instance.db", "update", [{ path: "password", after: "x" }])]), "net")).toMatch(/wrote as sensitive/);
+  });
+
+  test("a hole the approved plan did not have is not covered", () => {
+    const left = doc("2", [], { holes: [{ address: "data.aws_ami.latest", reason: "unknown" }] });
+    expect(planCoveredBy(approved, left, "net")).toMatch(/cannot read data.aws_ami.latest/);
+  });
+
+  test("a member the approved set did not have is not covered", () => {
+    const other = composeChangeSet([
+      { member: { member: "edge", lexicon: "terraform", planner: "tofu", status: "planned", planDigest: `jcs1-sha256:${"3".repeat(64)}`, holes: [] }, entries: [] },
+    ]);
+    expect(planCoveredBy(approved, other, "edge")).toBe("edge was not in the approved plan");
+  });
+});

package/src/components/pr-run.ts

@@ -18,9 +18,22 @@
  * it against the new value. A change that must carry a moved output through
  * its dependents in one rollout is what gated waves are for (#3049): each
  * wave is planned after the one before it applied.
+ *
+ * ## Finishing an apply that failed partway
+ *
+ * {@link resumePrSet} (#3464) is fan-out's resume (#2417, #3049) for a pull
+ * request. The attempt record (`./fan-out-record.ts`) keeps the change set
+ * the gate approved and the components that applied. A re-run on the same
+ * commit narrows the derivation to what is left (`remainingFanOut`), plans
+ * only that against the outputs the approved plans read, and checks each
+ * member it plans against the approved set. When every one is covered, the
+ * gate is decided against the approved digest, which still stands, and only
+ * the rest applies. Planning everything again would give the applied members
+ * empty plans and the set a new digest, and so ask for a second approval of
+ * work already approved.
  */
 
-import { composeChangeSet, type ChangeSetDocument, type ChangeSetPart } from "../change-set";
+import { composeChangeSet, verifyChangeSetDigest, type ChangeSetDocument, type ChangeSetEntry, type ChangeSetPart } from "../change-set";
 import { evaluateGate, type GateCheck, type GateLedgerPort } from "../op/gate";
 import {
   memberCounts,
@@ -32,10 +45,11 @@ import {
   type PrReport,
   type PrStatus,
 } from "../pr-loop";
-import { deployContext, type DriverComponent } from "./driver";
+import { deployContext, type DriverComponent, type DriverComponentResult } from "./driver";
 import { deployUnits } from "./deploy-units";
 import { runFanOut, type FanOutRunResult } from "./fan-out-run";
-import type { ComponentChangeSignal, FanOutPlan } from "./fan-out";
+import { remainingFanOut, type ComponentChangeSignal, type FanOutPlan } from "./fan-out";
+import type { PrApplyRecord } from "./fan-out-record";
 import type { CapabilityRegistry } from "./capability";
 import type { RunProgressEvent } from "./run-progress";
 import { planWaveComponent, readComponentOutputs } from "./wave-plan";
@@ -189,9 +203,17 @@ export interface PrApplyOptions {
   registry: CapabilityRegistry;
   env: string;
   vars?: Record<string, unknown>;
-  /** The set {@link planPrSet} planned and the gate approved. */
+  /** The set {@link planPrSet} planned and the gate approved, or {@link resumePrSet}'s set. */
   set: PrPlanSet;
   onProgress?: (event: RunProgressEvent) => void;
+  /**
+   * What an earlier attempt at this approved set applied (#3464): its
+   * components, which are reported applied and not run again, and the
+   * outputs they left.
+   */
+  resumed?: { completed: string[]; outputs: Record<string, Record<string, unknown>> };
+  /** Called as each component settles, so an attempt record survives a kill (`runFanOut`). */
+  onComponentSettled?: (result: DriverComponentResult, outputs: Record<string, unknown> | undefined) => void;
 }
 
 const sameJson = (a: unknown, b: unknown): boolean => JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
@@ -208,8 +230,19 @@ export async function applyPrSet(options: PrApplyOptions): Promise<{ members: Pr
     plans: options.set.plans,
     componentOutputs: { ...before },
     ...(options.onProgress ? { onProgress: options.onProgress } : {}),
+    ...(options.onComponentSettled ? { onComponentSettled: options.onComponentSettled } : {}),
   });
   const completed = new Set(run.completed);
+  const priorCompleted = new Set(options.resumed?.completed ?? []);
+  /**
+   * Whether a dependency's outputs moved from what the plans read: by this
+   * run's apply, or by the earlier attempt's when the record kept them.
+   */
+  const movedBy = (dep: string): boolean => {
+    if (completed.has(dep)) return !sameJson(before[dep], run.componentOutputs[dep]);
+    const earlier = options.resumed?.outputs;
+    return priorCompleted.has(dep) && earlier !== undefined && dep in earlier && !sameJson(before[dep], earlier[dep]);
+  };
   const blocked = new Set(run.blocked.map((b) => b.component));
   const errorOf = (component: string): string | undefined => {
     const result = run.results.find((r) => r.component === component);
@@ -218,9 +251,10 @@ export async function applyPrSet(options: PrApplyOptions): Promise<{ members: Pr
   };
   const byName = new Map(options.components.map((c) => [c.name, c]));
   const members = options.set.members.map((m): PrMember => {
+    if (priorCompleted.has(m.component)) return { ...m, status: "applied" };
     if (completed.has(m.component)) {
       const moved = (byName.get(m.component)?.dependsOn ?? [])
-        .filter((dep) => completed.has(dep) && !sameJson(before[dep], run.componentOutputs[dep]))
+        .filter(movedBy)
         .sort();
       return { ...m, status: "applied", ...(moved.length > 0 ? { inputsMoved: moved } : {}) };
     }
@@ -231,10 +265,165 @@ export async function applyPrSet(options: PrApplyOptions): Promise<{ members: Pr
   return { members, status: run.status === "ok" ? "applied" : "failed", run };
 }
 
+const entryKey = (e: Pick<ChangeSetEntry, "member" | "address" | "deposed">): string => `${e.member}\u0000${e.address}\u0000${e.deposed ?? ""}`;
+
+/**
+ * Why `member`'s fresh plan is not covered by the approved change set, or
+ * `undefined` when it is. A member left over from a partial apply is covered
+ * when it plans to the digest that was approved, or when every change it
+ * plans now is one the approved plan made:
+ *
+ * - an entry at the same address with the same action, or a `create` or a
+ *   `delete` where a `replace` was approved (the other half applied);
+ * - each attribute it writes was written by the approved entry, to the same
+ *   value or to one the approved plan knew only after apply. A sensitive
+ *   attribute carries no value to compare, so it is covered only where the
+ *   approved entry wrote that attribute as sensitive too. On a `create` left
+ *   from a `replace`, an attribute the replace did not change is not compared;
+ * - each side effect is one the approved plan ran.
+ *
+ * A member with holes the approved plan did not have is never covered by its
+ * entries, since its entries may not say everything it will do.
+ */
+export function planCoveredBy(approved: ChangeSetDocument, fresh: ChangeSetDocument, member: string): string | undefined {
+  const was = approved.members.find((m) => m.member === member);
+  const now = fresh.members.find((m) => m.member === member);
+  if (!now) return undefined;
+  if (!was) return `${member} was not in the approved plan`;
+  if (now.status === "failed" || now.planDigest === null) return `${member} failed to plan${now.error ? `: ${now.error}` : ""}`;
+  if (was.planDigest !== null && now.planDigest === was.planDigest) return undefined;
+  if (was.status === "failed") return `${member} had no approved plan`;
+  const knownHoles = new Set(was.holes.map((h) => h.address));
+  const newHole = now.holes.find((h) => !knownHoles.has(h.address));
+  if (newHole) return `${member} cannot read ${newHole.address} (${newHole.reason}), so its plan cannot be checked against the approved one`;
+
+  const approvedEntries = new Map(approved.entries.filter((e) => e.member === member).map((e) => [entryKey(e), e]));
+  for (const e of fresh.entries) {
+    if (e.member !== member || e.action === "no-op" || e.action === "read") continue;
+    // A create-before-destroy replace whose create applied leaves the old
+    // object deposed, and the plan now deletes it under a deposed key.
+    const a =
+      approvedEntries.get(entryKey(e)) ??
+      (e.action === "delete" && e.deposed ? approvedEntries.get(entryKey({ ...e, deposed: undefined })) : undefined);
+    if (!a) return `${member} now plans to ${e.action} ${e.address}, which the approved plan did not`;
+    const halfReplace = a.action === "replace" && (e.action === "create" || e.action === "delete");
+    if (a.action !== e.action && !halfReplace) {
+      return `${member} now plans to ${e.action} ${e.address}, where the approved plan would ${a.action} it`;
+    }
+    const approvedAttrs = new Map(a.attributes.map((attr) => [attr.path, attr]));
+    for (const attr of e.attributes) {
+      const at = `${e.address}.${attr.path}`;
+      const prior = approvedAttrs.get(attr.path);
+      if (!prior) {
+        if (halfReplace && e.action === "create") continue;
+        return `${member} now writes ${at}, which the approved plan did not`;
+      }
+      if (attr.sensitive) {
+        if (!prior.sensitive) return `${member} now writes ${at} as sensitive, so it cannot be compared with the approved value`;
+        continue;
+      }
+      if (prior.sensitive) return `${member} writes ${at}, which the approved plan wrote as sensitive, so it cannot be compared`;
+      if (prior.unknown) continue;
+      if (attr.unknown) return `${member} now writes ${at} with a value known only after apply, where the approved plan knew it`;
+      if (!sameJson(attr.after, prior.after)) return `${member} now writes ${at} to a different value than the approved plan`;
+    }
+  }
+  const approvedEffects = new Set(
+    (approved.sideEffects ?? []).filter((s) => s.member === member).map((s) => JSON.stringify([s.address, s.type, s.trigger ?? null, s.event ?? null])),
+  );
+  for (const s of fresh.sideEffects ?? []) {
+    if (s.member !== member) continue;
+    if (!approvedEffects.has(JSON.stringify([s.address, s.type, s.trigger ?? null, s.event ?? null]))) {
+      return `${member} now runs ${s.address}, which the approved plan did not`;
+    }
+  }
+  return undefined;
+}
+
+export interface PrResumeOptions {
+  components: DriverComponent[];
+  /** The fan-out the change derives now. The caller has checked it is the one the record was made from. */
+  plan: FanOutPlan;
+  registry: CapabilityRegistry;
+  env: string;
+  vars?: Record<string, unknown>;
+  seededOutputs?: Record<string, Record<string, unknown>>;
+  /** The approved set, from the attempt record. */
+  record: PrApplyRecord;
+  /** Components an earlier attempt applied. */
+  completed: string[];
+}
+
+export type PrResume =
+  | {
+      ok: true;
+      /** The derivation narrowed to what is left. */
+      plan: FanOutPlan;
+      /** The approved set, carrying fresh plans for the members left. Its digest is the approved one. */
+      set: PrPlanSet;
+      /** What the fresh plans of the members left came to. */
+      fresh: PrPlanSet;
+    }
+  | {
+      ok: false;
+      /** Why the approved set does not cover what is left, one line per member. */
+      reasons: string[];
+      fresh?: PrPlanSet;
+    };
+
+/**
+ * Plan what an apply that failed partway left (#3464): narrow the plan by
+ * the components that applied, plan the rest against the outputs the
+ * approved plans read, and check each member against the approved change
+ * set ({@link planCoveredBy}). Covered, the result is the approved set with
+ * the new plans in it, to decide the gate against `record.digest` and hand
+ * to {@link applyPrSet}. Not covered, it says why, and the caller plans
+ * everything and asks for a fresh approval.
+ *
+ * The record now travels between CI jobs through a cache (#3543), so its
+ * change set has to be the one its digest names: the document's own digest
+ * is `record.digest`, and the document gives that digest. The digest binds
+ * the entries, holes and side effects {@link planCoveredBy} reads (#3555),
+ * so a record cannot widen what a re-planned member may do.
+ */
+export async function resumePrSet(options: PrResumeOptions): Promise<PrResume> {
+  const { record } = options;
+  if (record.changeSet.digest !== record.digest || !verifyChangeSetDigest(record.changeSet)) {
+    return { ok: false, reasons: [`the record's change set is not the one its digest ${record.digest} names`] };
+  }
+  const plan = remainingFanOut(options.plan, options.components, { completed: options.completed });
+  const fresh = await planPrSet({
+    components: options.components,
+    plan,
+    registry: options.registry,
+    env: options.env,
+    ...(options.vars ? { vars: options.vars } : {}),
+    seededOutputs: { ...(options.seededOutputs ?? {}), ...options.record.planOutputs },
+  });
+  const reasons = fresh.doc.members
+    .map((m) => planCoveredBy(options.record.changeSet, fresh.doc, m.member))
+    .filter((r): r is string => r !== undefined);
+  if (reasons.length > 0) return { ok: false, reasons, fresh };
+  return {
+    ok: true,
+    plan,
+    fresh,
+    set: {
+      doc: options.record.changeSet,
+      members: options.record.members.map((m) => ({ ...m, status: "planned" as const })),
+      plans: fresh.plans,
+      outputs: options.record.planOutputs,
+      failed: false,
+    },
+  };
+}
+
 export interface PrReportInput {
   stage: "plan" | "apply";
   pr: number | null;
   env: string;
+  /** The workspace member whose pipeline runs the stage (#3465). */
+  member?: string;
   base: string;
   head: string;
   op: string;
@@ -247,6 +436,8 @@ export interface PrReportInput {
   members?: PrMember[];
   refusal?: PrRefusal;
   message?: string;
+  /** Components an earlier attempt at the approved set applied (#3464). */
+  resumed?: string[];
 }
 
 /** The report a stage writes. */
@@ -258,6 +449,7 @@ export function prReport(input: PrReportInput): PrReport {
     stage: input.stage,
     pr: input.pr,
     env: input.env,
+    ...(input.member ? { member: input.member } : {}),
     base: input.base,
     head: input.head,
     op: input.op,
@@ -266,6 +458,7 @@ export function prReport(input: PrReportInput): PrReport {
     status: input.status,
     ...(input.refusal ? { refusal: input.refusal } : {}),
     ...(input.message ? { message: input.message } : {}),
+    ...(input.resumed ? { resumed: { applied: [...input.resumed].sort() } } : {}),
     approval: input.approval,
     selection: {
       changed: input.plan.order.filter((c) => changed.has(c)),

package/src/gated-waves.test.ts

@@ -115,4 +115,15 @@ describe("wave records", () => {
     expect(line).toContain("nothing in it was applied");
     expect(line).toContain("chant approve fan-out release-wave-2 --plan");
   });
+
+  test("the approve hint on an unsealed gate has no --sign", () => {
+    const line = describeChangedWave(record);
+    expect(line).not.toContain("--sign");
+    expect(line.endsWith(`--plan ${record.digest}`)).toBe(true);
+  });
+
+  test("the approve hint on a sealed gate ends in --sign", () => {
+    const line = describeChangedWave(record, true);
+    expect(line.endsWith(`--plan ${record.digest} --sign`)).toBe(true);
+  });
 });

package/src/gated-waves.ts

@@ -230,11 +230,16 @@ export function withWaveRecord(records: readonly WaveRecord[], record: WaveRecor
 /**
  * The refusal for a wave whose set changed after it was approved: the digest
  * that was approved, the one planned now, and that nothing in the wave ran.
+ * On a sealed gate the approve hint ends in `--sign`.
  */
-export function describeChangedWave(record: Pick<WaveRecord, "wave" | "op" | "gate" | "digest" | "approved">): string {
+export function describeChangedWave(
+  record: Pick<WaveRecord, "wave" | "op" | "gate" | "digest" | "approved">,
+  /** True when the gate is sealed (`identity.gates`): an approval without `--sign` doesn't count, so the hint says to sign. */
+  sealed = false,
+): string {
   return (
     `wave ${record.wave} changed after it was approved, so nothing in it was applied. ` +
     `approved: ${record.approved ?? "(none)"}; planned now: ${record.digest}. ` +
-    `Read the new plan, then approve it: chant approve ${record.op} ${record.gate} --plan ${record.digest}`
+    `Read the new plan, then approve it: chant approve ${record.op} ${record.gate} --plan ${record.digest}${sealed ? " --sign" : ""}`
   );
 }