@caelo-cms/provisioning 0.10.55 → 0.10.58

package/src/state-store.ts

@@ -22,12 +22,24 @@
  */
 
 import { spawnSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
 import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { hostname } from "node:os";
 import { join } from "node:path";
 import { assertNoCheckpointSecrets, assertNoSecrets } from "./checkpoint-secrets.js";
 import { type GcloudResult, gcloudEnv } from "./gcloud.js";
 import { OPS_INSTALL_PREFIX } from "./state-backend.js";
+import {
+  describeExecutionArgs,
+  type ExecutionLockHolder,
+  judgeHeldLock,
+  type LocalHost,
+  type LockHolder,
+  lockBody,
+  PROCESS_LEASE_MS,
+  parseExecutionState,
+  parseLockBody,
+  renewedLockBody,
+} from "./state-lock-lease.js";
 
 /** The two documents a store holds. */
 export type StateFile = "install.json" | "progress.json";
@@ -176,58 +188,324 @@ export function bucketStateLockUrl(bucket: string): string {
   return `gs://${bucket}/${OPS_INSTALL_PREFIX}/state.lock`;
 }
 
-/** Who holds a lock, for the refusal message. */
-function lockBody(purpose: string): string {
-  return JSON.stringify({
-    purpose,
-    host: hostname(),
-    pid: process.pid,
-    startedAt: new Date().toISOString(),
-  });
+/** gcloud's wording for a failed `--if-generation-match` precondition. */
+const PRECONDITION_FAILED = /PreconditionFailed|412|At least one of the pre-conditions/i;
+
+/** How a bucket lock is taken: who holds it, and the clock to judge leases by. */
+export interface BucketLockOptions {
+  /**
+   * The Cloud Run execution taking the lock (#815): the lock's lease is the
+   * execution's task timeout, and a later run may break it once Cloud Run
+   * says this execution has finished. Without it the holder is this
+   * process (a CLI run): the lock names host + pid, carries a lease of
+   * {@link PROCESS_LEASE_MS} and is renewed by a heartbeat until released.
+   */
+  readonly holder?: ExecutionLockHolder;
+  /** Told when a dead holder's lock was broken, with why. */
+  readonly onBreak?: (message: string) => void;
+  /**
+   * Told when the heartbeat found the lock no longer this run's (another
+   * run broke it after the lease ran out, e.g. while this machine slept).
+   */
+  readonly onLost?: (message: string) => void;
+  readonly now?: () => Date;
+  /** Starts the heartbeat; returns its stop. Defaults to an unref'd setInterval. */
+  readonly schedule?: (tick: () => void, everyMs: number) => () => void;
+  /** This host and its pid check (state-lock-lease.ts LocalHost); tests pass a fake. */
+  readonly here?: LocalHost;
 }
 
+const defaultSchedule = (tick: () => void, everyMs: number): (() => void) => {
+  const timer = setInterval(tick, everyMs);
+  // The heartbeat must never keep a finished CLI alive.
+  timer.unref?.();
+  return () => clearInterval(timer);
+};
+
 /**
  * The install's state lock in its ops bucket, created only if absent
  * (`--if-generation-match=0`). Held by every writer of a bucket-backed
- * install's state — an installer run's `pulumi up`, `state migrate` — so
- * none writes a backend another is switching away from. A lock left by a
- * crashed run is reported with the command to remove it, never broken
- * automatically.
+ * install's state — an installer run's `pulumi up`, `state migrate`, the
+ * CLI `upgrade`, the upgrader job — so none writes a backend another is
+ * switching away from.
  *
- * @returns the release function
+ * A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
+ * execution has finished, whose execution is gone after its lease, whose
+ * CLI process is gone on this host, or whose CLI lease ran out unrenewed,
+ * is broken with a generation-matched delete and taken once more. The
+ * generation is read BEFORE the body, so the delete can only remove the
+ * lock that was judged: if another run replaced it in between, the
+ * delete's precondition fails and nothing is removed. Two runs breaking the
+ * same dead lock cannot both win either — the slower one's create fails
+ * its own precondition and reports the lock as held.
+ *
+ * A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
+ * with a generation-matched rewrite, so a run that is alive keeps its lock
+ * however long it takes, and a killed one loses it within one lease.
+ *
+ * @returns the release function; it stops the heartbeat and deletes only
+ *   this lock's generation
+ * @throws StateLockHeldError when another run holds the lock
+ * @throws Error when the lock or its holder cannot be read (no access)
  */
 export function acquireBucketStateLock(
   bucket: string,
   purpose: string,
   run: SyncGcloud = realSyncGcloud,
+  opts: BucketLockOptions = {},
 ): () => void {
   const url = bucketStateLockUrl(bucket);
-  const r = run(["storage", "cp", "-", url, "--if-generation-match=0"], lockBody(purpose));
-  if (!r.ok) {
-    if (/PreconditionFailed|412|At least one of the pre-conditions/i.test(r.stderr)) {
-      const held = run(["storage", "cat", url]);
+  const now = opts.now ?? (() => new Date());
+  const holder: LockHolder = opts.holder ?? { leaseMs: PROCESS_LEASE_MS };
+  const id = randomUUID();
+  const create = () =>
+    run(
+      ["storage", "cp", "-", url, "--if-generation-match=0"],
+      lockBody(purpose, now(), holder, id),
+    );
+  let r = create();
+  if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
+    breakDeadLock(url, run, now(), opts.onBreak, opts.here);
+    r = create();
+    if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
       throw new StateLockHeldError(
-        `Another run is changing this install's state (${held.ok ? held.stdout.trim() : url}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`,
+        `Another run took this install's state lock at the same moment (${readLockText(url, run)}). Re-run once it has finished.`,
       );
     }
-    throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
   }
+  if (!r.ok) throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
+  const owned = ownedGeneration(url, run, id);
+  if (owned.kind !== "owned") {
+    // Ours, just created — but without its generation a delete could hit a
+    // successor's lock. Give it back only once a second look proves it is
+    // still ours at a known generation; otherwise its lease ends it.
+    const retry = ownedGeneration(url, run, id);
+    if (retry.kind === "owned") {
+      run(["storage", "rm", url, `--if-generation-match=${retry.generation}`]);
+    }
+    throw new Error(
+      `Could not read the generation of the state lock ${url} this run just took (${owned.why}). ${
+        retry.kind === "owned"
+          ? "It was given back."
+          : "It was left in place and expires with its lease; re-run after that."
+      }`,
+    );
+  }
+  let generation = owned.generation;
+  let lost: string | null = null;
+  const stopHeartbeat =
+    "execution" in holder
+      ? () => {}
+      : (opts.schedule ?? defaultSchedule)(
+          () => {
+            if (lost !== null) return;
+            const renewed = renewLock(url, run, id, generation, now(), holder.leaseMs);
+            if (renewed.kind === "owned") {
+              generation = renewed.generation;
+              return;
+            }
+            if (renewed.kind === "lost") {
+              lost = renewed.why;
+              opts.onLost?.(
+                `This run no longer holds the install's state lock (${renewed.why}). Stop it and re-run once the other run has finished.`,
+              );
+            }
+            // "error": a transient failure; the next beat tries again within the lease.
+          },
+          Math.floor(holder.leaseMs / 3),
+        );
   return () => {
-    const rm = run(["storage", "rm", url]);
-    if (!rm.ok) throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
+    stopHeartbeat();
+    const rm = run(["storage", "rm", url, `--if-generation-match=${generation}`]);
+    if (rm.ok) return;
+    if (PRECONDITION_FAILED.test(rm.stderr) || NOT_FOUND.test(rm.stderr)) {
+      throw new Error(
+        `The state lock ${url} was no longer this run's when it finished (another run broke it); it was left alone.`,
+      );
+    }
+    throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
   };
 }
 
+type Ownership =
+  | { readonly kind: "owned"; readonly generation: string }
+  | { readonly kind: "lost"; readonly why: string }
+  | { readonly kind: "error"; readonly why: string };
+
+/**
+ * The lock's generation, provided the lock is still the one with `id`. The
+ * generation is read before the body: if another run replaced the lock in
+ * between, the body names its id and this returns "lost" — never a
+ * generation that is not ours.
+ */
+function ownedGeneration(url: string, run: SyncGcloud, id: string): Ownership {
+  const g = lockGeneration(url, run);
+  if (g.kind === "absent") return { kind: "lost", why: "the lock is gone" };
+  if (g.kind === "error") return { kind: "error", why: g.error };
+  const body = run(["storage", "cat", url]);
+  if (!body.ok) {
+    return NOT_FOUND.test(body.stderr)
+      ? { kind: "lost", why: "the lock is gone" }
+      : { kind: "error", why: body.stderr.trim() || `gcloud exited ${body.exitCode}` };
+  }
+  const parsed = parseLockBody(body.stdout);
+  if (parsed?.id !== id) {
+    return { kind: "lost", why: `another run holds it now (${body.stdout.trim().slice(0, 200)})` };
+  }
+  return { kind: "owned", generation: g.generation };
+}
+
+/**
+ * One heartbeat: rewrite the lock with a later lease, only if it is still
+ * at `generation` (ours), and return its new generation.
+ */
+function renewLock(
+  url: string,
+  run: SyncGcloud,
+  id: string,
+  generation: string,
+  now: Date,
+  leaseMs: number,
+): Ownership {
+  const current = run(["storage", "cat", url]);
+  if (!current.ok) {
+    return NOT_FOUND.test(current.stderr)
+      ? { kind: "lost", why: "the lock is gone" }
+      : { kind: "error", why: current.stderr.trim() };
+  }
+  const body = parseLockBody(current.stdout);
+  if (body?.id !== id) {
+    return {
+      kind: "lost",
+      why: `another run holds it now (${current.stdout.trim().slice(0, 200)})`,
+    };
+  }
+  const w = run(
+    ["storage", "cp", "-", url, `--if-generation-match=${generation}`],
+    renewedLockBody(body, now, leaseMs),
+  );
+  if (!w.ok) {
+    return PRECONDITION_FAILED.test(w.stderr)
+      ? { kind: "lost", why: "another run replaced it" }
+      : { kind: "error", why: w.stderr.trim() };
+  }
+  return ownedGeneration(url, run, id);
+}
+
+type LockGeneration =
+  | { readonly kind: "found"; readonly generation: string }
+  | { readonly kind: "absent" }
+  | { readonly kind: "error"; readonly error: string };
+
+/** The lock object's current generation. */
+function lockGeneration(url: string, run: SyncGcloud): LockGeneration {
+  const d = run(["storage", "objects", "describe", url, "--format=json"]);
+  if (!d.ok) {
+    return NOT_FOUND.test(d.stderr)
+      ? { kind: "absent" }
+      : { kind: "error", error: d.stderr.trim() || `gcloud exited ${d.exitCode}` };
+  }
+  try {
+    // `gcloud storage objects describe` prints the generation as a string
+    // (`generation: '1706304784726341'`, cloud.google.com/storage/docs/
+    // viewing-editing-metadata); a JSON number would lose digits past 2^53,
+    // so only a string of digits is a generation the delete may match.
+    const g = (JSON.parse(d.stdout) as { generation?: unknown }).generation;
+    if (typeof g === "string" && /^[1-9][0-9]*$/.test(g)) {
+      return { kind: "found", generation: g };
+    }
+  } catch {
+    // Reported below with the same message as a missing field.
+  }
+  return { kind: "error", error: "gcloud described the lock without a generation" };
+}
+
+function readLockText(url: string, run: SyncGcloud): string {
+  const held = run(["storage", "cat", url]);
+  return held.ok ? held.stdout.trim() : url;
+}
+
+/**
+ * Judge the held lock at `url` and break it when its holder is dead.
+ * Returns normally when the lock was broken or vanished meanwhile, so the
+ * caller creates it once more; throws when it is live or cannot be judged.
+ */
+function breakDeadLock(
+  url: string,
+  run: SyncGcloud,
+  now: Date,
+  onBreak: ((message: string) => void) | undefined,
+  here: LocalHost | undefined,
+): void {
+  const found = lockGeneration(url, run);
+  if (found.kind === "absent") return;
+  if (found.kind === "error") {
+    throw new Error(`Could not read the state lock ${url} another run holds: ${found.error}`);
+  }
+  const held = run(["storage", "cat", url]);
+  if (!held.ok) {
+    if (NOT_FOUND.test(held.stderr)) return;
+    throw new Error(`Could not read the state lock ${url}: ${held.stderr.trim()}`);
+  }
+  const verdict = judgeHeldLock(
+    held.stdout,
+    (ref) => parseExecutionState(run(describeExecutionArgs(ref))),
+    now,
+    here,
+  );
+  if (verdict.kind === "unknown") {
+    // Not contention: retrying will not help, and guessing "dead" could let
+    // two runs write the same state.
+    throw new Error(`The state lock ${url} is held and ${verdict.why}`);
+  }
+  if (verdict.kind === "held") {
+    throw new StateLockHeldError(
+      verdict.manualRemoval
+        ? `Another run is changing this install's state (${verdict.why}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`
+        : `Another run is changing this install's state: ${verdict.why}. Re-run once it has finished.`,
+    );
+  }
+  const rm = run(["storage", "rm", url, `--if-generation-match=${found.generation}`]);
+  if (!rm.ok && !NOT_FOUND.test(rm.stderr)) {
+    if (PRECONDITION_FAILED.test(rm.stderr)) {
+      throw new StateLockHeldError(
+        `Another run took this install's state lock while this one removed a stale one (${readLockText(url, run)}). Re-run once it has finished.`,
+      );
+    }
+    throw new Error(`Could not remove the stale state lock ${url}: ${rm.stderr.trim()}`);
+  }
+  onBreak?.(`Removed a stale state lock: ${verdict.why}.`);
+}
+
 /**
  * The same lock for an install whose state is on this machine:
- * `~/.caelo-<id>/state.lock`, created exclusively.
+ * `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
+ * process that is gone on this machine is taken over (#815).
  *
+ * @param here this host and its pid check; tests pass a fake
  * @returns the release function
  */
-export function acquireLocalStateLock(root: string, purpose: string): () => void {
+export function acquireLocalStateLock(root: string, purpose: string, here?: LocalHost): () => void {
   const path = join(root, "state.lock");
+  const take = () =>
+    writeFileSync(path, lockBody(purpose, new Date()), { flag: "wx", mode: 0o600 });
   try {
-    writeFileSync(path, lockBody(purpose), { flag: "wx", mode: 0o600 });
+    try {
+      take();
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+      // #815 — the file is on this machine, so a holder whose process has
+      // ended is known dead: take its lock over instead of asking for `rm`.
+      const verdict = judgeHeldLock(
+        readFileSync(path, "utf8"),
+        () => ({ kind: "unknown", error: "a machine-local lock names no Cloud Run execution" }),
+        new Date(),
+        here,
+      );
+      if (verdict.kind !== "break") throw e;
+      rmSync(path, { force: true });
+      take();
+    }
   } catch (e) {
     if ((e as NodeJS.ErrnoException).code === "EEXIST") {
       throw new StateLockHeldError(
@@ -245,8 +523,50 @@ export function acquireStateLock(
   root: string,
   purpose: string,
   run: SyncGcloud = realSyncGcloud,
+  opts: BucketLockOptions = {},
 ): () => void {
   return backend.kind === "gcs"
-    ? acquireBucketStateLock(backend.bucket, purpose, run)
-    : acquireLocalStateLock(root, purpose);
+    ? acquireBucketStateLock(backend.bucket, purpose, run, opts)
+    : acquireLocalStateLock(root, purpose, opts.here);
+}
+
+/** The part of `process` {@link releaseOnSignal} needs; tests pass a fake. */
+export interface SignalTarget {
+  once(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
+  removeListener(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
+  exit(code: number): never;
+}
+
+/**
+ * Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
+ * Shell tab's SIGTERM), then exit with the signal's conventional code
+ * (#815). Without this a stopped run leaves its lock until its lease ends.
+ *
+ * @param report told what happened to the lock
+ * @returns removes the handlers; call it once the run released the lock itself
+ */
+export function releaseOnSignal(
+  release: () => void,
+  report: (message: string) => void,
+  proc: SignalTarget = process,
+): () => void {
+  const dispose = () => {
+    proc.removeListener("SIGINT", onInt);
+    proc.removeListener("SIGTERM", onTerm);
+  };
+  const stop = (signal: "SIGINT" | "SIGTERM", code: number): never => {
+    dispose();
+    try {
+      release();
+      report(`Stopped (${signal}); the install's state lock was released.`);
+    } catch (e) {
+      report(`Stopped (${signal}); ${e instanceof Error ? e.message : String(e)}`);
+    }
+    return proc.exit(code);
+  };
+  const onInt = () => stop("SIGINT", 130);
+  const onTerm = () => stop("SIGTERM", 143);
+  proc.once("SIGINT", onInt);
+  proc.once("SIGTERM", onTerm);
+  return dispose;
 }

package/src/upgrade/app-roll.ts

@@ -27,7 +27,13 @@
 import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
 import { gatewayServiceAccountEmail, runServiceAccountEmail } from "../gcp-names.js";
 import type { ImageDigests } from "../install-state.js";
-import { adminMediaVolume } from "../stack-contract.js";
+import {
+  adminMediaVolume,
+  type CloudRunEnvVar,
+  GATEWAY_TIMEOUT_SECONDS,
+  RETIRED_SECRET_FILES,
+  SERVICE_SECRET_FILES,
+} from "../stack-contract.js";
 import {
   type DeployedService,
   type EnvChange,
@@ -35,10 +41,13 @@ import {
   type LiveVolumes,
   liveContainerEnv,
   liveContainerMemory,
+  liveTimeoutSeconds,
   liveVolumes,
   planAdminMemory,
   planContractEnv,
   planMediaVolume,
+  planRequestTimeout,
+  planSecretFiles,
   rollService,
   serviceRollArgs,
 } from "../stack-converge.js";
@@ -89,15 +98,25 @@ export interface ServicePlan {
   /** Container memory limit now (null = Cloud Run's default). */
   readonly liveMemory: string | null;
   readonly liveVolumes: LiveVolumes;
+  /** Request timeout now, in seconds (null = Cloud Run's default). */
+  readonly liveTimeout: number | null;
 }
 
 /** A service plan plus the changes its roll applies. */
 export interface RollPlan extends ServicePlan {
   readonly envFlags: readonly string[];
   readonly envChanges: readonly EnvChange[];
-  /** #553 — e.g. `--memory=2Gi` when the admin runs below the stack default. */
+  /**
+   * #553 — e.g. `--memory=2Gi` when the admin runs below the stack default;
+   * #806 — `--timeout=3600` when the gateway runs below its contract timeout.
+   */
   readonly resourceFlags: readonly string[];
-  /** The admin's media bucket volume when it is (partly) missing; empty otherwise. */
+  readonly resourceChanges: readonly string[];
+  /**
+   * The admin's media bucket volume when it is (partly) missing, the
+   * service's secret files (#806, the gateway's OAuth client) when not
+   * mounted; empty otherwise.
+   */
   readonly volumeFlags: readonly string[];
   readonly volumeChanges: readonly string[];
 }
@@ -164,9 +183,10 @@ export async function planAppRoll(
       liveEnv: liveContainerEnv(json),
       liveMemory: liveContainerMemory(json),
       liveVolumes: liveVolumes(json),
+      liveTimeout: liveTimeoutSeconds(json),
     });
   }
-  const env = planEnv(install, services, []);
+  const env = planEnv(install, services, {});
   if (!env.ok)
     return { ok: false, error: `the install's env contract can't be applied: ${env.error}` };
   // Without the media volume every upload is lost on the next revision or
@@ -178,28 +198,65 @@ export async function planAppRoll(
   );
   if (!media.ok)
     return { ok: false, error: `the admin's media volume can't be set up: ${media.error}` };
+  // #806/#914 — each service's secret files (the gateway's Google sign-in
+  // client). A file the roll cannot mount fails the plan: the new gateway
+  // reads its path from the env contract.
+  const secretFiles = {} as Record<RollSlug, { flags: string[]; changes: string[] }>;
+  for (const s of services) {
+    const files = planSecretFiles(
+      s.liveVolumes,
+      SERVICE_SECRET_FILES[s.slug],
+      install.env,
+      RETIRED_SECRET_FILES[s.slug],
+    );
+    if (!files.ok) {
+      return { ok: false, error: `the ${s.slug}'s secret files can't be mounted: ${files.error}` };
+    }
+    secretFiles[s.slug] = files;
+  }
   const warnings: string[] = [];
   const rolls = services.map((s): RollPlan => {
     // #553 — the admin runs the Lighthouse quality audit; raise it to the
     // stack's memory default (never lowered; an unparsable value is kept).
     const memory = s.slug === "admin" ? planAdminMemory(s.liveMemory) : null;
     if (memory && !memory.ok) warnings.push(`admin memory left as is: ${memory.error}`);
+    // #806 — the gateway streams remote-MCP chat turns (stack-contract.ts
+    // GATEWAY_TIMEOUT_SECONDS). The admin's timeout is the stacks' alone.
+    const timeout =
+      s.slug === "gateway" ? planRequestTimeout(s.liveTimeout, GATEWAY_TIMEOUT_SECONDS) : null;
+    const memoryFlags = memory?.ok ? memory.flags : [];
+    const timeoutFlags = timeout?.ok ? timeout.flags : [];
     return {
       ...s,
       envFlags: env.services[s.slug].flags,
       envChanges: env.services[s.slug].changes,
-      resourceFlags: memory?.ok ? memory.flags : [],
-      volumeFlags: s.slug === "admin" ? media.flags : [],
-      volumeChanges: s.slug === "admin" ? media.changes : [],
+      resourceFlags: [...memoryFlags, ...timeoutFlags],
+      resourceChanges: [
+        ...memoryFlags.map((f) => `${s.liveMemory ?? "(default)"} → ${f}`),
+        ...(timeout?.ok ? timeout.changes : []),
+      ],
+      volumeFlags: [...(s.slug === "admin" ? media.flags : []), ...secretFiles[s.slug].flags],
+      volumeChanges: [...(s.slug === "admin" ? media.changes : []), ...secretFiles[s.slug].changes],
     };
   });
   return { ok: true, plan: { install, services: rolls, warnings } };
 }
 
+/** Env the converge half adds to, or keeps out of, the rolls ({@link AppRollHooks.afterMigrations}). */
+export interface EnvAdjustments {
+  /** Contract vars to leave exactly as the services have them. */
+  readonly leaveUntouched?: readonly string[];
+  /**
+   * Vars outside the env contract the rolls set too — #806's remote-MCP
+   * identity env, which only the converge (operator identity) can resolve.
+   */
+  readonly extraEnv?: Readonly<Partial<Record<RollSlug, readonly CloudRunEnvVar[]>>>;
+}
+
 function planEnv(
   install: AppRollInstall,
   services: readonly ServicePlan[],
-  leaveUntouched: readonly string[],
+  adjust: EnvAdjustments,
 ) {
   const deployed = Object.fromEntries(
     services.map((s) => [s.slug, { serviceName: s.serviceName, liveEnv: s.liveEnv }]),
@@ -214,17 +271,22 @@ function planEnv(
       stagingHost: install.stagingHost,
     },
     deployed,
-    { leaveUntouched },
+    {
+      leaveUntouched: adjust.leaveUntouched ?? [],
+      ...(adjust.extraEnv ? { extraEnv: adjust.extraEnv } : {}),
+    },
   );
 }
 
 /**
- * The plan with `names` left exactly as the services have them — for env
- * vars that point at something a converge step could not set up.
+ * The plan with the converge's adjustments: vars left exactly as the
+ * services have them (they point at something a converge step could not set
+ * up) and vars outside the contract the rolls set too.
  */
-export function leavingEnvUntouched(plan: AppRollPlan, names: readonly string[]): AppRollPlan {
-  if (names.length === 0) return plan;
-  const env = planEnv(plan.install, plan.services, names);
+export function adjustingEnv(plan: AppRollPlan, adjust: EnvAdjustments): AppRollPlan {
+  const extra = Object.values(adjust.extraEnv ?? {}).some((v) => (v?.length ?? 0) > 0);
+  if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra) return plan;
+  const env = planEnv(plan.install, plan.services, adjust);
   if (!env.ok) return plan;
   return {
     ...plan,
@@ -240,10 +302,8 @@ export function leavingEnvUntouched(plan: AppRollPlan, names: readonly string[])
 export function reportPlan(plan: AppRollPlan, reporter: UpgradeReporter): void {
   for (const w of plan.warnings) reporter.warn(w);
   for (const s of plan.services) {
-    for (const c of s.volumeChanges) reporter.info(`${s.slug} media: ${c}`);
-    for (const f of s.resourceFlags) {
-      reporter.info(`${s.slug} resources: ${s.liveMemory ?? "(default)"} → ${f}`);
-    }
+    for (const c of s.volumeChanges) reporter.info(`${s.slug} volumes: ${c}`);
+    for (const c of s.resourceChanges) reporter.info(`${s.slug} resources: ${c}`);
     for (const c of s.envChanges) {
       reporter.info(
         `${s.slug} env ${c.name}: ${c.from === undefined ? "(unset)" : c.from} → ${c.to === undefined ? "(removed)" : c.to}`,
@@ -260,7 +320,11 @@ export interface AppRollHooks {
    * not be set up; `ok: false` aborts the upgrade (nothing rolled yet).
    */
   afterMigrations?(): Promise<
-    { readonly ok: true; readonly leaveUntouched: readonly string[] } | { readonly ok: false }
+    | ({ readonly ok: true; readonly leaveUntouched: readonly string[] } & Pick<
+        EnvAdjustments,
+        "extraEnv"
+      >)
+    | { readonly ok: false }
   >;
   /** Right before `slug` rolls; `ok: false` aborts and rolls back what rolled. */
   beforeRoll?(
@@ -309,6 +373,13 @@ export type AppRollResult =
       readonly phase: AppRollPhase;
       /** Whether migrations ran before the failure (the backup matters then). */
       readonly migrated: boolean;
+      /**
+       * Whether services had rolled and EVERY one of them was pinned back to
+       * the revision that served before — i.e. the previous release serves
+       * again. False when nothing had rolled, or when a pin failed (then a
+       * person must pin it; the reporter named which).
+       */
+      readonly rolledBack: boolean;
     };
 
 /** Run the app-roll. See the file comment for the sequence. */
@@ -320,10 +391,13 @@ export async function executeAppRoll(
   const { reporter: r, run } = deps;
   const { install } = input.plan;
   let plan = input.plan;
+  /** Set by {@link rollBack}: null until one ran with services to restore. */
+  let rollbackOk: boolean | null = null;
   const fail = (phase: AppRollPhase, migrated: boolean): AppRollResult => ({
     ok: false,
     phase,
     migrated,
+    rolledBack: rollbackOk === true,
   });
 
   // A probe that cannot reach the install now would only fail after the
@@ -363,7 +437,10 @@ export async function executeAppRoll(
   if (hooks.afterMigrations) {
     const converged = await hooks.afterMigrations();
     if (!converged.ok) return fail("converge", true);
-    plan = leavingEnvUntouched(plan, converged.leaveUntouched);
+    plan = adjustingEnv(plan, {
+      leaveUntouched: converged.leaveUntouched,
+      ...(converged.extraEnv ? { extraEnv: converged.extraEnv } : {}),
+    });
   }
 
   const rolled: RollPlan[] = [];
@@ -381,6 +458,7 @@ export async function executeAppRoll(
     const services = alsoCurrent ? [...rolled, alsoCurrent] : rolled;
     if (services.length === 0) return;
     r.warn(`Rolling back ${services.length} service(s) to the revisions that served before...`);
+    rollbackOk = true;
     for (const p of [...services].reverse()) {
       const ok = await pinTraffic(
         run,
@@ -390,7 +468,10 @@ export async function executeAppRoll(
         p.priorRevision,
       );
       if (ok) r.info(`  ${p.slug} traffic restored to ${p.priorRevision}`);
-      else r.error(`  ${p.slug} rollback FAILED — pin ${p.priorRevision} by hand`);
+      else {
+        rollbackOk = false;
+        r.error(`  ${p.slug} rollback FAILED — pin ${p.priorRevision} by hand`);
+      }
       await hooks.afterRollback?.(p.slug);
     }
   };