@caelo-cms/provisioning 0.10.46 → 0.10.48

package/src/upgrade/upgrade.ts

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * `cms-provision upgrade` for one gcp / gcp-firebase install: converge, then
+ * app-roll (epic #696). The order, and why:
+ *
+ *   1. plan (read-only): release digests, both services, the env contract
+ *   2. verify the images' signatures
+ *   3. pre-upgrade Cloud SQL backup — the first change; if it fails, the
+ *      install is exactly as it was
+ *   4. converge, the part before the roll (keyless provisioner, IAM, secrets,
+ *      MCP access)
+ *   5. app-roll: health baseline, migrations, the converge steps that must
+ *      follow migrations, admin + gateway rolls, deep health check, rollback
+ *      on failure, recorded digests
+ *   6. converge, the part after the roll; a complete converge records the
+ *      install's infrastructure contract version
+ *
+ * Every side effect goes through {@link GcpUpgradeDeps}, so the sequence runs
+ * against a fake gcloud in tests. lifecycle.ts wires the real ones.
+ */
+
+import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
+import type { ImageDigests } from "../install-state.js";
+import type { HttpFetch } from "../runtime-secrets.js";
+import {
+  type AppRollPlan,
+  executeAppRoll,
+  planAppRoll,
+  type ReleaseImage,
+  type RollSlug,
+  reportPlan,
+} from "./app-roll.js";
+import { backupFailureHelp, restoreCommand, takeSqlBackup } from "./backup.js";
+import { startConverge } from "./converge.js";
+import type { HealthProbe } from "./health.js";
+import type { UpgradeReporter } from "./reporter.js";
+
+export interface GcpUpgradeTarget {
+  readonly provider: "gcp" | "gcp-firebase";
+  readonly projectId: string;
+  readonly region: string;
+  /** Pulumi stack name (`production`). */
+  readonly env: string;
+  readonly domain: string;
+  readonly stagingHost: string;
+  readonly ownerEmail: string;
+  /** The tag rolled to: a version (`0.11.0`) or a channel tag (`latest`, `rc`). */
+  readonly targetTag: string;
+  /** The infrastructure contract version this CLI converges to (infra-contract.ts). */
+  readonly infraContractVersion: number;
+  readonly skipVerify: boolean;
+  readonly skipHealthCheck: boolean;
+}
+
+export interface GcpUpgradeDeps {
+  readonly run: GcloudRunner;
+  readonly reporter: UpgradeReporter;
+  readonly sleep?: Sleep;
+  /** Cloud SQL Admin API calls (database role passwords). */
+  readonly http?: HttpFetch;
+  readonly resolveImages: (
+    tag: string,
+  ) => Promise<
+    | { readonly ok: true; readonly images: Record<RollSlug, ReleaseImage> }
+    | { readonly ok: false; readonly error: string }
+  >;
+  readonly verifyImages: (images: readonly { slug: string; digest: string }[]) => Promise<boolean>;
+  readonly migrate: (
+    imageRef: string,
+  ) => Promise<{ readonly ok: boolean; readonly error?: string }>;
+  readonly healthProbe: (
+    plan: AppRollPlan,
+  ) => Promise<
+    | { readonly ok: true; readonly probe: HealthProbe }
+    | { readonly ok: false; readonly error: string }
+  >;
+  readonly recordDigests: (digests: ImageDigests) => Promise<void>;
+  /**
+   * The release's caelo-ops image (`…/ops@sha256:…`) for the upgrader job
+   * the converge sets up (#696), or null when the tag has none.
+   */
+  readonly resolveOpsImage: (tag: string) => Promise<string | null>;
+  readonly recordInfraContractVersion: (version: number) => void;
+  /** The operator-identity converge step (keyless provisioner); see ConvergeDeps. */
+  readonly provisionerIdentity?: () => Promise<boolean>;
+  /** Today, for the backup's description. */
+  readonly now?: () => Date;
+}
+
+/** Upgrade one install; true on success. See the file comment for the order. */
+export async function upgradeGcpInstall(
+  target: GcpUpgradeTarget,
+  deps: GcpUpgradeDeps,
+): Promise<boolean> {
+  const { reporter: r, run } = deps;
+  const { projectId } = target;
+
+  // 1. Plan. Nothing changes until every lookup succeeded.
+  const sPre = r.step("Pre-flight: resolving services + image digests + current revisions...");
+  const images = await deps.resolveImages(target.targetTag);
+  if (!images.ok) {
+    sPre.failed("Couldn't resolve the release images");
+    r.error(images.error);
+    return false;
+  }
+  const planned = await planAppRoll(
+    {
+      provider: target.provider,
+      projectId,
+      region: target.region,
+      env: target.env,
+      domain: target.domain,
+      stagingHost: target.stagingHost,
+    },
+    images.images,
+    run,
+  );
+  if (!planned.ok) {
+    sPre.failed("Pre-flight failed");
+    r.error(planned.error);
+    return false;
+  }
+  const plan = planned.plan;
+  sPre.done(`Pre-flight ok — ${plan.services.length} services planned`);
+  reportPlan(plan, r);
+
+  // 2. P21 ship 4 — signatures against the Caelo release workflow's
+  // identity: refuses a compromised registry, a typosquat or a fork's image.
+  if (!target.skipVerify) {
+    if (!(await deps.verifyImages(plan.services))) return false;
+  } else {
+    r.warn("--skip-verify set — image signatures NOT verified.");
+  }
+
+  // 3. The first change: a failure leaves the install exactly as it was.
+  const sBak = r.step("Taking a pre-upgrade Cloud SQL backup...");
+  const backup = await takeSqlBackup(
+    run,
+    { projectId, env: target.env },
+    `pre-upgrade to ${target.targetTag}`,
+    deps.now,
+  );
+  if (!backup.ok) {
+    sBak.failed(`Pre-upgrade backup failed: ${backup.error}`);
+    r.warn(backupFailureHelp({ projectId }));
+    return false;
+  }
+  sBak.done(`Pre-upgrade backup ${backup.backup.backupId} of ${backup.backup.instance} taken`);
+
+  // 4. Converge, the part before the roll. The upgrader job runs the ops
+  //    image with rights over the whole app, so it is deployed only when its
+  //    signature checks out (an unverified image is never set up).
+  const opsImageRef = await verifiedOpsImage(target, deps);
+  const converge = startConverge(plan, target.ownerEmail, {
+    opsImageRef,
+    run,
+    reporter: r,
+    ...(deps.sleep ? { sleep: deps.sleep } : {}),
+    ...(deps.http ? { http: deps.http } : {}),
+    ...(deps.provisionerIdentity ? { provisionerIdentity: deps.provisionerIdentity } : {}),
+  });
+  if (!(await converge.beforeAppRoll())) return false;
+
+  // 5. App-roll, with the converge steps that sit between its own.
+  let health: HealthProbe | null = null;
+  if (!target.skipHealthCheck) {
+    const probe = await deps.healthProbe(plan);
+    if (!probe.ok) {
+      r.error(probe.error);
+      r.warn("No migrations ran and no traffic moved.");
+      return false;
+    }
+    health = probe.probe;
+  }
+  const rolled = await executeAppRoll(
+    {
+      plan,
+      backup: backup.backup,
+      targetVersion: /^\d+\.\d+\.\d+/.test(target.targetTag) ? target.targetTag : null,
+      // Unknown until the signed release manifest names it (epic #696).
+      migrationsContract: null,
+    },
+    converge.hooks,
+    {
+      run,
+      reporter: r,
+      ...(deps.sleep ? { sleep: deps.sleep } : {}),
+      migrate: deps.migrate,
+      health,
+      recordDigests: deps.recordDigests,
+    },
+  );
+  if (!rolled.ok) return false;
+  r.success(`Upgrade to ${target.targetTag} complete (admin + gateway revisions serving).`);
+  r.info(
+    `Pre-upgrade backup: ${backup.backup.backupId} (restore: ${restoreCommand(backup.backup)})`,
+  );
+
+  // 6. Converge, the part after the roll.
+  await converge.afterAppRoll();
+  if (converge.complete()) {
+    try {
+      deps.recordInfraContractVersion(target.infraContractVersion);
+    } catch (e) {
+      r.warn(
+        `Could not record the infrastructure contract version (${e instanceof Error ? e.message : String(e)}); the next upgrade converges again.`,
+      );
+    }
+  } else {
+    r.warn(
+      "Some converge steps did not succeed (see above); the install is not marked as converged, so in-product updates stay unavailable until a re-run of upgrade completes them.",
+    );
+  }
+  return true;
+}
+
+/**
+ * The release's caelo-ops image for the upgrader job, signature-verified
+ * like the admin and gateway images; null (with the reason printed) when
+ * there is none, it does not verify, or verification was skipped.
+ */
+async function verifiedOpsImage(
+  target: GcpUpgradeTarget,
+  deps: GcpUpgradeDeps,
+): Promise<string | null> {
+  const r = deps.reporter;
+  const ref = await deps.resolveOpsImage(target.targetTag);
+  const digest = ref ? /@(sha256:[0-9a-f]{64})$/.exec(ref)?.[1] : undefined;
+  if (!ref || !digest) {
+    r.warn(
+      `The release ${target.targetTag} has no caelo-ops image in the registry, so the upgrader job (in-product updates) cannot be set up this time.`,
+    );
+    return null;
+  }
+  if (target.skipVerify) {
+    r.warn(
+      "--skip-verify set — the upgrader job is not set up or updated without a verified image.",
+    );
+    return null;
+  }
+  if (!(await deps.verifyImages([{ slug: "ops", digest }]))) {
+    r.warn(
+      "The caelo-ops image's signature did not verify, so the upgrader job is not set up or updated.",
+    );
+    return null;
+  }
+  return ref;
+}

package/src/upgrader-assess.test.ts

@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The upgrader's refusal matrix (upgrader-assess.ts), table-driven: every
+ * row is one way a request can be refused, handed to Cloud Shell, or let
+ * through.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { StoredUpdatePolicy } from "@caelo-cms/shared";
+import type { ReleaseManifest, ReleaseManifestResult } from "./release-manifest.js";
+import { assessUpgradeRequest, type InstalledState } from "./upgrader-assess.js";
+
+const D = (c: string) => `sha256:${c.repeat(64)}`;
+
+const MANIFEST: ReleaseManifest = {
+  manifestVersion: 1,
+  version: "0.11.0",
+  channel: "stable",
+  digests: { admin: D("a"), gateway: D("b"), ops: D("c") },
+  minFromVersion: "0.10.0",
+  infraContractVersion: 2,
+  migrations: { contract: false, lastContractVersion: null, head: "0267_platform_updates" },
+  security: false,
+  advisories: [],
+};
+
+const POLICY: StoredUpdatePolicy = {
+  mode: "auto",
+  autoSecurity: true,
+  autoPatch: true,
+  autoMinor: true,
+  channel: "stable",
+  windowStart: null,
+  windowEnd: null,
+  windowTimezone: "UTC",
+  soakHours: 24,
+  paused: false,
+  pausedReason: null,
+  pausedAt: null,
+  updatedAt: "2026-10-13T00:00:00Z",
+};
+
+const INSTALLED: InstalledState = { version: "0.10.45", infraContractVersion: 2 };
+const NOW = new Date("2026-10-13T12:00:00Z");
+
+const verified = (m: Partial<ReleaseManifest> = {}): ReleaseManifestResult => ({
+  ok: true,
+  manifest: { ...MANIFEST, ...m },
+});
+const broken = (
+  code: "registry" | "signature" | "schema" | "blob-digest" | "version-mismatch" | "artifact-shape",
+): ReleaseManifestResult => ({ ok: false, error: { code, message: `${code} problem` } });
+
+interface Row {
+  readonly name: string;
+  readonly manifest?: ReleaseManifestResult;
+  readonly installed?: InstalledState;
+  readonly trigger?: "manual" | "auto";
+  readonly policy?: StoredUpdatePolicy;
+  readonly target?: string;
+  /** "proceed" or `<status>/<code>`. */
+  readonly expect: string;
+  readonly reasonContains?: string;
+}
+
+const ROWS: readonly Row[] = [
+  { name: "a signed, newer, app-only release", expect: "proceed" },
+  {
+    name: "registry unreachable",
+    manifest: broken("registry"),
+    expect: "failed/registry",
+    reasonContains: "Nothing was changed",
+  },
+  { name: "bad signature", manifest: broken("signature"), expect: "refused/signature" },
+  { name: "tampered blob", manifest: broken("blob-digest"), expect: "refused/signature" },
+  { name: "replayed manifest", manifest: broken("version-mismatch"), expect: "refused/signature" },
+  { name: "not an artifact", manifest: broken("artifact-shape"), expect: "refused/signature" },
+  {
+    name: "manifest too new for this upgrader",
+    manifest: broken("schema"),
+    expect: "needs_cloud_shell/manifest-unreadable",
+  },
+  {
+    name: "installed version never recorded",
+    installed: { version: null, infraContractVersion: 2 },
+    expect: "needs_cloud_shell/install-version-unknown",
+  },
+  {
+    name: "same version",
+    installed: { version: "0.11.0", infraContractVersion: 2 },
+    expect: "refused/not-newer",
+  },
+  {
+    name: "downgrade",
+    installed: { version: "0.12.0", infraContractVersion: 2 },
+    expect: "refused/not-newer",
+    reasonContains: "never installs an older release",
+  },
+  {
+    name: "below minFromVersion",
+    manifest: verified({ minFromVersion: "0.10.50" }),
+    expect: "refused/min-from-version",
+    reasonContains: "Install v0.10.50 first",
+  },
+  {
+    name: "infrastructure contract raised",
+    installed: { version: "0.10.45", infraContractVersion: 1 },
+    expect: "needs_cloud_shell/infra-contract",
+    reasonContains: "Cloud Shell",
+  },
+  {
+    name: "never converged at all",
+    installed: { version: "0.10.45", infraContractVersion: 0 },
+    expect: "needs_cloud_shell/infra-contract",
+  },
+  {
+    name: "contract migrations, Owner's click",
+    manifest: verified({ migrations: { ...MANIFEST.migrations, contract: true } }),
+    expect: "proceed",
+  },
+  {
+    name: "contract migrations, automatic",
+    manifest: verified({ migrations: { ...MANIFEST.migrations, contract: true } }),
+    trigger: "auto",
+    expect: "refused/contract-migrations",
+  },
+  {
+    name: "skips a contract release, automatic",
+    manifest: verified({ migrations: { ...MANIFEST.migrations, lastContractVersion: "0.10.50" } }),
+    trigger: "auto",
+    expect: "refused/contract-migrations",
+  },
+  { name: "automatic, allowed by policy", trigger: "auto", expect: "proceed" },
+  {
+    name: "automatic, policy off",
+    trigger: "auto",
+    policy: { ...POLICY, mode: "notify" },
+    expect: "refused/policy",
+  },
+  {
+    name: "automatic, paused",
+    trigger: "auto",
+    policy: { ...POLICY, paused: true },
+    expect: "refused/policy",
+  },
+  {
+    name: "automatic, minor not enabled",
+    trigger: "auto",
+    policy: { ...POLICY, autoMinor: false },
+    expect: "refused/policy",
+  },
+  {
+    name: "automatic security fix, only security enabled",
+    trigger: "auto",
+    policy: { ...POLICY, autoMinor: false, autoPatch: false },
+    manifest: verified({ security: true, advisories: ["GHSA-abcd-efgh-ijkm"] }),
+    expect: "proceed",
+  },
+  {
+    name: "automatic, outside the window",
+    trigger: "auto",
+    policy: { ...POLICY, windowStart: "02:00", windowEnd: "03:00" },
+    expect: "refused/policy",
+  },
+  {
+    name: "automatic, release candidate",
+    trigger: "auto",
+    target: "0.11.0-rc.1",
+    manifest: verified({ version: "0.11.0-rc.1", channel: "rc" }),
+    expect: "refused/policy",
+  },
+];
+
+describe("assessUpgradeRequest — refusal matrix", () => {
+  it.each(ROWS.map((r) => [r.name, r] as const))("%s", (_name, row) => {
+    const d = assessUpgradeRequest({
+      request: { targetVersion: row.target ?? "0.11.0", trigger: row.trigger ?? "manual" },
+      manifest: row.manifest ?? verified(),
+      installed: row.installed ?? INSTALLED,
+      policy: row.policy ?? POLICY,
+      now: NOW,
+    });
+    const got = d.kind === "proceed" ? "proceed" : `${d.status}/${d.codes.join(",")}`;
+    expect(got).toBe(row.expect);
+    if (row.reasonContains) expect(d.kind === "stop" ? d.reason : "").toContain(row.reasonContains);
+  });
+
+  it("warns the Owner about destructive migrations it lets through", () => {
+    const d = assessUpgradeRequest({
+      request: { targetVersion: "0.11.0", trigger: "manual" },
+      manifest: verified({ migrations: { ...MANIFEST.migrations, contract: true } }),
+      installed: INSTALLED,
+      policy: POLICY,
+      now: NOW,
+    });
+    expect(d.kind === "proceed" ? d.warnings.join(" ") : "").toContain("backup must be restored");
+  });
+
+  it("keeps the verified manifest on a stop, for the facts the admin explains", () => {
+    const d = assessUpgradeRequest({
+      request: { targetVersion: "0.11.0", trigger: "manual" },
+      manifest: verified(),
+      installed: { version: "0.10.45", infraContractVersion: 1 },
+      policy: POLICY,
+      now: NOW,
+    });
+    expect(d.kind === "stop" && d.manifest?.infraContractVersion).toBe(2);
+  });
+});
+
+describe("assessUpgradeRequest — self-hosted (the host agent, #696 Phase 4c)", () => {
+  const SELF_HOSTED: InstalledState = { version: "0.10.45", infraContractVersion: null };
+  const assess = (row: {
+    installed?: InstalledState;
+    manifest?: ReleaseManifestResult;
+    trigger?: "manual" | "auto";
+  }) =>
+    assessUpgradeRequest({
+      request: { targetVersion: "0.11.0", trigger: row.trigger ?? "manual" },
+      manifest: row.manifest ?? verified(),
+      installed: row.installed ?? SELF_HOSTED,
+      policy: POLICY,
+      now: NOW,
+      manualPath: "server",
+    });
+
+  it("never sends a release to Cloud Shell for its infrastructure contract", () => {
+    // The target's own CLI renders the Compose file in the same run.
+    expect(assess({ manifest: verified({ infraContractVersion: 9 }) }).kind).toBe("proceed");
+  });
+
+  it("does not count the infrastructure contract as a reason to keep an automatic run manual", () => {
+    expect(assess({ manifest: verified({ infraContractVersion: 9 }), trigger: "auto" }).kind).toBe(
+      "proceed",
+    );
+  });
+
+  it("names the upgrade command on the server, not Cloud Shell, for a hand-run upgrade", () => {
+    const d = assess({ installed: { version: null, infraContractVersion: null } });
+    expect(d.kind === "stop" ? `${d.status}/${d.codes.join(",")}` : d.kind).toBe(
+      "needs_cloud_shell/install-version-unknown",
+    );
+    const reason = d.kind === "stop" ? d.reason : "";
+    expect(reason).toContain("upgrade command on the server");
+    expect(reason).not.toContain("Cloud Shell");
+  });
+
+  it("keeps every other refusal: downgrade, signature, contract migrations unattended", () => {
+    const code = (d: ReturnType<typeof assess>) =>
+      d.kind === "stop" ? `${d.status}/${d.codes.join(",")}` : d.kind;
+    expect(code(assess({ installed: { version: "0.12.0", infraContractVersion: null } }))).toBe(
+      "refused/not-newer",
+    );
+    expect(code(assess({ manifest: broken("signature") }))).toBe("refused/signature");
+    expect(
+      code(
+        assess({
+          manifest: verified({ migrations: { ...MANIFEST.migrations, contract: true } }),
+          trigger: "auto",
+        }),
+      ),
+    ).toBe("refused/contract-migrations");
+  });
+});