@caelo-cms/provisioning 0.10.53 → 0.10.57

package/src/upgrader-assess.test.ts

@@ -30,6 +30,7 @@ const POLICY: StoredUpdatePolicy = {
   autoSecurity: true,
   autoPatch: true,
   autoMinor: true,
+  autoConverge: false,
   channel: "stable",
   windowStart: null,
   windowEnd: null,
@@ -53,7 +54,11 @@ const verified = (m: Partial<ReleaseManifest> = {}): ReleaseManifestResult => ({
 });
 const broken = (
   code: "registry" | "signature" | "schema" | "blob-digest" | "version-mismatch" | "artifact-shape",
-): ReleaseManifestResult => ({ ok: false, error: { code, message: `${code} problem` } });
+  transient?: true,
+): ReleaseManifestResult => ({
+  ok: false,
+  error: { code, message: `${code} problem`, ...(transient ? { transient } : {}) },
+});
 
 interface Row {
   readonly name: string;
@@ -72,9 +77,16 @@ const ROWS: readonly Row[] = [
   { name: "a signed, newer, app-only release", expect: "proceed" },
   {
     name: "registry unreachable",
-    manifest: broken("registry"),
+    manifest: broken("registry", true),
     expect: "failed/registry",
-    reasonContains: "Nothing was changed",
+    reasonContains: "try again later",
+  },
+  {
+    // Regression #865: a 404 never heals, so it must not end retryable.
+    name: "no signed manifest for the target (404)",
+    manifest: broken("registry"),
+    expect: "refused/manifest-missing",
+    reasonContains: "has no signed release manifest",
   },
   { name: "bad signature", manifest: broken("signature"), expect: "refused/signature" },
   { name: "tampered blob", manifest: broken("blob-digest"), expect: "refused/signature" },
@@ -94,6 +106,7 @@ const ROWS: readonly Row[] = [
     name: "same version",
     installed: { version: "0.11.0", infraContractVersion: 2 },
     expect: "refused/not-newer",
+    reasonContains: "v0.11.0 already serves on this install",
   },
   {
     name: "downgrade",
@@ -326,3 +339,80 @@ describe("assessUpgradeRequest — self-hosted (the host agent, #696 Phase 4c)",
     ).toBe("refused/contract-migrations");
   });
 });
+
+describe("assessUpgradeRequest — the converge route (#819)", () => {
+  const assess = (row: {
+    manifest?: ReleaseManifestResult;
+    installed?: InstalledState;
+    trigger?: "manual" | "auto";
+    policy?: Partial<StoredUpdatePolicy>;
+  }) =>
+    assessUpgradeRequest({
+      request: { targetVersion: "0.11.0", trigger: row.trigger ?? "manual" },
+      manifest: row.manifest ?? verified({ infraContractVersion: 3 }),
+      installed: row.installed ?? INSTALLED,
+      policy: { ...POLICY, ...row.policy },
+      now: NOW,
+      route: "converge",
+    });
+  const code = (d: ReturnType<typeof assess>) =>
+    d.kind === "stop" ? `${d.status}/${d.codes.join(",")}` : d.kind;
+
+  it("installs a release that raises the infrastructure contract: that is its job", () => {
+    expect(code(assess({}))).toBe("proceed");
+  });
+
+  it("the contract bump is no reason to warn about destructive migrations", () => {
+    const d = assess({});
+    expect(d.kind === "proceed" ? d.warnings : ["stopped"]).toEqual([]);
+  });
+
+  it("converges automatically only with the Owner's autoConverge (#864), default off", () => {
+    expect(code(assess({ trigger: "auto" }))).toBe("refused/policy");
+    const off = assess({ trigger: "auto" });
+    expect(off.kind === "stop" ? off.reason : "").toContain("Settings → Updates");
+    expect(code(assess({ trigger: "auto", policy: { autoConverge: true } }))).toBe("proceed");
+  });
+
+  it("autoConverge never widens the rest of the policy: kinds, mode and pause still decide", () => {
+    const on = { autoConverge: true };
+    // 0.10.45 → 0.11.0 is a minor release.
+    expect(code(assess({ trigger: "auto", policy: { ...on, autoMinor: false } }))).toBe(
+      "refused/policy",
+    );
+    expect(code(assess({ trigger: "auto", policy: { ...on, mode: "notify" } }))).toBe(
+      "refused/policy",
+    );
+    expect(code(assess({ trigger: "auto", policy: { ...on, paused: true } }))).toBe(
+      "refused/policy",
+    );
+    // The Owner's click needs no autoConverge.
+    expect(code(assess({ trigger: "manual" }))).toBe("proceed");
+  });
+
+  it("keeps every other refusal: signature, downgrade, minFromVersion, unknown version", () => {
+    expect(code(assess({ manifest: broken("signature") }))).toBe("refused/signature");
+    expect(code(assess({ manifest: broken("blob-digest") }))).toBe("refused/signature");
+    expect(code(assess({ installed: { version: "0.11.0", infraContractVersion: 2 } }))).toBe(
+      "refused/not-newer",
+    );
+    expect(
+      code(assess({ manifest: verified({ infraContractVersion: 3, minFromVersion: "0.10.50" }) })),
+    ).toBe("refused/min-from-version");
+    expect(code(assess({ installed: { version: null, infraContractVersion: 2 } }))).toBe(
+      "needs_cloud_shell/install-version-unknown",
+    );
+  });
+
+  it("the upgrader's own route still hands the contract bump on, naming the converge job first", () => {
+    const d = assessUpgradeRequest({
+      request: { targetVersion: "0.11.0", trigger: "manual" },
+      manifest: verified({ infraContractVersion: 3 }),
+      installed: INSTALLED,
+      policy: POLICY,
+      now: NOW,
+    });
+    expect(code(d)).toBe("needs_cloud_shell/infra-contract");
+    expect(d.kind === "stop" ? d.reason : "").toContain("converge job");
+  });
+});

package/src/upgrader-assess.ts

@@ -27,6 +27,8 @@
  * | target not newer than installed (downgrade/same)    | refused           |
  * | installed below the release's minFromVersion        | refused           |
  * | release raises the infrastructure contract          | needs_cloud_shell |
+ * |   … on the converge route (#819, the converge job)  | proceed           |
+ * |   … converge route, automatic, autoConverge off     | refused           |
  * | contract (destructive) migrations, automatic run    | refused           |
  * | contract migrations, Owner's click                  | proceed + warning |
  * | automatic run the update policy does not allow      | refused           |
@@ -38,6 +40,13 @@
  * that rolls the images. So for them `installed.infraContractVersion` is
  * null and the infra-contract row never applies; every "needs_cloud_shell"
  * stop names the upgrade command on the server instead of Cloud Shell.
+ *
+ * The converge job (#819, converge-job.ts) asks with `route: "converge"`:
+ * it runs as the install's provisioner, so an infrastructure change is what
+ * it is for — every other row still applies. An automatic run on the
+ * converge route needs the Owner's `autoConverge` (#864) on top of
+ * everything an automatic install needs; the scheduler queues one only
+ * then, and the job re-checks it here against the policy it claimed.
  */
 
 import {
@@ -60,6 +69,7 @@ import { needsConverge } from "./upgrade/converge.js";
 /** Machine-readable reason codes (`platform_update_requests.refusal_codes`). */
 export type RefusalCode =
   | "registry"
+  | "manifest-missing"
   | "signature"
   | "manifest-unreadable"
   | "install-version-unknown"
@@ -134,6 +144,11 @@ export function assessUpgradeRequest(input: {
   readonly now: Date;
   /** Default `cloud-shell` (the GCP upgrader job). */
   readonly manualPath?: ManualUpgradePath;
+  /**
+   * Which job asks (#819). `converge` = the converge job: an infrastructure
+   * contract bump is installed, not handed to Cloud Shell. Default `app_roll`.
+   */
+  readonly route?: "app_roll" | "converge";
 }): RequestDecision {
   const { request, installed } = input;
   const by = manualRun(input.manualPath ?? "cloud-shell");
@@ -142,10 +157,20 @@ export function assessUpgradeRequest(input: {
     const e = input.manifest.error;
     switch (e.code) {
       case "registry":
+        // Only an outage is worth retrying: a manifest the registry says
+        // does not exist (404 / other 4xx) never appears, so retrying would
+        // loop on "try again later" forever.
+        if (e.transient) {
+          return stop(
+            "failed",
+            "registry",
+            `Could not reach the Caelo release registry to verify v${target} (${e.message}). Nothing was changed; try again later.`,
+          );
+        }
         return stop(
-          "failed",
-          "registry",
-          `Could not reach the Caelo release registry to verify v${target} (${e.message}). Nothing was changed; try again later.`,
+          "refused",
+          "manifest-missing",
+          `v${target} has no signed release manifest in the Caelo release registry, so it cannot be verified and will not be installed. Nothing was changed; retrying will not help. (${e.message})`,
         );
       case "schema":
         return stop(
@@ -170,7 +195,18 @@ export function assessUpgradeRequest(input: {
       manifest,
     );
   }
-  if (compareSemver(target, installed.version) <= 0) {
+  const order = compareSemver(target, installed.version);
+  if (order === 0) {
+    // Reached after a run that rolled but could not record it (#860): the
+    // next run re-records the serving release, which then is the target.
+    return stop(
+      "refused",
+      "not-newer",
+      `v${target} already serves on this install, so there is nothing to install. Nothing was changed.`,
+      manifest,
+    );
+  }
+  if (order < 0) {
     return stop(
       "refused",
       "not-newer",
@@ -188,20 +224,33 @@ export function assessUpgradeRequest(input: {
   }
   // Self-hosted: the target's own CLI renders the host setup in this run.
   const infraContractVersion = installed.infraContractVersion ?? manifest.infraContractVersion;
-  if (
+  const converge = input.route === "converge";
+  const infraChange =
     installed.infraContractVersion !== null &&
-    needsConverge({ infraContractVersion }, manifest.infraContractVersion)
-  ) {
+    needsConverge({ infraContractVersion }, manifest.infraContractVersion);
+  if (infraChange && !converge) {
     return stop(
       "needs_cloud_shell",
       "infra-contract",
-      `v${target} changes this install's cloud setup (infrastructure contract ${infraContractVersion} → ${manifest.infraContractVersion}: ${infraStepsText(infraContractVersion, manifest.infraContractVersion)}). The admin may not change cloud permissions, so this release is installed once with your own Google account in Cloud Shell.`,
+      `v${target} changes this install's cloud setup (infrastructure contract ${infraContractVersion} → ${manifest.infraContractVersion}: ${infraStepsText(infraContractVersion, manifest.infraContractVersion)}). The upgrader may not change cloud permissions, so Caelo installs this release with its converge job: start it again from Settings → Updates. On an install without the converge job, install it once with your own Google account in Cloud Shell.`,
+      manifest,
+    );
+  }
+  if (converge && request.trigger === "auto" && !input.policy.autoConverge) {
+    return stop(
+      "refused",
+      "policy",
+      `Not installed automatically: v${target} changes this install's cloud setup, and the update policy does not install those automatically. Install it from Settings → Updates.`,
       manifest,
     );
   }
+  // On the converge route the contract bump is the job's work, not a
+  // reason for a manual install: only the migrations decide below.
   const assessment = assessAutoInstall(manifest, {
     version: installed.version,
-    infraContractVersion,
+    infraContractVersion: converge
+      ? Math.max(infraContractVersion, manifest.infraContractVersion)
+      : infraContractVersion,
   });
   const destructive = assessment.mode === "manual";
   const warnings: string[] = [];

package/src/upgrader-gcp.ts

@@ -3,8 +3,8 @@
 /**
  * The real side effects of the upgrader job on Google Cloud (epic #696):
  * gcloud as the job's own service account, the install's ops bucket for
- * install.json, the fixed release registry, the migration job and the IAP
- * health probe. upgrader-job.ts takes them as {@link UpgraderJobDeps}; the
+ * install.json, the fixed release registry, the migration job, the IAP
+ * health probe and the IAM self-check (#818). upgrader-job.ts takes them as {@link UpgraderJobDeps}; the
  * database queue and the job log come from `@caelo-cms/platform-updater`,
  * which owns the Query API connection.
  *
@@ -24,12 +24,14 @@ import { readAdminConfig, runMigrationsViaCloudRunJob } from "./migration-runner
 import { fetchVerifiedReleaseManifest } from "./release-manifest.js";
 import type { HttpFetch } from "./runtime-secrets.js";
 import { opsBucketName } from "./state-backend.js";
-import { acquireBucketStateLock, GcsStateStore } from "./state-store.js";
+import { acquireBucketStateLock, GcsStateStore, realSyncGcloud } from "./state-store.js";
 import type { AppRollPlan } from "./upgrade/app-roll.js";
 import { findServiceUrl } from "./upgrade/gcp-lookup.js";
 import { createIapHealthProbe } from "./upgrade/health.js";
+import { restIamTester } from "./upgrader-iam-preflight.js";
 import type { UpdateQueue, UpgraderJobDeps } from "./upgrader-job.js";
 import type { JobLog } from "./upgrader-reporter.js";
+import { UPGRADER_TASK_TIMEOUT_MINUTES } from "./upgrader-setup.js";
 
 /** Where the job runs, from the metadata server. */
 export interface UpgraderPlacement {
@@ -55,8 +57,11 @@ export async function upgraderDatabaseHost(
   return cfg.databaseHost;
 }
 
-/** The IAP-backed deep health probe for `plan`'s install. */
-async function healthProbeFor(run: GcloudRunner, http: HttpFetch, plan: AppRollPlan) {
+/**
+ * The IAP-backed deep health probe for `plan`'s install (the upgrader job;
+ * the converge job, #819, too).
+ */
+export async function healthProbeFor(run: GcloudRunner, http: HttpFetch, plan: AppRollPlan) {
   const { install } = plan;
   const urls: Partial<Record<"admin" | "gateway", string>> = {};
   for (const s of plan.services) {
@@ -102,12 +107,46 @@ export function gcpUpgraderDeps(
     region: placement.region,
     env: GCP_STACK_ENV,
     execution: placement.execution,
+    // #818 — the IAM self-check asks as the job's own account (gcloud's
+    // token from the metadata server), so it sees exactly the job's grants.
+    iam: restIamTester({
+      http,
+      token: async () => {
+        const t = await run(["auth", "print-access-token"]);
+        if (!t.ok) throw new Error(`gcloud auth print-access-token: ${t.stderr.trim()}`);
+        return t.stdout.trim();
+      },
+    }),
     // The default deps: the fixed registry and signer, never overridden.
     fetchManifest: (version) => fetchVerifiedReleaseManifest(version),
-    lockState: () =>
-      acquireBucketStateLock(opsBucketName(placement.projectId), "caelo upgrader job"),
+    // #815 — the lock names this execution, so a later run can break it
+    // once Cloud Run says this one has ended (killed at its task timeout,
+    // OOM). Outside Cloud Run there is no execution to vouch for it.
+    lockState: (onBreak) =>
+      acquireBucketStateLock(
+        opsBucketName(placement.projectId),
+        "caelo upgrader job",
+        realSyncGcloud,
+        {
+          onBreak,
+          ...(placement.execution
+            ? {
+                holder: {
+                  execution: {
+                    projectId: placement.projectId,
+                    region: placement.region,
+                    name: placement.execution,
+                  },
+                  maxRuntimeMs: UPGRADER_TASK_TIMEOUT_MINUTES * 60_000,
+                },
+              }
+            : {}),
+        },
+      ),
+    // Uncached on purpose: the job re-reads install.json once it holds the
+    // lock to see whether another writer changed it meanwhile (#815).
     readInstall: () => {
-      const text = store.read("install.json");
+      const text = store.peek("install.json");
       return text === null ? null : (JSON.parse(text) as InstallMetadata);
     },
     writeInstall: (meta) => store.write("install.json", meta),