@caelo-cms/provisioning 0.10.53 → 0.10.57

package/src/compose.ts

@@ -400,6 +400,11 @@ ${serviceRuntime(spec.images, "gateway", "bun run apps/api-gateway/src/server.ts
       GATEWAY_PORT: "${COMPOSE_GATEWAY_PORT}"
       # Caddy is the one proxy in front (client-ip.ts in the gateway).
       ${TRUSTED_PROXY_HOPS_ENV}: "${GATEWAY_TRUSTED_PROXY_HOPS["self-hosted"]}"
+      # #806 — no remote MCP here: it rides Google IAP, which this install
+      # does not have. CAELO_ADMIN_URL / CAELO_MCP_PUBLIC_URL /
+      # CAELO_MCP_OAUTH_CLIENT_FILE stay unset, so the gateway's /api/mcp/*
+      # answers 503 remote_mcp_not_configured and Settings → MCP says
+      # "not available on this install".
       NODE_ENV: production`;
 }
 

package/src/converge-checkpoint.ts

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The converge job's checkpoint (#819): `converge/converge-job.json` in the
+ * install's ops bucket. NOT under `install/` next to install.json: the
+ * upgrader's service account may write `install/` (its object grant is
+ * conditioned on that prefix, upgrader-setup.ts), and this file steers a
+ * run as the provisioner — its backup and rollback targets would be
+ * tamperable from there (#883 review). Only the provisioner and the
+ * operator, who hold the whole bucket, can write `converge/`. It has the
+ * install job's status shape
+ * (install-job.ts {@link InstallJobStatusSchema}: which execution runs, its
+ * phase, whether it finished) plus what a killed run's successor must
+ * reuse instead of recomputing:
+ *
+ *   - `targetVersion` / `fromVersion`: the update it belongs to. A run for
+ *     another target starts fresh; a run for the same target resumes;
+ *   - `backup`: the pre-upgrade backup (a new one could hold the
+ *     half-migrated schema);
+ *   - `priorRevisions`: the revisions that served before the update (the
+ *     rollback targets; the serving one may already be the new release's).
+ *
+ * Whether a successor starts, attaches or is done is {@link decideInstallJob}'s
+ * call, as for the install job. No secret ever lands here (assertNoSecrets).
+ */
+
+import { z } from "zod";
+import { InstallJobStatusSchema, readJobStatusDoc, writeJobStatusDoc } from "./install-job.js";
+import type { SyncGcloud } from "./state-store.js";
+
+const VERSION = /^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$/;
+
+export const ConvergeCheckpointSchema = InstallJobStatusSchema.extend({
+  /** The release this converge installs. */
+  targetVersion: z.string().regex(VERSION),
+  /** install.json's release when the update first started (the version it is judged from). */
+  fromVersion: z.string().regex(VERSION),
+  /** The update request it serves (the last one that ran it). */
+  requestId: z.string().uuid(),
+  backup: z
+    .object({
+      projectId: z.string().min(1),
+      instance: z.string().min(1),
+      backupId: z.string().min(1),
+    })
+    .optional(),
+  priorRevisions: z.object({ admin: z.string().min(1), gateway: z.string().min(1) }).optional(),
+});
+/**
+ * The ops-bucket prefix of the converge job's own files — outside the
+ * upgrader's `install/` grant on purpose (see the file comment).
+ */
+export const CONVERGE_STATE_PREFIX = "converge";
+
+/** What `converge/converge-job.json` records. */
+export type ConvergeCheckpoint = z.infer<typeof ConvergeCheckpointSchema>;
+
+/** `gs://<bucket>/converge/converge-job.json`. */
+export function convergeCheckpointUrl(bucket: string): string {
+  return `gs://${bucket}/${CONVERGE_STATE_PREFIX}/converge-job.json`;
+}
+
+/** The checkpoint as the bucket holds it now (never cached), or null. */
+export function readConvergeCheckpoint(bucket: string, run: SyncGcloud): ConvergeCheckpoint | null {
+  return readJobStatusDoc(convergeCheckpointUrl(bucket), ConvergeCheckpointSchema, run);
+}
+
+/** Replace the checkpoint. */
+export function writeConvergeCheckpoint(
+  bucket: string,
+  checkpoint: ConvergeCheckpoint,
+  run: SyncGcloud,
+): void {
+  writeJobStatusDoc(convergeCheckpointUrl(bucket), ConvergeCheckpointSchema, checkpoint, run);
+}

package/src/converge-entrypoint.test.ts

@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #883 review — the converge job runs as caelo-provisioner, so an
+ * execution override must be useless even if someone held
+ * run.jobs.runWithOverrides:
+ *
+ *   - the job's entry point (ops/caelo-converge-job) refuses every argument
+ *     but `converge-job` before any code runs;
+ *   - it starts the job with an empty environment plus a fixed allowlist,
+ *     so CLOUDSDK_*, PULUMI_*, BUN_*, LD_* overrides never reach gcloud,
+ *     Pulumi or Bun;
+ *   - and nobody is granted a role that carries runWithOverrides: not the
+ *     upgrader (anywhere), not the admin (on the converge or upgrader job).
+ *
+ * The script runs here under /bin/sh with its final `bun …` target swapped
+ * for `env`, so the test sees exactly the environment the job would get.
+ */
+
+import { describe, expect, it } from "bun:test";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { CONVERGE_ADMIN_ROLES, convergeJobDeployArgs } from "./converge-job-setup.js";
+import type { GcloudResult } from "./gcloud.js";
+import {
+  ensureUpgraderJob,
+  RUN_WITH_OVERRIDES_ROLES,
+  UPGRADER_FORBIDDEN_ROLES,
+  UPGRADER_PROJECT_ROLES,
+  UPGRADER_RUN_ROLE,
+} from "./upgrader-setup.js";
+
+const SCRIPT = resolve(import.meta.dir, "..", "ops", "caelo-converge-job");
+const TARGET_LINE = "bun /opt/caelo-ops/packages/platform-updater/src/converge-main.ts";
+
+/** The entry point with its final target replaced by `env` (prints the environment). */
+function scriptPrintingEnv(dir: string): string {
+  const text = readFileSync(SCRIPT, "utf8");
+  expect(text.split(TARGET_LINE).length).toBe(2);
+  const path = join(dir, "caelo-converge-job");
+  writeFileSync(path, text.replace(TARGET_LINE, "env"));
+  chmodSync(path, 0o755);
+  return path;
+}
+
+function runScript(path: string, args: string[], env: Record<string, string>) {
+  const r = Bun.spawnSync(["/bin/sh", path, ...args], {
+    env,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  return { code: r.exitCode, out: r.stdout.toString(), err: r.stderr.toString() };
+}
+
+describe("ops/caelo-converge-job (the converge job's entry point)", () => {
+  it("refuses every argument list but exactly `converge-job`, before any code runs", () => {
+    const dir = mkdtempSync(join(tmpdir(), "caelo-entry-"));
+    try {
+      const path = scriptPrintingEnv(dir);
+      for (const args of [
+        [],
+        ["destroy"],
+        ["owner-reset", "--email", "evil@example.com"],
+        ["rotate-secret"],
+        ["converge-job", "--extra"],
+        ["upgrader"],
+      ]) {
+        const r = runScript(path, args, { PATH: "/usr/bin:/bin" });
+        expect(r.code).toBe(64);
+        expect(r.err).toContain("refused");
+        expect(r.out).toBe("");
+      }
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("starts the job with a scrubbed environment: overrides never reach gcloud, Pulumi or Bun", () => {
+    const dir = mkdtempSync(join(tmpdir(), "caelo-entry-"));
+    try {
+      const r = runScript(scriptPrintingEnv(dir), ["converge-job"], {
+        PATH: "/usr/bin:/bin",
+        CLOUDSDK_API_ENDPOINT_OVERRIDES_RUN: "https://evil.example/",
+        CLOUDSDK_AUTH_ACCESS_TOKEN: "stolen",
+        PULUMI_BACKEND_URL: "gs://evil",
+        PULUMI_CONFIG_PASSPHRASE: "x",
+        BUN_OPTIONS: "--preload=/tmp/x.ts",
+        LD_PRELOAD: "/tmp/x.so",
+        NODE_OPTIONS: "--require=/tmp/x.js",
+        CLOUD_RUN_EXECUTION: "caelo-production-converge-abc12",
+        CLOUD_RUN_JOB: "caelo-production-converge",
+      });
+      expect(r.code).toBe(0);
+      const env = Object.fromEntries(
+        r.out
+          .trim()
+          .split("\n")
+          .map((l) => [l.slice(0, l.indexOf("=")), l.slice(l.indexOf("=") + 1)]),
+      );
+      expect(Object.keys(env).sort()).toEqual(
+        [
+          "CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK",
+          "CLOUDSDK_CORE_DISABLE_PROMPTS",
+          "CLOUD_RUN_EXECUTION",
+          "CLOUD_RUN_JOB",
+          "CLOUD_RUN_TASK_ATTEMPT",
+          "CLOUD_RUN_TASK_COUNT",
+          "CLOUD_RUN_TASK_INDEX",
+          "HOME",
+          "PATH",
+          "PULUMI_SKIP_UPDATE_CHECK",
+        ].sort(),
+      );
+      expect(env.CLOUD_RUN_EXECUTION).toBe("caelo-production-converge-abc12");
+      expect(env.PATH).toBe("/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("the deployed job runs exactly that entry point with that one argument", () => {
+    const args = convergeJobDeployArgs({
+      projectId: "acme",
+      region: "europe-west1",
+      env: "production",
+      opsImageRef: `x/ops@sha256:${"c".repeat(64)}`,
+      network: "n",
+      subnet: "s",
+    });
+    expect(args).toContain("--command=caelo-converge-job");
+    expect(args).toContain("--args=converge-job");
+    expect(args).toContain("--clear-env-vars");
+  });
+});
+
+describe("no upgrader or admin role carries run.jobs.runWithOverrides (#883 review)", () => {
+  it("the forbidden list covers every predefined role that does", () => {
+    for (const role of [
+      "roles/run.developer",
+      "roles/run.admin",
+      "roles/run.invoker",
+      "roles/run.jobsExecutorWithOverrides",
+      "roles/editor",
+      "roles/owner",
+    ]) {
+      expect(RUN_WITH_OVERRIDES_ROLES).toContain(role as (typeof RUN_WITH_OVERRIDES_ROLES)[number]);
+      expect(UPGRADER_FORBIDDEN_ROLES.some((f) => role.startsWith(f))).toBe(true);
+    }
+  });
+
+  it("the upgrader's own Cloud Run role leaves it out", () => {
+    expect(UPGRADER_RUN_ROLE.permissions).not.toContain("run.jobs.runWithOverrides");
+    expect(UPGRADER_RUN_ROLE.permissions.some((p) => /setIamPolicy|actAs/.test(p))).toBe(false);
+    for (const role of UPGRADER_PROJECT_ROLES) {
+      expect(RUN_WITH_OVERRIDES_ROLES as readonly string[]).not.toContain(role);
+    }
+  });
+
+  it("setup grants the upgrader no such role, and removes the run.developer an earlier setup gave it", async () => {
+    const calls: string[][] = [];
+    const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+    const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+    const upgrader = "serviceAccount:caelo-production-upgrader@acme.iam.gserviceaccount.com";
+    const rules: [RegExp, GcloudResult][] = [
+      [/^iam roles describe/, fail("NOT_FOUND")],
+      [/^iam service-accounts describe/, ok("x")],
+      [
+        /^projects get-iam-policy/,
+        ok(JSON.stringify({ bindings: [{ role: "roles/run.developer", members: [upgrader] }] })),
+      ],
+      [/^sql instances list/, ok("caelo-production-pg-abc\n")],
+      [/^sql instances describe/, ok('{"settings":{"databaseFlags":[]}}')],
+      [/^sql users list/, ok("")],
+    ];
+    const run = async (args: string[]) => {
+      calls.push(args);
+      return rules.find(([re]) => re.test(args.join(" ")))?.[1] ?? ok("");
+    };
+    const r = await ensureUpgraderJob(
+      {
+        projectId: "acme",
+        region: "europe-west1",
+        env: "production",
+        opsImageRef: `x/ops@sha256:${"c".repeat(64)}`,
+        network: "n",
+        subnet: "s",
+      },
+      { run, sleep: async () => {} },
+    );
+    expect(r.ok).toBe(true);
+    const roleOf = (c: string[]) => c.find((a) => a.startsWith("--role="))?.slice(7) ?? "";
+    const granted = calls.filter((c) => c.includes("add-iam-policy-binding")).map(roleOf);
+    for (const role of granted) {
+      expect(RUN_WITH_OVERRIDES_ROLES as readonly string[]).not.toContain(role);
+    }
+    const removed = calls.filter((c) => c.includes("remove-iam-policy-binding"));
+    expect(removed.map((c) => `${roleOf(c)} ${c.find((a) => a.startsWith("--member="))}`)).toEqual([
+      `roles/run.developer --member=${upgrader}`,
+    ]);
+    // The custom role is bound before the old one is removed.
+    const bindRun = calls.findIndex((c) => roleOf(c) === "projects/acme/roles/caeloUpgraderRun");
+    const removeDev = calls.findIndex((c) => c.includes("remove-iam-policy-binding"));
+    expect(bindRun).toBeGreaterThan(-1);
+    expect(bindRun).toBeLessThan(removeDev);
+  });
+
+  it("the admin's roles on the converge job carry none", () => {
+    for (const role of CONVERGE_ADMIN_ROLES) {
+      expect(RUN_WITH_OVERRIDES_ROLES as readonly string[]).not.toContain(role);
+    }
+  });
+});

package/src/converge-job-gcp.test.ts

@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The parts of the converge job's Google Cloud wiring (#819) that parse
+ * what gcloud says — which image an execution runs, which execution a
+ * hand-over started — and its checkpoint document in the ops bucket.
+ */
+
+import { describe, expect, it } from "bun:test";
+import {
+  type ConvergeCheckpoint,
+  convergeCheckpointUrl,
+  readConvergeCheckpoint,
+  writeConvergeCheckpoint,
+} from "./converge-checkpoint.js";
+import {
+  convergeExecutionNameError,
+  deployedOwnerMismatch,
+  deployedSiteMismatch,
+  executionImageDigest,
+  startedExecutionName,
+} from "./converge-job-gcp.js";
+import type { GcloudResult } from "./gcloud.js";
+import type { SyncGcloud } from "./state-store.js";
+import { installStateCondition } from "./upgrader-setup.js";
+
+const D = `sha256:${"c".repeat(64)}`;
+const OPS = `europe-west1-docker.pkg.dev/caelo-website/caelo-cms-images/ops@${D}`;
+
+describe("executionImageDigest", () => {
+  it("reads the digest from gcloud's v1 execution (and the v2 shape)", () => {
+    const v1 = { spec: { template: { spec: { containers: [{ image: OPS }] } } } };
+    expect(executionImageDigest(JSON.stringify(v1))).toBe(D);
+    expect(
+      executionImageDigest(JSON.stringify({ template: { containers: [{ image: OPS }] } })),
+    ).toBe(D);
+  });
+
+  it("is null for a tag, no container or unreadable output — never a guess", () => {
+    const tag = { spec: { template: { spec: { containers: [{ image: "x/ops:0.11.0" }] } } } };
+    expect(executionImageDigest(JSON.stringify(tag))).toBeNull();
+    expect(executionImageDigest("{}")).toBeNull();
+    expect(executionImageDigest("not json")).toBeNull();
+  });
+});
+
+describe("startedExecutionName", () => {
+  it("takes the name gcloud prints, or the one it announces on stderr", () => {
+    expect(startedExecutionName("caelo-production-converge-x7k2p\n", "")).toBe(
+      "caelo-production-converge-x7k2p",
+    );
+    expect(
+      startedExecutionName(
+        "",
+        "Execution [caelo-production-converge-q1w2e] has successfully started",
+      ),
+    ).toBe("caelo-production-converge-q1w2e");
+    expect(startedExecutionName("", "")).toBeNull();
+    expect(startedExecutionName("Bad Name; rm -rf", "")).toBeNull();
+  });
+});
+
+describe("the converge checkpoint", () => {
+  const BUCKET = "acme-caelo-ops";
+  const fakeBucket = () => {
+    const objects = new Map<string, string>();
+    const run: SyncGcloud = (args, input): GcloudResult => {
+      if (args[1] === "cat") {
+        const v = objects.get(args[2] as string);
+        return v === undefined
+          ? { ok: false, stdout: "", stderr: "One or more URLs matched no objects.", exitCode: 1 }
+          : { ok: true, stdout: v, stderr: "", exitCode: 0 };
+      }
+      objects.set(args[3] as string, input ?? "");
+      return { ok: true, stdout: "", stderr: "", exitCode: 0 };
+    };
+    return { run, objects };
+  };
+  const CHECKPOINT: ConvergeCheckpoint = {
+    statusVersion: 1,
+    installId: "gcp-acme",
+    execution: "caelo-production-converge-x7k2p",
+    state: "running",
+    phase: "converge",
+    caeloVersion: "0.11.0",
+    startedAt: "2026-10-19T12:00:00Z",
+    updatedAt: "2026-10-19T12:05:00Z",
+    targetVersion: "0.11.0",
+    fromVersion: "0.10.53",
+    requestId: "11111111-2222-4333-8444-555555555555",
+    backup: { projectId: "acme", instance: "caelo-production-pg-x", backupId: "1760000000" },
+    priorRevisions: { admin: "a-00001", gateway: "g-00001" },
+  };
+
+  it("lives outside install/, where the upgrader may write, and round-trips (#883 review)", () => {
+    const { run, objects } = fakeBucket();
+    expect(readConvergeCheckpoint(BUCKET, run)).toBeNull();
+    writeConvergeCheckpoint(BUCKET, CHECKPOINT, run);
+    expect([...objects.keys()]).toEqual(["gs://acme-caelo-ops/converge/converge-job.json"]);
+    expect(convergeCheckpointUrl(BUCKET)).toBe("gs://acme-caelo-ops/converge/converge-job.json");
+    expect(readConvergeCheckpoint(BUCKET, run)).toEqual(CHECKPOINT);
+    // The upgrader's only object grant on the bucket is conditioned on
+    // install/ — the checkpoint is not under it.
+    const condition = installStateCondition(BUCKET);
+    const prefix = /objects\/([^"]+)"/.exec(condition)?.[1] ?? "";
+    expect(prefix).toBe("install/");
+    expect(convergeCheckpointUrl(BUCKET).startsWith(`gs://${BUCKET}/${prefix}`)).toBe(false);
+  });
+
+  it("refuses a document it does not understand", () => {
+    const { run, objects } = fakeBucket();
+    objects.set(convergeCheckpointUrl(BUCKET), JSON.stringify({ ...CHECKPOINT, fromVersion: "x" }));
+    expect(() => readConvergeCheckpoint(BUCKET, run)).toThrow(/not a job status/);
+  });
+});
+
+describe("deployedOwnerMismatch (#883 review)", () => {
+  const job = (members: string | null) =>
+    JSON.stringify({
+      spec: {
+        template: {
+          spec: {
+            template: {
+              spec: {
+                containers: [
+                  {
+                    env: [
+                      { name: "CAELO_OPERATOR_ACCESS_IAP_WEB", value: "x" },
+                      ...(members === null
+                        ? []
+                        : [{ name: "CAELO_OPERATOR_ACCESS_STATIC_MEMBERS", value: members }]),
+                    ],
+                  },
+                ],
+              },
+            },
+          },
+        },
+      },
+    });
+
+  it("accepts install.json's Owner when the deployed IAP members are exactly it", () => {
+    expect(deployedOwnerMismatch(job("user:owner@acme.com"), "Owner@Acme.com")).toBeNull();
+  });
+
+  it("refuses an Owner email the deployed access settings do not admit (a tampered install.json)", () => {
+    const why = deployedOwnerMismatch(job("user:owner@acme.com"), "attacker@evil.example");
+    expect(why).toContain("user:owner@acme.com");
+    expect(why).toContain("user:attacker@evil.example");
+  });
+
+  it("refuses when the deployed members cannot be read — never assumes they match", () => {
+    expect(deployedOwnerMismatch(job(null), "owner@acme.com")).toContain("sets no");
+    expect(deployedOwnerMismatch("not json", "owner@acme.com")).toContain("not JSON");
+    expect(
+      deployedOwnerMismatch(job("user:owner@acme.com,user:x@y.z"), "owner@acme.com"),
+    ).not.toBeNull();
+  });
+});
+
+describe("deployedSiteMismatch (#883 review, round 2)", () => {
+  const admin = (env: Record<string, string>) =>
+    JSON.stringify({
+      spec: {
+        template: {
+          spec: {
+            containers: [{ env: Object.entries(env).map(([name, value]) => ({ name, value })) }],
+          },
+        },
+      },
+    });
+  const META = { domain: "acme.com", region: "europe-west1" } as const;
+  const SERVED = { CAELO_SITE_URL: "https://acme.com", CAELO_STAGING_HOST: "staging.acme.com" };
+
+  it("accepts the records of the site the admin serves, in the job's own region", () => {
+    expect(deployedSiteMismatch(admin(SERVED), META, "europe-west1")).toBeNull();
+    // An admin from before #650 has no staging host yet: the default is fine.
+    expect(
+      deployedSiteMismatch(admin({ CAELO_SITE_URL: "https://acme.com" }), META, "europe-west1"),
+    ).toBeNull();
+  });
+
+  it("refuses a tampered domain, staging host or region: the provisioner would act on it", () => {
+    expect(
+      deployedSiteMismatch(admin(SERVED), { ...META, domain: "evil.example" }, "europe-west1"),
+    ).toContain("https://evil.example");
+    expect(
+      deployedSiteMismatch(
+        admin(SERVED),
+        { ...META, stagingDomain: "staging.evil.example" },
+        "europe-west1",
+      ),
+    ).toContain("staging.evil.example");
+    expect(
+      deployedSiteMismatch(
+        admin({ CAELO_SITE_URL: "https://acme.com" }),
+        { ...META, stagingDomain: "staging.evil.example" },
+        "europe-west1",
+      ),
+    ).not.toBeNull();
+    expect(
+      deployedSiteMismatch(admin(SERVED), { ...META, region: "us-central1" }, "europe-west1"),
+    ).toContain("us-central1");
+  });
+
+  it("refuses when the admin's site cannot be read — never assumes it matches", () => {
+    expect(deployedSiteMismatch("not json", META, "europe-west1")).not.toBeNull();
+    expect(deployedSiteMismatch(admin({}), META, "europe-west1")).toContain("CAELO_SITE_URL");
+  });
+});
+
+describe("convergeExecutionNameError (#883 review)", () => {
+  it("accepts a run of the converge job and no name at all (outside Cloud Run)", () => {
+    expect(convergeExecutionNameError("caelo-production-converge-x7k2p")).toBeNull();
+    expect(convergeExecutionNameError(null)).toBeNull();
+  });
+
+  it("refuses a run of another job or a value that is no Cloud Run name", () => {
+    expect(convergeExecutionNameError("caelo-production-upgrader-x1")).not.toBeNull();
+    expect(convergeExecutionNameError("caelo-production-converge-X;rm")).not.toBeNull();
+  });
+});