@caelo-cms/provisioning 0.11.10 → 0.11.11

package/src/runtime-secrets.ts

@@ -572,6 +572,12 @@ async function setSqlUserPassword(
   http: HttpFetch,
   sleep: Sleep,
   user: { projectId: string; instance: string; role: string; password: string },
+  /**
+   * `update` sets an existing user's password (`users.update`); `create`
+   * adds the user with it (`users.insert`, #1329: the staging gateway's
+   * login on an install from before it).
+   */
+  mode: "update" | "create",
 ): Promise<{ ok: true } | { ok: false; error: string }> {
   const token = await run(["auth", "print-access-token"]);
   const bearer = token.ok ? token.stdout.trim() : "";
@@ -587,11 +593,17 @@ async function setSqlUserPassword(
     ok: false as const,
     error: `${what}: HTTP ${res.status} ${(await res.text()).slice(0, 500)}`,
   });
-  const put = await http(
-    `${project}/instances/${encodeURIComponent(user.instance)}/users?name=${encodeURIComponent(user.role)}`,
-    { method: "PUT", headers, body: JSON.stringify({ name: user.role, password: user.password }) },
-  );
-  if (!put.ok) return failed("update the user", put);
+  const users = `${project}/instances/${encodeURIComponent(user.instance)}/users`;
+  const body = JSON.stringify({ name: user.role, password: user.password });
+  const put =
+    mode === "update"
+      ? await http(`${users}?name=${encodeURIComponent(user.role)}`, {
+          method: "PUT",
+          headers,
+          body,
+        })
+      : await http(users, { method: "POST", headers, body });
+  if (!put.ok) return failed(mode === "update" ? "update the user" : "create the user", put);
   type Operation = {
     name?: string;
     status?: string;
@@ -693,12 +705,13 @@ export async function rotateRuntimeSecret(
     }
     const previous = current.stdout;
     const setPassword = (role: string, password: string) =>
-      setSqlUserPassword(run, http, sleep, {
-        projectId: target.projectId,
-        instance,
-        role,
-        password,
-      });
+      setSqlUserPassword(
+        run,
+        http,
+        sleep,
+        { projectId: target.projectId, instance, role, password },
+        "update",
+      );
     const changed: string[] = [];
     const restore = async (): Promise<string> => {
       const failed: string[] = [];
@@ -807,7 +820,12 @@ export async function readSecretValue(
 export async function ensureDatabaseRolePassword(
   target: DatabaseRoleTarget,
   role: DatabaseRole,
-  deps: RuntimeDeps & { readonly http?: HttpFetch; readonly password?: string } = {},
+  deps: RuntimeDeps & {
+    readonly http?: HttpFetch;
+    readonly password?: string;
+    /** Create the Cloud SQL user with the password instead (it does not exist yet, #1329). */
+    readonly create?: boolean;
+  } = {},
 ): Promise<EnsureOutcome> {
   const run = deps.run ?? defaultGcloud;
   const sleep = deps.sleep ?? realSleep;
@@ -819,11 +837,12 @@ export async function ensureDatabaseRolePassword(
     if (!value.ok) return { id, status: "failed", error: value.error };
     password = value.value;
   }
-  const set = await setSqlUserPassword(run, http, sleep, {
-    projectId: target.projectId,
-    instance: target.sqlInstance,
-    role,
-    password,
-  });
+  const set = await setSqlUserPassword(
+    run,
+    http,
+    sleep,
+    { projectId: target.projectId, instance: target.sqlInstance, role, password },
+    deps.create === true ? "create" : "update",
+  );
   return set.ok ? { id, status: "applied" } : { id, status: "failed", error: set.error };
 }

package/src/scheduler-setup.test.ts

@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The scheduler tick's setup (scheduler-setup.ts) against a fake gcloud. The
+ * load-bearing assertions are the trust boundary (#1140): the scheduler's
+ * account may sign as itself, pass the admin's IAP, start workflows and
+ * write logs — no secret, database, storage, run or IAM-admin right — and
+ * the one Cloud Scheduler job starts the workflow at the knob's cadence.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { GcloudResult } from "./gcloud.js";
+import {
+  ensureSchedulerTick,
+  SCHEDULER_FORBIDDEN_ROLES,
+  SCHEDULER_TICK_PATH,
+  type SchedulerTickTarget,
+  schedulerWorkflowSource,
+  tickOrigin,
+} from "./scheduler-setup.js";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+const SA = "caelo-production-scheduler@acme.iam.gserviceaccount.com";
+const MEMBER = `serviceAccount:${SA}`;
+
+const TARGET: SchedulerTickTarget = {
+  projectId: "acme",
+  region: "europe-west1",
+  env: "production",
+  adminUrl: "https://admin.acme.com",
+  resource: { kind: "backend-services", service: "caelo-production-admin-backend-1a2b" },
+  intervalMinutes: 5,
+};
+
+function fakeGcloud(overrides: [RegExp, GcloudResult][] = []) {
+  const calls: string[][] = [];
+  const sources: string[] = [];
+  const rules: [RegExp, GcloudResult][] = [
+    ...overrides,
+    [/^iam service-accounts describe/, fail("NOT_FOUND")],
+    [/^scheduler jobs describe/, fail("NOT_FOUND")],
+  ];
+  const run = async (args: string[]) => {
+    calls.push(args);
+    const src = args.find((a) => a.startsWith("--source="));
+    if (src) sources.push(await Bun.file(src.slice("--source=".length)).text());
+    const line = args.join(" ");
+    return rules.find(([re]) => re.test(line))?.[1] ?? ok("");
+  };
+  return { run, calls, sources };
+}
+
+const flag = (args: string[], name: string) =>
+  args.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3);
+
+function grants(calls: string[][]) {
+  return calls
+    .filter((c) => c.includes("add-iam-policy-binding"))
+    .map((c) => ({
+      where: c.slice(0, c.indexOf("add-iam-policy-binding") + 2).join(" "),
+      member: flag(c, "member"),
+      role: flag(c, "role") as string,
+    }));
+}
+
+const sleep = async () => {};
+
+describe("ensureSchedulerTick", () => {
+  it("grants the scheduler only self-signing, IAP access, workflow start and logs", async () => {
+    const { run, calls } = fakeGcloud();
+    const r = await ensureSchedulerTick(TARGET, { run, sleep });
+    expect(r.ok).toBe(true);
+    const mine = grants(calls).filter((g) => g.member === MEMBER);
+    expect(mine.map((g) => [g.where, g.role])).toEqual([
+      [`iam service-accounts add-iam-policy-binding ${SA}`, "roles/iam.serviceAccountTokenCreator"],
+      ["projects add-iam-policy-binding acme", "roles/workflows.invoker"],
+      ["projects add-iam-policy-binding acme", "roles/logging.logWriter"],
+      [
+        "iap web add-iam-policy-binding --resource-type=backend-services",
+        "roles/iap.httpsResourceAccessor",
+      ],
+    ]);
+    for (const g of mine) {
+      for (const forbidden of SCHEDULER_FORBIDDEN_ROLES) {
+        expect(g.role.startsWith(forbidden)).toBe(false);
+      }
+    }
+    // Nobody else is granted anything.
+    expect(grants(calls).every((g) => g.member === MEMBER)).toBe(true);
+    // The IAP grant names the admin's resource.
+    const iap = calls.find((c) => c[0] === "iap") as string[];
+    expect(iap).toContain("--resource-type=backend-services");
+    expect(iap).toContain("--service=caelo-production-admin-backend-1a2b");
+  });
+
+  it("signs as itself only: token creator on its own account, never on another", async () => {
+    const { run, calls } = fakeGcloud();
+    await ensureSchedulerTick(TARGET, { run, sleep });
+    const saGrants = calls.filter(
+      (c) => c[0] === "iam" && c[1] === "service-accounts" && c[2] === "add-iam-policy-binding",
+    );
+    expect(saGrants.map((c) => c[3])).toEqual([SA]);
+  });
+
+  it("deploys the workflow as the scheduler account and schedules it every 5 minutes", async () => {
+    const { run, calls, sources } = fakeGcloud();
+    await ensureSchedulerTick(TARGET, { run, sleep });
+    const deploy = calls.find((c) => c[0] === "workflows" && c[1] === "deploy") as string[];
+    expect(deploy[2]).toBe("caelo-production-scheduler-tick");
+    expect(flag(deploy, "service-account")).toBe(SA);
+    expect(flag(deploy, "location")).toBe("europe-west1");
+    expect(sources[0]).toContain(`url: "https://admin.acme.com${SCHEDULER_TICK_PATH}"`);
+    const create = calls.find(
+      (c) => c[0] === "scheduler" && c[1] === "jobs" && c[2] === "create",
+    ) as string[];
+    expect(create[4]).toBe("caelo-production-scheduler-tick");
+    expect(flag(create, "schedule")).toBe("*/5 * * * *");
+    expect(flag(create, "time-zone")).toBe("Etc/UTC");
+    expect(flag(create, "uri")).toBe(
+      "https://workflowexecutions.googleapis.com/v1/projects/acme/locations/europe-west1/workflows/caelo-production-scheduler-tick/executions",
+    );
+    expect(flag(create, "oauth-service-account-email")).toBe(SA);
+    // OAuth (a googleapis.com target), never an OIDC token IAP would refuse.
+    expect(create.some((a) => a.startsWith("--oidc"))).toBe(false);
+  });
+
+  it("updates the existing job on a re-run, at the knob's new cadence", async () => {
+    const { run, calls } = fakeGcloud([[/^scheduler jobs describe/, ok("projects/acme/…")]]);
+    const r = await ensureSchedulerTick({ ...TARGET, intervalMinutes: 1 }, { run, sleep });
+    expect(r.ok).toBe(true);
+    const update = calls.find(
+      (c) => c[0] === "scheduler" && c[1] === "jobs" && c[2] === "update",
+    ) as string[];
+    expect(flag(update, "schedule")).toBe("* * * * *");
+    expect(calls.some((c) => c[0] === "scheduler" && c[2] === "create")).toBe(false);
+  });
+
+  it("refuses an interval outside 1-15 minutes before touching the project", async () => {
+    const { run, calls } = fakeGcloud();
+    const r = await ensureSchedulerTick({ ...TARGET, intervalMinutes: 30 }, { run, sleep });
+    expect(r.ok).toBe(false);
+    expect(r.ok ? "" : r.error).toContain("between 1 and 15");
+    expect(calls).toEqual([]);
+  });
+
+  it("refuses an admin URL that is not a plain https origin", async () => {
+    for (const bad of ["http://admin.acme.com", "https://admin.acme.com/x", "https://a@b.com"]) {
+      const { run, calls } = fakeGcloud();
+      const r = await ensureSchedulerTick({ ...TARGET, adminUrl: bad }, { run, sleep });
+      expect(r.ok).toBe(false);
+      expect(calls).toEqual([]);
+    }
+  });
+
+  it("stops at the first failing step and says which", async () => {
+    const { run, calls } = fakeGcloud([[/^workflows deploy/, fail("PERMISSION_DENIED: nope")]]);
+    const r = await ensureSchedulerTick(TARGET, { run, sleep });
+    expect(r.ok).toBe(false);
+    expect(r.ok ? "" : r.error).toContain("deploy workflow caelo-production-scheduler-tick");
+    expect(calls.some((c) => c[0] === "scheduler")).toBe(false);
+  });
+});
+
+describe("schedulerWorkflowSource", () => {
+  const src = schedulerWorkflowSource({ serviceAccount: SA, origin: "https://admin.acme.com" });
+
+  it("signs a JWT for the admin origin as the scheduler itself", () => {
+    expect(src).toContain(`iss: "${SA}"`);
+    expect(src).toContain(`sub: "${SA}"`);
+    expect(src).toContain('aud: "https://admin.acme.com/*"');
+    expect(src).toContain(
+      `url: "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${SA}:signJwt"`,
+    );
+    expect(src).toContain("type: OAuth2");
+    expect(src).toContain(`exp: \${now + 600}`);
+  });
+
+  it("posts the tick with the signed JWT and a bounded timeout", () => {
+    expect(src).toContain('url: "https://admin.acme.com/api/internal/scheduler/tick"');
+    expect(src).toContain(`Authorization: \${"Bearer " + signed.body.signedJwt}`);
+    expect(src).toMatch(/timeout: (\d+)/);
+    const timeout = Number(/timeout: (\d+)/.exec(src)?.[1]);
+    expect(timeout).toBeGreaterThan(300);
+    expect(timeout).toBeLessThanOrEqual(1800);
+  });
+});
+
+describe("tickOrigin", () => {
+  it("accepts the admin's origin, with or without a trailing slash", () => {
+    expect(tickOrigin("https://admin.acme.com/")).toEqual({
+      ok: true,
+      origin: "https://admin.acme.com",
+    });
+    expect(tickOrigin("https://caelo-production-admin-x-ew.a.run.app")).toEqual({
+      ok: true,
+      origin: "https://caelo-production-admin-x-ew.a.run.app",
+    });
+  });
+});

package/src/scheduler-setup.ts

@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #1140 — set up (and converge) the scheduler tick on a Google Cloud install
+ * (`gcp`, `gcp-firebase`): the ONE Cloud Scheduler job that calls the
+ * admin's `/api/internal/scheduler/tick` every `schedulerIntervalMinutes`
+ * (1–15, default 5). Each task's own cadence lives in the admin's
+ * due-table; the tick only says "run what is due now". Run by the wizard
+ * after migrations and by `cms-provision upgrade`'s converge (and the
+ * converge job), as the operator's or the provisioner's identity.
+ *
+ * Why not Cloud Scheduler → admin directly. The admin sits behind IAP, and
+ * IAP accepts a service account's OIDC ID token only when its `aud` is an
+ * OAuth client allowlisted for programmatic access; an install on the
+ * Google-managed IAP client has none (cloud.google.com/iap/docs/
+ * authentication-howto). Cloud Scheduler's other credential, an OAuth
+ * access token, it sends only to `*.googleapis.com` (cloud.google.com/
+ * scheduler/docs/http-target-auth). What passes IAP for a service account
+ * on every install is a self-signed service-account JWT with `aud` =
+ * `<admin origin>/*` — the path caelo-mcp and admin-internal-google.ts
+ * take. Cloud Scheduler cannot sign one, so the job starts a Cloud
+ * Workflows execution (a googleapis.com target, OAuth), and the workflow
+ * signs the JWT as the scheduler account through the IAM Credentials API
+ * and POSTs the tick. No container per tick and no image to keep in step
+ * with the app-roll; cents a month (wizards/gcp-cost.ts).
+ *
+ * Why the CLI owns it and not the stacks: the same as audit-job-setup.ts —
+ * `upgrade` never runs Pulumi, and existing installs get the tick from here.
+ *
+ * Trust boundary. The scheduler's identity, `caelo-<env>-scheduler`, holds:
+ *
+ *   - `roles/iam.serviceAccountTokenCreator` on ITSELF only (signJwt);
+ *   - `roles/iap.httpsResourceAccessor` on the admin's IAP resource (the
+ *     operator-access sync job keeps it there, operator-access.ts);
+ *   - `roles/workflows.invoker` on the project (Cloud Scheduler starts the
+ *     workflow as it; Workflows has no per-workflow IAM policy to bind to);
+ *   - `roles/logging.logWriter` (the workflow's error call logs).
+ *
+ * Nothing else: no Secret Manager, no Cloud SQL, no Cloud Run rights. What
+ * it can do with the admin is the admin's to decide: the tick endpoint
+ * accepts exactly this account's IAP identity (`CAELO_SCHEDULER_SERVICE_
+ * ACCOUNT` on the admin) and only ever runs the due-table, never a build
+ * or a publish (scripts/no-automatic-production-deploy.test.ts).
+ * {@link SCHEDULER_FORBIDDEN_ROLES} names that line; the tests assert no
+ * grant crosses it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import {
+  schedulerJobName,
+  schedulerServiceAccountEmail,
+  schedulerServiceAccountId,
+  schedulerWorkflowName,
+} from "./gcp-names.js";
+import { type IapResource, iapResourceArgs } from "./mcp-iap.js";
+import { schedulerCron } from "./stack-contract.js";
+
+/** The admin path the workflow calls (apps/admin `api/internal/scheduler/tick`). */
+export const SCHEDULER_TICK_PATH = "/api/internal/scheduler/tick";
+
+/**
+ * How long the workflow waits for the admin's answer, in seconds: Workflows'
+ * maximum (cloud.google.com/workflows/docs/reference/stdlib/http/post). The
+ * admin holds the request until the tick's work is done: it claims tasks for
+ * at most its budget (4 minutes, admin-core scheduler/tick.ts) and then waits
+ * at most one task's lease (10 minutes) for the last one, so 14 minutes in
+ * the worst case, inside the admin's own 3600 s request timeout.
+ */
+export const SCHEDULER_TICK_TIMEOUT_SECONDS = 1800;
+
+/** Lifetime of the IAP credential the workflow signs per tick. */
+const JWT_LIFETIME_SECONDS = 600;
+
+/** APIs the tick needs on the project. */
+export const SCHEDULER_APIS = [
+  "cloudscheduler.googleapis.com",
+  "workflows.googleapis.com",
+  "iamcredentials.googleapis.com",
+] as const;
+
+/**
+ * Roles the scheduler's identity must never hold anywhere. Prefix matches
+ * (`roles/secretmanager.` = every Secret Manager role).
+ */
+export const SCHEDULER_FORBIDDEN_ROLES = [
+  "roles/owner",
+  "roles/editor",
+  "roles/viewer",
+  "roles/run.",
+  "roles/iam.serviceAccountUser",
+  "roles/iam.serviceAccountAdmin",
+  "roles/iam.securityAdmin",
+  "roles/iap.admin",
+  "roles/iap.settingsAdmin",
+  "roles/secretmanager.",
+  "roles/cloudsql.",
+  "roles/compute.",
+  "roles/storage.",
+  "roles/resourcemanager.",
+  "roles/workflows.admin",
+  "roles/workflows.editor",
+  "roles/cloudscheduler.",
+] as const;
+
+export interface SchedulerTickTarget {
+  readonly projectId: string;
+  readonly region: string;
+  /** Pulumi stack name (`production`). */
+  readonly env: string;
+  /**
+   * Where the admin answers behind IAP, an origin: `https://admin.<domain>`
+   * on gcp, the admin service's URL on gcp-firebase (remote-mcp.ts
+   * RemoteMcpIdentity.adminUrl, the same value as the admin's
+   * CAELO_ADMIN_URL).
+   */
+  readonly adminUrl: string;
+  /** The admin's IAP resource (the service on gcp-firebase, the LB backend on gcp). */
+  readonly resource: IapResource;
+  /** The `schedulerIntervalMinutes` knob (1–15). */
+  readonly intervalMinutes: number;
+}
+
+export type SchedulerTickResult =
+  | { readonly ok: true; readonly done: string[]; readonly serviceAccount: string }
+  | { readonly ok: false; readonly done: string[]; readonly error: string };
+
+/**
+ * The tick's admin origin, or an error naming why it is not one: an https
+ * origin without path, query or credentials (it is written into the
+ * workflow's source, so nothing but an origin may get there).
+ */
+export function tickOrigin(
+  adminUrl: string,
+): { ok: true; origin: string } | { ok: false; error: string } {
+  let u: URL;
+  try {
+    u = new URL(adminUrl);
+  } catch {
+    return { ok: false, error: `the admin URL ${JSON.stringify(adminUrl)} is not a URL` };
+  }
+  if (
+    u.protocol !== "https:" ||
+    u.username ||
+    u.password ||
+    (u.pathname !== "/" && u.pathname !== "") ||
+    u.search ||
+    u.hash ||
+    !/^[a-z0-9.-]+$/.test(u.hostname)
+  ) {
+    return {
+      ok: false,
+      error: `the admin URL ${JSON.stringify(adminUrl)} is not a plain https origin`,
+    };
+  }
+  return { ok: true, origin: u.origin };
+}
+
+/**
+ * The workflow one tick runs (Cloud Workflows YAML): sign a service-account
+ * JWT for the admin's IAP as the scheduler account itself (the IAM
+ * Credentials API, authenticated with the workflow's own OAuth token), then
+ * POST the tick with it. A refused or failed tick (HTTP ≥ 400) fails the
+ * execution, so it shows in the workflow's execution history and logs.
+ */
+export function schedulerWorkflowSource(opts: {
+  readonly serviceAccount: string;
+  readonly origin: string;
+}): string {
+  const { serviceAccount: sa, origin } = opts;
+  return `# Caelo scheduler tick (#1140) — generated by cms-provision, do not edit.
+main:
+  steps:
+    - claims:
+        assign:
+          - now: \${int(sys.now())}
+          - jwtClaims:
+              iss: "${sa}"
+              sub: "${sa}"
+              aud: "${origin}/*"
+              iat: \${now}
+              exp: \${now + ${JWT_LIFETIME_SECONDS}}
+    - sign:
+        call: http.post
+        args:
+          url: "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${sa}:signJwt"
+          auth:
+            type: OAuth2
+          body:
+            payload: \${json.encode_to_string(jwtClaims)}
+        result: signed
+    - tick:
+        call: http.post
+        args:
+          url: "${origin}${SCHEDULER_TICK_PATH}"
+          headers:
+            Authorization: \${"Bearer " + signed.body.signedJwt}
+          timeout: ${SCHEDULER_TICK_TIMEOUT_SECONDS}
+        result: answer
+    - done:
+        return: \${answer.code}
+`;
+}
+
+/** Ensure the scheduler tick and everything it needs. Never throws for a gcloud failure. */
+export async function ensureSchedulerTick(
+  target: SchedulerTickTarget,
+  deps: { run?: GcloudRunner; sleep?: Sleep } = {},
+): Promise<SchedulerTickResult> {
+  const run = deps.run ?? defaultGcloud;
+  const sleep = deps.sleep ?? realSleep;
+  const retry = (args: string[]) => runWithRetry(run, sleep, args);
+  const { projectId, region, env } = target;
+  const project = `--project=${projectId}`;
+  const sa = schedulerServiceAccountEmail(projectId, env);
+  const member = `serviceAccount:${sa}`;
+  const workflow = schedulerWorkflowName(env);
+  const job = schedulerJobName(env);
+  const done: string[] = [];
+  const fail = (step: string, r: GcloudResult): SchedulerTickResult => ({
+    ok: false,
+    done,
+    error: `${step}: ${r.stderr.trim()}`,
+  });
+
+  const origin = tickOrigin(target.adminUrl);
+  if (!origin.ok) return { ok: false, done, error: origin.error };
+  let cron: string;
+  try {
+    cron = schedulerCron(target.intervalMinutes);
+  } catch (e) {
+    return { ok: false, done, error: (e as Error).message };
+  }
+
+  // 1. APIs.
+  const api = await run(["services", "enable", ...SCHEDULER_APIS, project, "--quiet"]);
+  if (!api.ok) return fail(`enable ${SCHEDULER_APIS.join(", ")}`, api);
+  done.push(`APIs enabled (${SCHEDULER_APIS.join(", ")})`);
+
+  // 2. The scheduler's own service account.
+  const describe = await run([
+    "iam",
+    "service-accounts",
+    "describe",
+    sa,
+    project,
+    "--format=value(email)",
+  ]);
+  if (!describe.ok) {
+    const create = await run([
+      "iam",
+      "service-accounts",
+      "create",
+      schedulerServiceAccountId(env),
+      "--display-name=Caelo scheduler tick",
+      "--description=Calls the admin's scheduler tick through IAP every few minutes; signs its own IAP credential, nothing else.",
+      project,
+    ]);
+    if (!create.ok && !/already exists/i.test(create.stderr)) return fail(`create ${sa}`, create);
+  }
+  done.push(`service account ${sa}`);
+
+  // 3. Its grants (see the file comment).
+  const self = await retry([
+    "iam",
+    "service-accounts",
+    "add-iam-policy-binding",
+    sa,
+    `--member=${member}`,
+    "--role=roles/iam.serviceAccountTokenCreator",
+    "--condition=None",
+    project,
+    "--quiet",
+    "--format=none",
+  ]);
+  if (!self.ok) return fail("let the scheduler sign its IAP credential as itself", self);
+  done.push(`scheduler: roles/iam.serviceAccountTokenCreator on ${sa} (itself, signJwt)`);
+
+  for (const g of [
+    { role: "roles/workflows.invoker", why: "Cloud Scheduler starts the tick workflow as it" },
+    { role: "roles/logging.logWriter", why: "the workflow logs failed calls" },
+  ]) {
+    const r = await retry([
+      "projects",
+      "add-iam-policy-binding",
+      projectId,
+      `--member=${member}`,
+      `--role=${g.role}`,
+      "--condition=None",
+      "--quiet",
+      "--format=none",
+    ]);
+    if (!r.ok) return fail(`grant the scheduler ${g.role} (${g.why})`, r);
+    done.push(`scheduler: ${g.role} on the project`);
+  }
+
+  const iap = await retry([
+    "iap",
+    "web",
+    "add-iam-policy-binding",
+    ...iapResourceArgs(target.resource),
+    `--member=${member}`,
+    "--role=roles/iap.httpsResourceAccessor",
+    "--condition=None",
+    project,
+    "--quiet",
+  ]);
+  if (!iap.ok) return fail("let the scheduler pass the admin's IAP", iap);
+  done.push(`scheduler: roles/iap.httpsResourceAccessor on the admin (${target.resource.service})`);
+
+  // 4. The workflow. A private directory (mkdtemp, mode 700): no other local
+  //    user can swap the file between write and read.
+  const dir = mkdtempSync(join(tmpdir(), "caelo-scheduler-"));
+  try {
+    const file = join(dir, "tick.yaml");
+    writeFileSync(file, schedulerWorkflowSource({ serviceAccount: sa, origin: origin.origin }), {
+      mode: 0o600,
+    });
+    const deploy = await retry([
+      "workflows",
+      "deploy",
+      workflow,
+      `--source=${file}`,
+      `--location=${region}`,
+      `--service-account=${sa}`,
+      "--call-log-level=log-errors-only",
+      `--description=Caelo scheduler tick (#1140): calls ${origin.origin}${SCHEDULER_TICK_PATH}`,
+      project,
+      "--quiet",
+    ]);
+    if (!deploy.ok) return fail(`deploy workflow ${workflow}`, deploy);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+  done.push(`workflow ${workflow} (calls ${origin.origin}${SCHEDULER_TICK_PATH})`);
+
+  // 5. The one Cloud Scheduler job.
+  const scheduleArgs = [
+    job,
+    `--location=${region}`,
+    project,
+    `--schedule=${cron}`,
+    "--time-zone=Etc/UTC",
+    `--uri=https://workflowexecutions.googleapis.com/v1/projects/${projectId}/locations/${region}/workflows/${workflow}/executions`,
+    "--http-method=POST",
+    `--oauth-service-account-email=${sa}`,
+    // A tick that has not answered when the next one is due is the admin's
+    // lease to sort out (its due-table), not a reason to retry this one.
+    "--max-retry-attempts=0",
+    "--quiet",
+  ];
+  const existing = await run([
+    "scheduler",
+    "jobs",
+    "describe",
+    job,
+    `--location=${region}`,
+    project,
+    "--format=value(name)",
+  ]);
+  const sched = await retry([
+    "scheduler",
+    "jobs",
+    existing.ok ? "update" : "create",
+    "http",
+    ...scheduleArgs,
+  ]);
+  if (!sched.ok) return fail(`schedule ${job}`, sched);
+  done.push(`Cloud Scheduler job ${job} (${cron}, UTC)`);
+
+  return { ok: true, done, serviceAccount: sa };
+}

package/src/self-hosted-backup.test.ts

@@ -203,6 +203,7 @@ describe("restoreSelfHosted", () => {
       "<converge roles>",
       `${MIGRATE} admin`,
       `${MIGRATE} public`,
+      `${MIGRATE} public-staging`,
       "run --rm --no-deps -T --entrypoint sh caelo-admin -c set -e; find /srv/caelo /data/media -mindepth 1 -delete; tar -C / -xf -",
       "up -d --remove-orphans",
       "exec -T caelo-admin bun -e <probe>",