@caelo-cms/provisioning 0.11.9 → 0.11.11

package/src/scheduler-setup.ts

@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #1140 — set up (and converge) the scheduler tick on a Google Cloud install
+ * (`gcp`, `gcp-firebase`): the ONE Cloud Scheduler job that calls the
+ * admin's `/api/internal/scheduler/tick` every `schedulerIntervalMinutes`
+ * (1–15, default 5). Each task's own cadence lives in the admin's
+ * due-table; the tick only says "run what is due now". Run by the wizard
+ * after migrations and by `cms-provision upgrade`'s converge (and the
+ * converge job), as the operator's or the provisioner's identity.
+ *
+ * Why not Cloud Scheduler → admin directly. The admin sits behind IAP, and
+ * IAP accepts a service account's OIDC ID token only when its `aud` is an
+ * OAuth client allowlisted for programmatic access; an install on the
+ * Google-managed IAP client has none (cloud.google.com/iap/docs/
+ * authentication-howto). Cloud Scheduler's other credential, an OAuth
+ * access token, it sends only to `*.googleapis.com` (cloud.google.com/
+ * scheduler/docs/http-target-auth). What passes IAP for a service account
+ * on every install is a self-signed service-account JWT with `aud` =
+ * `<admin origin>/*` — the path caelo-mcp and admin-internal-google.ts
+ * take. Cloud Scheduler cannot sign one, so the job starts a Cloud
+ * Workflows execution (a googleapis.com target, OAuth), and the workflow
+ * signs the JWT as the scheduler account through the IAM Credentials API
+ * and POSTs the tick. No container per tick and no image to keep in step
+ * with the app-roll; cents a month (wizards/gcp-cost.ts).
+ *
+ * Why the CLI owns it and not the stacks: the same as audit-job-setup.ts —
+ * `upgrade` never runs Pulumi, and existing installs get the tick from here.
+ *
+ * Trust boundary. The scheduler's identity, `caelo-<env>-scheduler`, holds:
+ *
+ *   - `roles/iam.serviceAccountTokenCreator` on ITSELF only (signJwt);
+ *   - `roles/iap.httpsResourceAccessor` on the admin's IAP resource (the
+ *     operator-access sync job keeps it there, operator-access.ts);
+ *   - `roles/workflows.invoker` on the project (Cloud Scheduler starts the
+ *     workflow as it; Workflows has no per-workflow IAM policy to bind to);
+ *   - `roles/logging.logWriter` (the workflow's error call logs).
+ *
+ * Nothing else: no Secret Manager, no Cloud SQL, no Cloud Run rights. What
+ * it can do with the admin is the admin's to decide: the tick endpoint
+ * accepts exactly this account's IAP identity (`CAELO_SCHEDULER_SERVICE_
+ * ACCOUNT` on the admin) and only ever runs the due-table, never a build
+ * or a publish (scripts/no-automatic-production-deploy.test.ts).
+ * {@link SCHEDULER_FORBIDDEN_ROLES} names that line; the tests assert no
+ * grant crosses it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
+import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
+import {
+  schedulerJobName,
+  schedulerServiceAccountEmail,
+  schedulerServiceAccountId,
+  schedulerWorkflowName,
+} from "./gcp-names.js";
+import { type IapResource, iapResourceArgs } from "./mcp-iap.js";
+import { schedulerCron } from "./stack-contract.js";
+
+/** The admin path the workflow calls (apps/admin `api/internal/scheduler/tick`). */
+export const SCHEDULER_TICK_PATH = "/api/internal/scheduler/tick";
+
+/**
+ * How long the workflow waits for the admin's answer, in seconds: Workflows'
+ * maximum (cloud.google.com/workflows/docs/reference/stdlib/http/post). The
+ * admin holds the request until the tick's work is done: it claims tasks for
+ * at most its budget (4 minutes, admin-core scheduler/tick.ts) and then waits
+ * at most one task's lease (10 minutes) for the last one, so 14 minutes in
+ * the worst case, inside the admin's own 3600 s request timeout.
+ */
+export const SCHEDULER_TICK_TIMEOUT_SECONDS = 1800;
+
+/** Lifetime of the IAP credential the workflow signs per tick. */
+const JWT_LIFETIME_SECONDS = 600;
+
+/** APIs the tick needs on the project. */
+export const SCHEDULER_APIS = [
+  "cloudscheduler.googleapis.com",
+  "workflows.googleapis.com",
+  "iamcredentials.googleapis.com",
+] as const;
+
+/**
+ * Roles the scheduler's identity must never hold anywhere. Prefix matches
+ * (`roles/secretmanager.` = every Secret Manager role).
+ */
+export const SCHEDULER_FORBIDDEN_ROLES = [
+  "roles/owner",
+  "roles/editor",
+  "roles/viewer",
+  "roles/run.",
+  "roles/iam.serviceAccountUser",
+  "roles/iam.serviceAccountAdmin",
+  "roles/iam.securityAdmin",
+  "roles/iap.admin",
+  "roles/iap.settingsAdmin",
+  "roles/secretmanager.",
+  "roles/cloudsql.",
+  "roles/compute.",
+  "roles/storage.",
+  "roles/resourcemanager.",
+  "roles/workflows.admin",
+  "roles/workflows.editor",
+  "roles/cloudscheduler.",
+] as const;
+
+export interface SchedulerTickTarget {
+  readonly projectId: string;
+  readonly region: string;
+  /** Pulumi stack name (`production`). */
+  readonly env: string;
+  /**
+   * Where the admin answers behind IAP, an origin: `https://admin.<domain>`
+   * on gcp, the admin service's URL on gcp-firebase (remote-mcp.ts
+   * RemoteMcpIdentity.adminUrl, the same value as the admin's
+   * CAELO_ADMIN_URL).
+   */
+  readonly adminUrl: string;
+  /** The admin's IAP resource (the service on gcp-firebase, the LB backend on gcp). */
+  readonly resource: IapResource;
+  /** The `schedulerIntervalMinutes` knob (1–15). */
+  readonly intervalMinutes: number;
+}
+
+export type SchedulerTickResult =
+  | { readonly ok: true; readonly done: string[]; readonly serviceAccount: string }
+  | { readonly ok: false; readonly done: string[]; readonly error: string };
+
+/**
+ * The tick's admin origin, or an error naming why it is not one: an https
+ * origin without path, query or credentials (it is written into the
+ * workflow's source, so nothing but an origin may get there).
+ */
+export function tickOrigin(
+  adminUrl: string,
+): { ok: true; origin: string } | { ok: false; error: string } {
+  let u: URL;
+  try {
+    u = new URL(adminUrl);
+  } catch {
+    return { ok: false, error: `the admin URL ${JSON.stringify(adminUrl)} is not a URL` };
+  }
+  if (
+    u.protocol !== "https:" ||
+    u.username ||
+    u.password ||
+    (u.pathname !== "/" && u.pathname !== "") ||
+    u.search ||
+    u.hash ||
+    !/^[a-z0-9.-]+$/.test(u.hostname)
+  ) {
+    return {
+      ok: false,
+      error: `the admin URL ${JSON.stringify(adminUrl)} is not a plain https origin`,
+    };
+  }
+  return { ok: true, origin: u.origin };
+}
+
+/**
+ * The workflow one tick runs (Cloud Workflows YAML): sign a service-account
+ * JWT for the admin's IAP as the scheduler account itself (the IAM
+ * Credentials API, authenticated with the workflow's own OAuth token), then
+ * POST the tick with it. A refused or failed tick (HTTP ≥ 400) fails the
+ * execution, so it shows in the workflow's execution history and logs.
+ */
+export function schedulerWorkflowSource(opts: {
+  readonly serviceAccount: string;
+  readonly origin: string;
+}): string {
+  const { serviceAccount: sa, origin } = opts;
+  return `# Caelo scheduler tick (#1140) — generated by cms-provision, do not edit.
+main:
+  steps:
+    - claims:
+        assign:
+          - now: \${int(sys.now())}
+          - jwtClaims:
+              iss: "${sa}"
+              sub: "${sa}"
+              aud: "${origin}/*"
+              iat: \${now}
+              exp: \${now + ${JWT_LIFETIME_SECONDS}}
+    - sign:
+        call: http.post
+        args:
+          url: "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${sa}:signJwt"
+          auth:
+            type: OAuth2
+          body:
+            payload: \${json.encode_to_string(jwtClaims)}
+        result: signed
+    - tick:
+        call: http.post
+        args:
+          url: "${origin}${SCHEDULER_TICK_PATH}"
+          headers:
+            Authorization: \${"Bearer " + signed.body.signedJwt}
+          timeout: ${SCHEDULER_TICK_TIMEOUT_SECONDS}
+        result: answer
+    - done:
+        return: \${answer.code}
+`;
+}
+
+/** Ensure the scheduler tick and everything it needs. Never throws for a gcloud failure. */
+export async function ensureSchedulerTick(
+  target: SchedulerTickTarget,
+  deps: { run?: GcloudRunner; sleep?: Sleep } = {},
+): Promise<SchedulerTickResult> {
+  const run = deps.run ?? defaultGcloud;
+  const sleep = deps.sleep ?? realSleep;
+  const retry = (args: string[]) => runWithRetry(run, sleep, args);
+  const { projectId, region, env } = target;
+  const project = `--project=${projectId}`;
+  const sa = schedulerServiceAccountEmail(projectId, env);
+  const member = `serviceAccount:${sa}`;
+  const workflow = schedulerWorkflowName(env);
+  const job = schedulerJobName(env);
+  const done: string[] = [];
+  const fail = (step: string, r: GcloudResult): SchedulerTickResult => ({
+    ok: false,
+    done,
+    error: `${step}: ${r.stderr.trim()}`,
+  });
+
+  const origin = tickOrigin(target.adminUrl);
+  if (!origin.ok) return { ok: false, done, error: origin.error };
+  let cron: string;
+  try {
+    cron = schedulerCron(target.intervalMinutes);
+  } catch (e) {
+    return { ok: false, done, error: (e as Error).message };
+  }
+
+  // 1. APIs.
+  const api = await run(["services", "enable", ...SCHEDULER_APIS, project, "--quiet"]);
+  if (!api.ok) return fail(`enable ${SCHEDULER_APIS.join(", ")}`, api);
+  done.push(`APIs enabled (${SCHEDULER_APIS.join(", ")})`);
+
+  // 2. The scheduler's own service account.
+  const describe = await run([
+    "iam",
+    "service-accounts",
+    "describe",
+    sa,
+    project,
+    "--format=value(email)",
+  ]);
+  if (!describe.ok) {
+    const create = await run([
+      "iam",
+      "service-accounts",
+      "create",
+      schedulerServiceAccountId(env),
+      "--display-name=Caelo scheduler tick",
+      "--description=Calls the admin's scheduler tick through IAP every few minutes; signs its own IAP credential, nothing else.",
+      project,
+    ]);
+    if (!create.ok && !/already exists/i.test(create.stderr)) return fail(`create ${sa}`, create);
+  }
+  done.push(`service account ${sa}`);
+
+  // 3. Its grants (see the file comment).
+  const self = await retry([
+    "iam",
+    "service-accounts",
+    "add-iam-policy-binding",
+    sa,
+    `--member=${member}`,
+    "--role=roles/iam.serviceAccountTokenCreator",
+    "--condition=None",
+    project,
+    "--quiet",
+    "--format=none",
+  ]);
+  if (!self.ok) return fail("let the scheduler sign its IAP credential as itself", self);
+  done.push(`scheduler: roles/iam.serviceAccountTokenCreator on ${sa} (itself, signJwt)`);
+
+  for (const g of [
+    { role: "roles/workflows.invoker", why: "Cloud Scheduler starts the tick workflow as it" },
+    { role: "roles/logging.logWriter", why: "the workflow logs failed calls" },
+  ]) {
+    const r = await retry([
+      "projects",
+      "add-iam-policy-binding",
+      projectId,
+      `--member=${member}`,
+      `--role=${g.role}`,
+      "--condition=None",
+      "--quiet",
+      "--format=none",
+    ]);
+    if (!r.ok) return fail(`grant the scheduler ${g.role} (${g.why})`, r);
+    done.push(`scheduler: ${g.role} on the project`);
+  }
+
+  const iap = await retry([
+    "iap",
+    "web",
+    "add-iam-policy-binding",
+    ...iapResourceArgs(target.resource),
+    `--member=${member}`,
+    "--role=roles/iap.httpsResourceAccessor",
+    "--condition=None",
+    project,
+    "--quiet",
+  ]);
+  if (!iap.ok) return fail("let the scheduler pass the admin's IAP", iap);
+  done.push(`scheduler: roles/iap.httpsResourceAccessor on the admin (${target.resource.service})`);
+
+  // 4. The workflow. A private directory (mkdtemp, mode 700): no other local
+  //    user can swap the file between write and read.
+  const dir = mkdtempSync(join(tmpdir(), "caelo-scheduler-"));
+  try {
+    const file = join(dir, "tick.yaml");
+    writeFileSync(file, schedulerWorkflowSource({ serviceAccount: sa, origin: origin.origin }), {
+      mode: 0o600,
+    });
+    const deploy = await retry([
+      "workflows",
+      "deploy",
+      workflow,
+      `--source=${file}`,
+      `--location=${region}`,
+      `--service-account=${sa}`,
+      "--call-log-level=log-errors-only",
+      `--description=Caelo scheduler tick (#1140): calls ${origin.origin}${SCHEDULER_TICK_PATH}`,
+      project,
+      "--quiet",
+    ]);
+    if (!deploy.ok) return fail(`deploy workflow ${workflow}`, deploy);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+  done.push(`workflow ${workflow} (calls ${origin.origin}${SCHEDULER_TICK_PATH})`);
+
+  // 5. The one Cloud Scheduler job.
+  const scheduleArgs = [
+    job,
+    `--location=${region}`,
+    project,
+    `--schedule=${cron}`,
+    "--time-zone=Etc/UTC",
+    `--uri=https://workflowexecutions.googleapis.com/v1/projects/${projectId}/locations/${region}/workflows/${workflow}/executions`,
+    "--http-method=POST",
+    `--oauth-service-account-email=${sa}`,
+    // A tick that has not answered when the next one is due is the admin's
+    // lease to sort out (its due-table), not a reason to retry this one.
+    "--max-retry-attempts=0",
+    "--quiet",
+  ];
+  const existing = await run([
+    "scheduler",
+    "jobs",
+    "describe",
+    job,
+    `--location=${region}`,
+    project,
+    "--format=value(name)",
+  ]);
+  const sched = await retry([
+    "scheduler",
+    "jobs",
+    existing.ok ? "update" : "create",
+    "http",
+    ...scheduleArgs,
+  ]);
+  if (!sched.ok) return fail(`schedule ${job}`, sched);
+  done.push(`Cloud Scheduler job ${job} (${cron}, UTC)`);
+
+  return { ok: true, done, serviceAccount: sa };
+}

package/src/seal-key-rotation.test.ts

@@ -15,7 +15,7 @@ const T = {
   projectId: "acme",
   region: "europe-west1",
   env: "production",
-  gatewayService: "caelo-production-gateway-abc",
+  proxyService: "caelo-production-gateway-abc",
 };
 const DAY = 24 * 60 * 60 * 1000;
 const NOW = Date.UTC(2026, 9, 20);
@@ -122,7 +122,7 @@ describe("seal-key rotation (#806)", () => {
 
   it("keeps the window equal to the broker's refresh-token lifetime", async () => {
     const src = await Bun.file(
-      new URL("../../../apps/api-gateway/src/mcp/token.ts", import.meta.url),
+      new URL("../../../apps/mcp-auth-proxy/src/token.ts", import.meta.url),
     ).text();
     expect(src).toContain(
       `REFRESH_MAX_LIFETIME_SECONDS = ${SEAL_KEY_ROTATION_WINDOW_DAYS} * 24 * 60 * 60`,

package/src/seal-key-rotation.ts

@@ -5,7 +5,7 @@
  * every connected MCP client out.
  *
  * The broker seals its OAuth state, codes and refresh tokens under
- * `CAELO_MCP_OAUTH_SEAL_KEY` (api-gateway mcp/seal.ts, envelope v2 with a
+ * `CAELO_MCP_OAUTH_SEAL_KEY` (apps/mcp-auth-proxy seal.ts, envelope v2 with a
  * key id). While `CAELO_MCP_OAUTH_SEAL_KEY_PREVIOUS` is set it also opens
  * values sealed under that key, and seals everything new under the current
  * one. Refresh tokens live at most {@link SEAL_KEY_ROTATION_WINDOW_DAYS}
@@ -14,7 +14,7 @@
  *
  * Two steps, both `cms-provision rotate-secret mcp-oauth-seal-key`:
  *
- *   start   — add a new version to the secret, then roll the gateway with
+ *   start   — add a new version to the secret, then roll the MCP-Auth-Proxy with
  *             the current key at `latest` and the previous key PINNED to
  *             the version that was latest before (a pinned version never
  *             follows the new one), labelled with the start time.
@@ -27,7 +27,7 @@
  * progress (a second start would pin the CURRENT key as "previous" and drop
  * the real previous one early), and finishing before the window ends.
  *
- * A `cms-provision upgrade` whose `pulumi up` rewrites the gateway template
+ * A `cms-provision upgrade` whose `pulumi up` rewrites the MCP-Auth-Proxy template
  * during the window drops the previous key (the stack does not declare it):
  * clients sealed under it sign in again. That fails safe — no token lives
  * longer than intended — and is documented in the rotate output.
@@ -39,27 +39,27 @@ import { gcpSecretId } from "./gcp-names.js";
 import { initialSecretValue } from "./runtime-secrets.js";
 import { liveContainerEnv } from "./stack-converge.js";
 
-/** The gateway env var holding the previous key during a rotation. */
+/** The MCP-Auth-Proxy env var holding the previous key during a rotation. */
 export const SEAL_KEY_PREVIOUS_ENV = "CAELO_MCP_OAUTH_SEAL_KEY_PREVIOUS";
-/** The gateway env var holding the current key (stack-contract SERVICE_SECRET_ENV). */
+/** The MCP-Auth-Proxy env var holding the current key (stack-contract SERVICE_SECRET_ENV). */
 export const SEAL_KEY_ENV = "CAELO_MCP_OAUTH_SEAL_KEY";
-/** Label on the gateway service: when the rotation started (epoch ms). */
+/** Label on the MCP-Auth-Proxy service: when the rotation started (epoch ms). */
 export const SEAL_ROTATION_LABEL = "caelo-seal-rotation-started";
 /**
  * Days the previous key must stay: the broker's refresh-token lifetime
- * (api-gateway mcp/token.ts REFRESH_MAX_LIFETIME_SECONDS, 30 days).
+ * (apps/mcp-auth-proxy token.ts REFRESH_MAX_LIFETIME_SECONDS, 30 days).
  */
 export const SEAL_KEY_ROTATION_WINDOW_DAYS = 30;
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 
-/** The deployed gateway a rotation works on. */
+/** The deployed MCP-Auth-Proxy a rotation works on. */
 export interface SealRotationTarget {
   readonly projectId: string;
   readonly region: string;
   readonly env: string;
-  /** Deployed (Pulumi-suffixed) gateway Cloud Run service name. */
-  readonly gatewayService: string;
+  /** Deployed (Pulumi-suffixed or fixed) MCP-Auth-Proxy Cloud Run service name. */
+  readonly proxyService: string;
 }
 
 export interface SealRotationReport {
@@ -80,7 +80,7 @@ interface GatewayState {
   readonly startedAtMs: number | null;
 }
 
-async function gatewayState(
+async function proxyState(
   run: GcloudRunner,
   t: SealRotationTarget,
 ): Promise<{ ok: true; state: GatewayState } | { ok: false; error: string }> {
@@ -88,13 +88,13 @@ async function gatewayState(
     "run",
     "services",
     "describe",
-    t.gatewayService,
+    t.proxyService,
     "--region",
     t.region,
     `--project=${t.projectId}`,
     "--format=json",
   ]);
-  if (!r.ok) return { ok: false, error: `read the gateway service: ${r.stderr.trim()}` };
+  if (!r.ok) return { ok: false, error: `read the MCP-Auth-Proxy service: ${r.stderr.trim()}` };
   const env = liveContainerEnv(r.stdout);
   const labels =
     (JSON.parse(r.stdout) as { metadata?: { labels?: Record<string, string> } }).metadata?.labels ??
@@ -105,7 +105,7 @@ async function gatewayState(
   if (current?.kind !== "secret") {
     return {
       ok: false,
-      error: `${SEAL_KEY_ENV} on ${t.gatewayService} is not a Secret Manager reference — run \`cms-provision upgrade\` first (it installs remote MCP's secrets).`,
+      error: `${SEAL_KEY_ENV} on ${t.proxyService} is not a Secret Manager reference — run \`cms-provision upgrade\` first (it installs remote MCP's secrets).`,
     };
   }
   return {
@@ -119,7 +119,7 @@ function rollFlip(t: SealRotationTarget): string[] {
     "run",
     "services",
     "update-traffic",
-    t.gatewayService,
+    t.proxyService,
     "--region",
     t.region,
     `--project=${t.projectId}`,
@@ -140,7 +140,7 @@ export async function startSealKeyRotation(
   const project = `--project=${t.projectId}`;
   const steps: string[] = [];
 
-  const live = await gatewayState(run, t);
+  const live = await proxyState(run, t);
   if (!live.ok) return { ok: false, steps, error: live.error };
   if (live.state.previousPinned) {
     const since = live.state.startedAtMs;
@@ -181,7 +181,7 @@ export async function startSealKeyRotation(
     "run",
     "services",
     "update",
-    t.gatewayService,
+    t.proxyService,
     "--region",
     t.region,
     project,
@@ -195,11 +195,11 @@ export async function startSealKeyRotation(
     return {
       ok: false,
       steps,
-      error: `The new key is stored but the gateway does not use it yet. Do not rotate again — roll the gateway:\n    gcloud ${update.join(" ")}\n    gcloud ${rollFlip(t).join(" ")}\n(${flipped.stderr.trim()})`,
+      error: `The new key is stored but the MCP-Auth-Proxy does not use it yet. Do not rotate again — roll the MCP-Auth-Proxy:\n    gcloud ${update.join(" ")}\n    gcloud ${rollFlip(t).join(" ")}\n(${flipped.stderr.trim()})`,
     };
   }
   steps.push(
-    `rolled the gateway onto the new key; tokens sealed under version ${version} keep working`,
+    `rolled the MCP-Auth-Proxy onto the new key; tokens sealed under version ${version} keep working`,
     `finish after ${new Date(now + SEAL_KEY_ROTATION_WINDOW_DAYS * DAY_MS).toISOString()} with: cms-provision rotate-secret mcp-oauth-seal-key --finish`,
     "a `cms-provision upgrade` before then drops the previous key early: clients sealed under it sign in again",
   );
@@ -215,13 +215,13 @@ export async function finishSealKeyRotation(
   const run = deps.run ?? defaultGcloud;
   const now = (deps.nowMs ?? Date.now)();
   const steps: string[] = [];
-  const live = await gatewayState(run, t);
+  const live = await proxyState(run, t);
   if (!live.ok) return { ok: false, steps, error: live.error };
   if (!live.state.previousPinned) {
     return {
       ok: false,
       steps,
-      error: `No seal-key rotation is in progress on ${t.gatewayService} (${SEAL_KEY_PREVIOUS_ENV} is not set). Start one with \`cms-provision rotate-secret mcp-oauth-seal-key\`.`,
+      error: `No seal-key rotation is in progress on ${t.proxyService} (${SEAL_KEY_PREVIOUS_ENV} is not set). Start one with \`cms-provision rotate-secret mcp-oauth-seal-key\`.`,
     };
   }
   const started = live.state.startedAtMs;
@@ -232,7 +232,7 @@ export async function finishSealKeyRotation(
       steps,
       error:
         due === null
-          ? `The gateway has ${SEAL_KEY_PREVIOUS_ENV} but no ${SEAL_ROTATION_LABEL} label, so the rotation's start is unknown. Wait ${SEAL_KEY_ROTATION_WINDOW_DAYS} days from when it started, then run --finish --now.`
+          ? `The MCP-Auth-Proxy has ${SEAL_KEY_PREVIOUS_ENV} but no ${SEAL_ROTATION_LABEL} label, so the rotation's start is unknown. Wait ${SEAL_KEY_ROTATION_WINDOW_DAYS} days from when it started, then run --finish --now.`
           : `The previous key still opens refresh tokens until ${new Date(due).toISOString()} (${SEAL_KEY_ROTATION_WINDOW_DAYS}-day lifetime). Finishing now would sign those clients out; run --finish after that time, or --finish --now if the old key leaked.`,
     };
   }
@@ -240,7 +240,7 @@ export async function finishSealKeyRotation(
     "run",
     "services",
     "update",
-    t.gatewayService,
+    t.proxyService,
     "--region",
     t.region,
     `--project=${t.projectId}`,
@@ -251,8 +251,8 @@ export async function finishSealKeyRotation(
   const rolled = await run(update);
   const flipped = rolled.ok ? await run(rollFlip(t)) : rolled;
   if (!flipped.ok) {
-    return { ok: false, steps, error: `roll the gateway: ${flipped.stderr.trim()}` };
+    return { ok: false, steps, error: `roll the MCP-Auth-Proxy: ${flipped.stderr.trim()}` };
   }
-  steps.push("removed the previous seal key from the gateway; the rotation is complete");
+  steps.push("removed the previous seal key from the MCP-Auth-Proxy; the rotation is complete");
   return { ok: true, steps };
 }

package/src/self-hosted-backup.test.ts

@@ -203,6 +203,7 @@ describe("restoreSelfHosted", () => {
       "<converge roles>",
       `${MIGRATE} admin`,
       `${MIGRATE} public`,
+      `${MIGRATE} public-staging`,
       "run --rm --no-deps -T --entrypoint sh caelo-admin -c set -e; find /srv/caelo /data/media -mindepth 1 -delete; tar -C / -xf -",
       "up -d --remove-orphans",
       "exec -T caelo-admin bun -e <probe>",