@caelo-cms/provisioning 0.10.47 → 0.10.48

package/src/dns/manual.ts

@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Manual DNS adapter — fallback for registrars without an API
+ * integration (Namecheap, GoDaddy, Porkbun, etc.). Prints the records,
+ * waits for the operator to paste them at the registrar, polls public
+ * DNS until resolution succeeds.
+ *
+ * Per CLAUDE.md §11.C: this IS one of the few human-required steps
+ * the wizard can't skip. The polling makes it close to zero-friction —
+ * the operator pastes records, presses Enter, the wizard waits.
+ */
+
+import { promises as dns } from "node:dns";
+import { setTimeout as sleep } from "node:timers/promises";
+import { confirm, isCancel, log, note, spinner } from "@clack/prompts";
+import { bold, cyan, dim, green, red } from "kleur/colors";
+import type { DnsAdapter, DnsRecord } from "./types.js";
+
+const POLL_INTERVAL_MS = 5000;
+const POLL_TIMEOUT_MS = 30 * 60 * 1000; // 30 min — managed certs need ~15 min to issue after DNS
+/**
+ * Without a prompt nobody confirms the records were set, so the install job
+ * (#696) waits longer for the operator to reach their registrar; a timeout
+ * ends the run and the next one resumes at this step.
+ */
+const NON_INTERACTIVE_POLL_TIMEOUT_MS = 2 * 60 * 60 * 1000;
+
+/**
+ * @param opts.nonInteractive print the records and poll without asking for
+ *   confirmation first (the install job has no terminal)
+ */
+export function makeManualAdapter(opts: { domain: string; nonInteractive?: boolean }): DnsAdapter {
+  const timeoutMs = opts.nonInteractive ? NON_INTERACTIVE_POLL_TIMEOUT_MS : POLL_TIMEOUT_MS;
+  const timeoutLabel = `${Math.round(timeoutMs / 60_000)} min`;
+  return {
+    name: `Manual (paste at your registrar for ${opts.domain})`,
+    async applyRecords(records: DnsRecord[]): Promise<void> {
+      note(
+        [
+          bold("Paste these DNS records at your registrar:"),
+          "",
+          ...records.map(
+            (r) => `  ${dim(r.type.padEnd(6))} ${r.hostname.padEnd(35)} → ${bold(r.value)}`,
+          ),
+          "",
+          dim(
+            opts.nonInteractive
+              ? "Waiting until they resolve — the install continues on its own."
+              : "After pasting, press Enter to verify resolution.",
+          ),
+        ].join("\n"),
+        "DNS records",
+      );
+
+      if (!opts.nonInteractive) {
+        const ack = await confirm({
+          message: "Records pasted at your registrar?",
+          initialValue: true,
+        });
+        if (isCancel(ack) || !ack) {
+          log.error(red("Aborted at DNS step."));
+          throw new Error("DNS records not pasted; aborted by operator.");
+        }
+      }
+
+      const s = spinner();
+      s.start(`Polling DNS — up to ${timeoutLabel} while propagation + records settle...`);
+      const start = Date.now();
+      const remaining = new Set(records.map((r) => recordKey(r)));
+
+      while (remaining.size > 0) {
+        if (Date.now() - start > timeoutMs) {
+          s.stop(red(`DNS verify timeout (${timeoutLabel}).`));
+          throw new Error(
+            `Records still unresolved after ${timeoutLabel}: ${[...remaining].join(", ")}. Re-run after fixing.`,
+          );
+        }
+        for (const key of [...remaining]) {
+          const record = records.find((r) => recordKey(r) === key);
+          if (!record) {
+            remaining.delete(key);
+            continue;
+          }
+          if (await checkRecord(record)) {
+            remaining.delete(key);
+            log.success(
+              green(`✓ ${record.type} ${record.hostname} resolves to expected ${record.value}`),
+            );
+          }
+        }
+        if (remaining.size > 0) await sleep(POLL_INTERVAL_MS);
+      }
+      s.stop(green("All DNS records resolve."));
+      void cyan;
+    },
+  };
+}
+
+function recordKey(r: DnsRecord): string {
+  return `${r.type}:${r.hostname}`;
+}
+
+async function checkRecord(record: DnsRecord): Promise<boolean> {
+  try {
+    if (record.type === "A") {
+      const result = await dns.resolve4(record.hostname);
+      return result.includes(record.value);
+    }
+    if (record.type === "AAAA") {
+      const result = await dns.resolve6(record.hostname);
+      return result.includes(record.value);
+    }
+    if (record.type === "CNAME") {
+      const result = await dns.resolveCname(record.hostname);
+      // CNAME values often have a trailing dot; normalize
+      const expected = record.value.replace(/\.$/, "");
+      return result.some((v) => v.replace(/\.$/, "") === expected);
+    }
+    if (record.type === "TXT") {
+      const result = await dns.resolveTxt(record.hostname);
+      const expected = record.value.replace(/^"|"$/g, "");
+      return result.some((arr) => arr.join("") === expected);
+    }
+    return false;
+  } catch {
+    // ENOTFOUND / SERVFAIL — record not propagated yet
+    return false;
+  }
+}

package/src/dns/types.ts

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Shared DNS-adapter interface. Every adapter (Cloudflare / Route53 /
+ * Cloud DNS / manual) implements `applyRecords` end-to-end: takes the
+ * desired records, makes them real (via API or operator paste), and
+ * resolves only when public DNS lookups succeed.
+ */
+
+export interface DnsRecord {
+  hostname: string;
+  type: "A" | "AAAA" | "CNAME" | "TXT";
+  value: string;
+  ttl?: number;
+}
+
+export interface DnsAdapter {
+  /** Human-readable name shown in wizard output. */
+  readonly name: string;
+
+  /**
+   * Make the records exist + verify they resolve. Resolves when
+   * `dig <hostname> <type>` returns the expected value globally.
+   * Throws on terminal failure (operator should fix + re-run).
+   */
+  applyRecords(records: DnsRecord[]): Promise<void>;
+}

package/src/gateway-credentials.test.ts

@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: MPL-2.0
+
+import { describe, expect, it } from "bun:test";
+import {
+  ensureGatewayRolePassword,
+  findSqlInstance,
+  gatewayMovesPublicRole,
+  planPublicRoleSwitch,
+} from "./gateway-credentials.js";
+import type { GcloudResult } from "./gcloud.js";
+import type { HttpFetch } from "./runtime-secrets.js";
+import type { LiveEnvValue } from "./stack-converge.js";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+/** Fake gcloud: secret values by secret id, a token, everything else ok. */
+function fakeGcloud(secrets: Record<string, string>) {
+  const lines: string[] = [];
+  const run = async (args: string[]) => {
+    const line = args.join(" ");
+    lines.push(line);
+    if (line.startsWith("auth print-access-token")) return ok("tok\n");
+    const secret = /--secret=caelo-production-([a-z-]+)/.exec(line)?.[1];
+    if (line.startsWith("secrets versions access") && secret) {
+      const value = secrets[secret];
+      return value === undefined ? fail(`NOT_FOUND: ${secret}`) : ok(value);
+    }
+    if (line.startsWith("sql instances list")) return ok("caelo-production-pg-1a2b3c\n");
+    return ok();
+  };
+  return { run, lines };
+}
+
+/** Fake Cloud SQL Admin API recording `role password` per user update. */
+function fakeSqlApi(failRoles: string[] = []) {
+  const set: string[] = [];
+  const http: HttpFetch = async (url, init) => {
+    const role = new URL(url).searchParams.get("name") ?? "?";
+    if (init.method === "PUT") {
+      if (failRoles.includes(role)) {
+        return { ok: false, status: 403, text: async () => "PERMISSION_DENIED" };
+      }
+      set.push(`${role} ${(JSON.parse(init.body ?? "{}") as { password: string }).password}`);
+    }
+    return { ok: true, status: 200, text: async () => JSON.stringify({ status: "DONE" }) };
+  };
+  return { http, set };
+}
+
+const target = { projectId: "acme", env: "production", sqlInstance: "caelo-production-pg-1" };
+const SECRETS = {
+  "postgres-password": "admin-pw",
+  "public-role-password": "public-pw",
+  "gateway-role-password": "gateway-pw",
+};
+const secretRef = (secret: string): LiveEnvValue => ({
+  kind: "secret",
+  secret: `caelo-production-${secret}`,
+  version: "latest",
+});
+/** The gateway as #579 left it: public_role on admin_role's password. */
+const BEFORE_613 = new Map<string, LiveEnvValue>([
+  ["PUBLIC_DATABASE_PASSWORD", secretRef("postgres-password")],
+  ["ADMIN_DATABASE_PASSWORD", secretRef("postgres-password")],
+]);
+const AFTER_613 = new Map<string, LiveEnvValue>([
+  ["PUBLIC_DATABASE_PASSWORD", secretRef("public-role-password")],
+  ["GATEWAY_DATABASE_PASSWORD", secretRef("gateway-role-password")],
+]);
+
+describe("gatewayMovesPublicRole", () => {
+  it("is true until the gateway reads public_role's own secret", () => {
+    expect(gatewayMovesPublicRole(BEFORE_613, "production")).toBe(true);
+    // v0.10.29: the password inline in a plain URL.
+    expect(
+      gatewayMovesPublicRole(
+        new Map([
+          ["PUBLIC_DATABASE_URL", { kind: "value", value: "postgres://public_role:x@h/db" }],
+        ]),
+        "production",
+      ),
+    ).toBe(true);
+    expect(gatewayMovesPublicRole(AFTER_613, "production")).toBe(false);
+  });
+});
+
+describe("ensureGatewayRolePassword", () => {
+  it("sets gateway_role from its secret through the API, never on argv", async () => {
+    const { run, lines } = fakeGcloud(SECRETS);
+    const sql = fakeSqlApi();
+    const r = await ensureGatewayRolePassword(target, {
+      run,
+      http: sql.http,
+      sleep: async () => {},
+    });
+    expect(r.status).toBe("applied");
+    expect(sql.set).toEqual(["gateway_role gateway-pw"]);
+    expect(lines.some((l) => l.includes("gateway-pw"))).toBe(false);
+  });
+
+  it("fails, naming the secret, when the secret can't be read", async () => {
+    const { run } = fakeGcloud({});
+    const r = await ensureGatewayRolePassword(target, {
+      run,
+      http: fakeSqlApi().http,
+      sleep: async () => {},
+    });
+    expect(r.status).toBe("failed");
+    expect(r.error).toContain("caelo-production-gateway-role-password");
+  });
+});
+
+describe("planPublicRoleSwitch", () => {
+  it("first upgrade: moves public_role onto its own password before the gateway rolls", async () => {
+    const { run } = fakeGcloud(SECRETS);
+    const sql = fakeSqlApi();
+    const sw = planPublicRoleSwitch(target, BEFORE_613, {
+      run,
+      http: sql.http,
+      sleep: async () => {},
+    });
+    expect(sw.moves).toBe(true);
+    expect((await sw.beforeGatewayRoll()).status).toBe("applied");
+    expect(sql.set).toEqual(["public_role public-pw"]);
+  });
+
+  it("a failed gateway roll puts public_role back on the password the old revision reads", async () => {
+    const { run } = fakeGcloud(SECRETS);
+    const sql = fakeSqlApi();
+    const sw = planPublicRoleSwitch(target, BEFORE_613, {
+      run,
+      http: sql.http,
+      sleep: async () => {},
+    });
+    await sw.beforeGatewayRoll();
+    expect(await sw.afterGatewayRollFailed()).toContain("set back");
+    expect(sql.set).toEqual(["public_role public-pw", "public_role admin-pw"]);
+  });
+
+  it("later upgrades re-apply the same value and have nothing to undo", async () => {
+    const { run } = fakeGcloud(SECRETS);
+    const sql = fakeSqlApi();
+    const sw = planPublicRoleSwitch(target, AFTER_613, {
+      run,
+      http: sql.http,
+      sleep: async () => {},
+    });
+    expect(sw.moves).toBe(false);
+    await sw.beforeGatewayRoll();
+    expect(await sw.afterGatewayRollFailed()).toBe("public_role was not changed");
+    expect(sql.set).toEqual(["public_role public-pw"]);
+  });
+
+  it("says so loudly when public_role can't be set back", async () => {
+    const { run } = fakeGcloud(SECRETS);
+    let calls = 0;
+    const http: HttpFetch = async (_url, init) => {
+      if (init.method === "PUT" && ++calls === 2) {
+        return { ok: false, status: 403, text: async () => "PERMISSION_DENIED" };
+      }
+      return { ok: true, status: 200, text: async () => JSON.stringify({ status: "DONE" }) };
+    };
+    const sw = planPublicRoleSwitch(target, BEFORE_613, { run, http, sleep: async () => {} });
+    await sw.beforeGatewayRoll();
+    expect(await sw.afterGatewayRollFailed()).toMatch(/could NOT set public_role back.*by hand/);
+  });
+});
+
+describe("findSqlInstance", () => {
+  it("finds the Pulumi-suffixed instance by the stack's prefix", async () => {
+    const { run, lines } = fakeGcloud({});
+    expect(await findSqlInstance({ projectId: "acme", env: "production" }, { run })).toBe(
+      "caelo-production-pg-1a2b3c",
+    );
+    expect(lines).toContain(
+      "sql instances list --project=acme --filter=name~^caelo-production-pg --format=value(name)",
+    );
+  });
+});

package/src/gateway-credentials.ts

@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Moving an existing install's API gateway off admin_role (issue #613),
+ * as `cms-provision upgrade` does it. The env side — GATEWAY_DATABASE_URL
+ * and the per-role password secrets in, ADMIN_DATABASE_* out — rides the
+ * gateway's roll (stack-contract.ts). The database side is ordered around
+ * that roll so /api/* keeps working:
+ *
+ *   1. after migrations (which create `gateway_role` without a password):
+ *      `gateway_role` gets its password. Nothing uses the role yet, so this
+ *      is safe at any point before the gateway rolls.
+ *   2. right before the gateway rolls: `public_role` moves off admin_role's
+ *      password onto its own. Only the gateway logs in as public_role, so
+ *      the window in which the old revision cannot open NEW public
+ *      connections is the roll itself (open connections stay up).
+ *   3. if the gateway's roll fails and it is rolled back: `public_role`
+ *      goes back to the password the prior revision reads, so the rollback
+ *      leaves a working gateway, not a locked-out one.
+ *
+ * On every later upgrade the gateway already reads public_role's own
+ * secret: step 2 then re-applies the same value (a no-op for clients) and
+ * step 3 has nothing to undo.
+ */
+
+import { gcloud as defaultGcloud } from "./gcloud.js";
+import type { GcloudRunner, Sleep } from "./gcloud-retry.js";
+import { gcpNamePrefix, gcpSecretId } from "./gcp-names.js";
+import {
+  type DatabaseRoleTarget,
+  type EnsureOutcome,
+  ensureDatabaseRolePassword,
+  type HttpFetch,
+  readSecretValue,
+} from "./runtime-secrets.js";
+import { DATABASE_ROLE_SECRET } from "./stack-contract.js";
+import type { LiveEnvValue } from "./stack-converge.js";
+
+/**
+ * Whether this roll moves `public_role` onto its own password: true while
+ * the live gateway does not yet read PUBLIC_DATABASE_PASSWORD from the
+ * `public-role-password` secret (an install from before #613, where it was
+ * admin_role's `postgres-password` or inline in the URL).
+ */
+export function gatewayMovesPublicRole(
+  liveGatewayEnv: ReadonlyMap<string, LiveEnvValue>,
+  env: string,
+): boolean {
+  const live = liveGatewayEnv.get("PUBLIC_DATABASE_PASSWORD");
+  const own = gcpSecretId(env, DATABASE_ROLE_SECRET.public_role);
+  return !(live?.kind === "secret" && (live.secret === own || live.secret.endsWith(`/${own}`)));
+}
+
+export interface GatewayCredentialDeps {
+  readonly run?: GcloudRunner;
+  readonly sleep?: Sleep;
+  readonly http?: HttpFetch;
+}
+
+/** Step 1: `gateway_role`'s password from its secret. */
+export function ensureGatewayRolePassword(
+  target: DatabaseRoleTarget,
+  deps: GatewayCredentialDeps = {},
+): Promise<EnsureOutcome> {
+  return ensureDatabaseRolePassword(target, "gateway_role", deps);
+}
+
+/** Steps 2 and 3 for one roll of the gateway. */
+export interface PublicRoleSwitch {
+  /** Whether this roll moves public_role off admin_role's password. */
+  readonly moves: boolean;
+  /** Step 2 — run right before the gateway's `services update`. */
+  beforeGatewayRoll(): Promise<EnsureOutcome>;
+  /**
+   * Step 3 — run when the gateway's roll failed and it goes back to its
+   * prior revision. Returns what happened, for the operator.
+   */
+  afterGatewayRollFailed(): Promise<string>;
+}
+
+/** Plan steps 2 and 3 from the gateway's live env (read before the roll). */
+export function planPublicRoleSwitch(
+  target: DatabaseRoleTarget,
+  liveGatewayEnv: ReadonlyMap<string, LiveEnvValue>,
+  deps: GatewayCredentialDeps = {},
+): PublicRoleSwitch {
+  const moves = gatewayMovesPublicRole(liveGatewayEnv, target.env);
+  let switched = false;
+  return {
+    moves,
+    async beforeGatewayRoll() {
+      const outcome = await ensureDatabaseRolePassword(target, "public_role", deps);
+      switched = moves && outcome.status === "applied";
+      return outcome;
+    },
+    async afterGatewayRollFailed() {
+      if (!switched) return "public_role was not changed";
+      // The prior revision reads admin_role's password for public_role.
+      const prior = await readSecretValue(
+        deps.run ?? defaultGcloud,
+        target,
+        DATABASE_ROLE_SECRET.admin_role,
+      );
+      if (!prior.ok) {
+        return `could NOT set public_role back for the rolled-back gateway (${prior.error}); set its password to the latest version of ${gcpSecretId(target.env, "postgres-password")} by hand`;
+      }
+      const restored = await ensureDatabaseRolePassword(target, "public_role", {
+        ...deps,
+        password: prior.value,
+      });
+      if (restored.status !== "applied") {
+        return `could NOT set public_role back for the rolled-back gateway (${restored.error ?? "?"}); set its password to the latest version of ${gcpSecretId(target.env, "postgres-password")} by hand`;
+      }
+      switched = false;
+      return "public_role was set back to the password the rolled-back gateway uses";
+    },
+  };
+}
+
+/**
+ * The install's Cloud SQL instance (Pulumi suffixes its name), or null when
+ * none or the listing fails.
+ */
+export async function findSqlInstance(
+  install: { readonly projectId: string; readonly env: string },
+  deps: { readonly run?: GcloudRunner } = {},
+): Promise<string | null> {
+  const run = deps.run ?? defaultGcloud;
+  const r = await run([
+    "sql",
+    "instances",
+    "list",
+    `--project=${install.projectId}`,
+    `--filter=name~^${gcpNamePrefix(install.env)}-pg`,
+    "--format=value(name)",
+  ]);
+  if (!r.ok) return null;
+  return (
+    r.stdout
+      .split("\n")
+      .map((l) => l.trim())
+      .find((l) => l.length > 0) ?? null
+  );
+}

package/src/gcloud-components.test.ts

@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: MPL-2.0
+
+import { describe, expect, it } from "bun:test";
+import type { GcloudResult } from "./gcloud.js";
+import { ensureGcloudComponent } from "./gcloud-components.js";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+function fake(list: GcloudResult, install: GcloudResult = ok()) {
+  const calls: string[][] = [];
+  const logs: string[] = [];
+  const run = async (args: string[]) => {
+    calls.push(args);
+    return args[1] === "list" ? list : install;
+  };
+  return { calls, logs, opts: { run, log: (m: string) => logs.push(m) } };
+}
+
+const LIST_ARGS = [
+  "components",
+  "list",
+  "--only-local-state",
+  "--filter=id=beta",
+  "--format=value(id)",
+];
+
+describe("ensureGcloudComponent", () => {
+  it("does nothing when the component is installed", async () => {
+    const f = fake(ok("beta\n"));
+    expect(await ensureGcloudComponent("beta", f.opts)).toEqual({ ok: true, installed: false });
+    expect(f.calls).toEqual([LIST_ARGS]);
+    expect(f.logs).toEqual([]);
+  });
+
+  it("installs a missing component non-interactively and says so", async () => {
+    const f = fake(ok(""));
+    expect(await ensureGcloudComponent("beta", f.opts)).toEqual({ ok: true, installed: true });
+    expect(f.calls).toEqual([LIST_ARGS, ["components", "install", "beta", "--quiet"]]);
+    expect(f.logs).toHaveLength(1);
+    expect(f.logs[0]).toContain('"beta"');
+  });
+
+  it("names the package-manager command when the component manager is disabled", async () => {
+    const f = fake(
+      ok(""),
+      fail(
+        "ERROR: (gcloud.components.install) You cannot perform this action because this Google Cloud CLI installation is managed by an external package manager.",
+      ),
+    );
+    const r = await ensureGcloudComponent("beta", f.opts);
+    expect(r.ok).toBe(false);
+    const error = r.ok ? "" : r.error;
+    expect(error).toContain("sudo apt-get install google-cloud-cli");
+    expect(error).toContain("managed by an external package manager");
+  });
+
+  it("tells the operator to install it themselves when the install fails for another reason", async () => {
+    const f = fake(ok(""), fail("ERROR: Permission denied: /opt/google-cloud-sdk"));
+    const r = await ensureGcloudComponent("alpha", f.opts);
+    expect(r.ok).toBe(false);
+    const error = r.ok ? "" : r.error;
+    expect(error).toContain("Permission denied");
+    expect(error).toContain("gcloud components install alpha");
+  });
+
+  it("fails instead of guessing when gcloud cannot list components", async () => {
+    const f = fake(fail("gcloud crashed"));
+    const r = await ensureGcloudComponent("beta", f.opts);
+    expect(r).toEqual({
+      ok: false,
+      error: 'check whether the gcloud "beta" component is installed: gcloud crashed',
+    });
+    expect(f.calls).toHaveLength(1);
+  });
+});

package/src/gcloud-components.ts

@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Ensure an optional gcloud component (`beta`, `alpha`, …) is installed
+ * BEFORE a command that needs it runs.
+ *
+ * Why this exists: invoking `gcloud beta …` without the `beta` component
+ * makes gcloud ask "Do you want to continue (Y/n)?" to install it. Our
+ * runner captures gcloud's output, so that prompt was invisible and the CLI
+ * hung forever (`admin-domain enable`). The runner now sets
+ * `CLOUDSDK_CORE_DISABLE_PROMPTS=1`, which turns the hang into a fast
+ * failure; this helper turns the failure into the bootstrap step §11.C
+ * promises — check, install with a visible message, carry on.
+ *
+ * Installs from a system package manager (apt/dnf `google-cloud-cli`, snap)
+ * have the component manager disabled: `gcloud components install` refuses.
+ * There we cannot fix it ourselves, so we fail with the command to run —
+ * gcloud's own message already names the exact package-manager command,
+ * so it is passed through verbatim next to ours.
+ */
+
+import type { GcloudResult } from "./gcloud.js";
+
+type Run = (args: string[]) => Promise<GcloudResult>;
+
+export type EnsureComponentResult = { ok: true; installed: boolean } | { ok: false; error: string };
+
+/** gcloud's wording when its component manager is disabled for this installation. */
+const COMPONENT_MANAGER_DISABLED =
+  /component manager\s+is disabled|managed by an external package manager/i;
+
+/**
+ * Install `component` if it is not installed locally.
+ *
+ * @param opts.run - gcloud runner (injectable for tests).
+ * @param opts.log - receives the progress line shown while installing, so the
+ *   operator sees why the command pauses (the install downloads for ~1 min).
+ * @returns `installed: true` when this call installed it; `ok: false` with an
+ *   actionable message when it is missing and could not be installed.
+ */
+export async function ensureGcloudComponent(
+  component: string,
+  opts: { run: Run; log: (message: string) => void },
+): Promise<EnsureComponentResult> {
+  const listed = await opts.run([
+    "components",
+    "list",
+    "--only-local-state",
+    `--filter=id=${component}`,
+    "--format=value(id)",
+  ]);
+  if (!listed.ok) {
+    return {
+      ok: false,
+      error: `check whether the gcloud "${component}" component is installed: ${listed.stderr.trim()}`,
+    };
+  }
+  const installedIds = listed.stdout.split("\n").map((s) => s.trim());
+  if (installedIds.includes(component)) return { ok: true, installed: false };
+
+  opts.log(`Installing the gcloud "${component}" component (needed once; takes about a minute)…`);
+  const install = await opts.run(["components", "install", component, "--quiet"]);
+  if (install.ok) return { ok: true, installed: true };
+
+  const stderr = install.stderr.trim();
+  if (COMPONENT_MANAGER_DISABLED.test(stderr)) {
+    return {
+      ok: false,
+      error:
+        `the gcloud "${component}" component is missing, and this gcloud was installed by a ` +
+        "system package manager, so it cannot install components itself. Install it with " +
+        "your package manager (Debian/Ubuntu: `sudo apt-get install google-cloud-cli`; " +
+        "Fedora/RHEL: `sudo dnf install google-cloud-cli`), or switch to the standalone " +
+        "Google Cloud CLI from https://cloud.google.com/sdk/docs/install, then re-run this " +
+        `command.\ngcloud said:\n${stderr}`,
+    };
+  }
+  return {
+    ok: false,
+    error:
+      `install the gcloud "${component}" component: ${stderr}\n` +
+      `Run \`gcloud components install ${component}\` yourself (add \`sudo\` if your ` +
+      "Google Cloud CLI directory is not writable by your user), then re-run this command.",
+  };
+}

package/src/gcloud-retry.ts

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Retry for gcloud IAM writes that fail transiently: a service account
+ * created seconds ago is not yet bindable, and concurrent writers to the
+ * same IAM policy lose the etag race. Anything else fails fast — a
+ * PERMISSION_DENIED does not get better by waiting.
+ */
+
+import type { GcloudResult } from "./gcloud.js";
+
+/**
+ * Runs one gcloud command; `stdin` feeds secret payloads (`--data-file=-`),
+ * `env` replaces the base environment (default: this process's).
+ */
+export type GcloudRunner = (
+  args: string[],
+  opts?: { readonly stdin?: string; readonly env?: NodeJS.ProcessEnv },
+) => Promise<GcloudResult>;
+export type Sleep = (ms: number) => Promise<void>;
+
+/** Errors worth retrying: propagation delay and policy write races. */
+const TRANSIENT = /does not exist|not found|NOT_FOUND|concurrent|etag|ABORTED|409/i;
+const RETRY_DELAYS_MS: readonly number[] = [2_000, 4_000, 8_000, 16_000];
+
+export const realSleep: Sleep = (ms) => new Promise<void>((r) => setTimeout(r, ms));
+
+/** Run `args`, retrying with backoff while the failure looks transient. */
+export async function runWithRetry(
+  run: GcloudRunner,
+  sleep: Sleep,
+  args: string[],
+): Promise<GcloudResult> {
+  let r = await run(args);
+  for (const delay of RETRY_DELAYS_MS) {
+    if (r.ok || !TRANSIENT.test(r.stderr)) break;
+    await sleep(delay);
+    r = await run(args);
+  }
+  return r;
+}