@caelo-cms/provisioning 0.10.34 → 0.10.37

package/dist/lifecycle.js

@@ -7,49 +7,123 @@
  * into provider tools for any of them.
  *
  * Each command:
- *   - reads the install id from the active install (single install
- *     per machine for v1; multi-install via `--install-id` flag)
+ *   - acts on the install named by `--install <install-id or domain>`, or
+ *     on the only install on this machine; with several and no flag it
+ *     stops and lists them (selectInstall in install-state.ts)
  *   - dispatches to provider-specific implementations (gcp / aws /
  *     azure / self-hosted)
  *   - emits human-readable progress + a final summary
  */
-import { existsSync, readdirSync } from "node:fs";
-import { homedir } from "node:os";
 import { join, resolve as resolvePath } from "node:path";
-import { cancel, confirm, isCancel, log, note, spinner } from "@clack/prompts";
+import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
 import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
+import { ensureGatewayRolePassword, planPublicRoleSwitch } from "./gateway-credentials.js";
 import { gcloud } from "./gcloud.js";
 import { GCP_STACK_ENV, gatewayServiceAccountEmail, runServiceAccountEmail } from "./gcp-names.js";
-import { installRoot, readMetadata, readSecret, recordImageDigests, } from "./install-state.js";
+import { installFlag, installRoot, listInstalls, readSecret, recordImageDigests, selectInstall, } from "./install-state.js";
 import { ensureMcpIapAccess } from "./mcp-iap.js";
 import { ensureOperatorAccessSync, resolveOperatorAccessTarget } from "./operator-access.js";
 import { assertRegionUnchanged } from "./regions.js";
-import { ensureGatewayServiceAccount, ensureGeneratedSecrets, plainGeneratedSecretSeed, ROTATABLE_SECRETS, readSecretReplication, rotateRuntimeSecret, rotationRefusal, } from "./runtime-secrets.js";
+import { ensureGatewayServiceAccount, ensureGeneratedSecrets, plainGeneratedSecretSeed, ROTATABLE_SECRETS, readSecretReplication, rolesOfSecret, rotateRuntimeSecret, rotationRefusal, } from "./runtime-secrets.js";
 import { adminMediaVolume, MCP_ENV_VAR, OPERATOR_ACCESS_JOB_ENV_VAR } from "./stack-contract.js";
-import { ensureStackInvariants, liveContainerEnv, liveContainerMemory, liveEnvHasInlinePassword, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, rollService, serviceRollArgs, } from "./stack-converge.js";
-/** Find the single install on this machine — or warn if 0/multiple. */
-function findActiveInstall() {
-    const home = homedir();
-    if (!existsSync(home))
-        return null;
-    const candidates = readdirSync(home)
-        .filter((entry) => entry.startsWith(".caelo-"))
-        .map((entry) => entry.replace(/^\.caelo-/, ""));
-    for (const installId of candidates) {
-        const meta = readMetadata(installId);
-        if (meta)
-            return { installId, meta };
-    }
-    return null;
+import { ensureStackInvariants, liveContainerEnv, liveContainerMemory, liveEnvHasInlinePassword, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, removeRetiredIamBindings, rollService, serviceRollArgs, } from "./stack-converge.js";
+/**
+ * The installs a command acts on. `--install <id or domain>` names one
+ * (`--install all` every one, where the command allows it). Without the flag:
+ * the only install on this machine; with several, an interactive terminal
+ * asks which one (or all), and a non-interactive run stops with the list.
+ * Never guesses — acting on the wrong install upgrades or wipes the wrong
+ * site (see selectInstall).
+ */
+async function chooseInstalls(opts) {
+    const installs = listInstalls();
+    let wanted;
+    try {
+        wanted = installFlag(process.argv);
+    }
+    catch (e) {
+        log.error(red(e instanceof Error ? e.message : String(e)));
+        process.exit(2);
+    }
+    if (wanted === "all") {
+        if (!opts.allowAll) {
+            log.error(red(`${opts.verb} runs on one install at a time; pass --install <install-id or domain>.`));
+            process.exit(2);
+        }
+        if (installs.length === 0)
+            return exitNoInstall("No Caelo install found on this machine.");
+        return installs;
+    }
+    if (wanted === undefined && installs.length > 1 && process.stdin.isTTY) {
+        const choice = await select({
+            message: `Which install should ${opts.verb} act on?`,
+            options: [
+                ...installs.map((m) => ({
+                    value: m.installId,
+                    label: m.domain,
+                    hint: `${m.installId}, ${m.provider}`,
+                })),
+                ...(opts.allowAll
+                    ? [{ value: "all", label: `All ${installs.length} installs`, hint: "one after another" }]
+                    : []),
+            ],
+        });
+        if (isCancel(choice)) {
+            cancel("Aborted.");
+            process.exit(1);
+        }
+        if (choice === "all")
+            return installs;
+        wanted = choice;
+    }
+    const selected = selectInstall(installs, wanted);
+    if (!selected.ok)
+        return exitNoInstall(selected.message);
+    return [selected.meta];
 }
-function requireInstall() {
-    const found = findActiveInstall();
-    if (!found) {
-        log.error(red("No Caelo install found on this machine."));
+function exitNoInstall(message) {
+    log.error(red(message));
+    if (listInstalls().length === 0) {
         log.warn(`Run ${bold("bunx @caelo-cms/provisioning")} first to provision an install, OR copy ${dim("~/.caelo-<install-id>/")} from the provisioning machine.`);
-        process.exit(1);
     }
-    return found;
+    process.exit(1);
+}
+/** The one install a single-install command (rotate-secret, truncate, destroy) acts on. */
+async function requireInstall(verb) {
+    const [meta] = await chooseInstalls({ verb, allowAll: false });
+    if (!meta)
+        return exitNoInstall("No Caelo install found on this machine.");
+    return { installId: meta.installId, meta };
+}
+/**
+ * Run `run` for each chosen install in turn, with a header when there are
+ * several. `run` reports whether the install succeeded; the first failure
+ * stops the batch (the installs after it are left untouched) and sets a
+ * non-zero exit code, so scripts see it too.
+ *
+ * @param check validates the chosen set before anything runs (e.g. a
+ *   --region that cannot match several installs); returns an error or null
+ */
+async function forEachInstall(verb, run, check) {
+    const chosen = await chooseInstalls({ verb, allowAll: true });
+    const problem = check?.(chosen) ?? null;
+    if (problem) {
+        log.error(red(problem));
+        process.exit(2);
+    }
+    for (const [i, meta] of chosen.entries()) {
+        if (chosen.length > 1) {
+            note(`${meta.domain} ${dim(`(${meta.installId})`)}`, `${verb} ${i + 1}/${chosen.length}`);
+        }
+        if (await run(meta.installId, meta))
+            continue;
+        process.exitCode = 1;
+        const skipped = chosen.slice(i + 1);
+        if (skipped.length > 0) {
+            log.warn(`${verb} failed for ${meta.domain}; stopped before ${skipped.map((m) => m.domain).join(", ")}.`);
+        }
+        return;
+    }
 }
 /**
  * The install's recorded region (#607); exits with the actionable reason
@@ -101,7 +175,9 @@ async function resolveGcpResourceName(kind, prefix, projectId, region) {
 // status — health check + monthly cost
 // =========================================================================
 export async function statusCommand() {
-    const { installId, meta } = requireInstall();
+    await forEachInstall("status", statusOf);
+}
+async function statusOf(installId, meta) {
     log.info(`Install: ${bold(installId)} ${dim(`(${meta.provider})`)}`);
     log.info(`Domain:  ${bold(meta.domain)}`);
     log.info(`Project: ${bold(meta.projectId ?? "<self-hosted>")}`);
@@ -111,6 +187,7 @@ export async function statusCommand() {
     else {
         log.warn(`status command for provider ${meta.provider} not yet implemented.`);
     }
+    return true;
 }
 /**
  * P20 — fetch the latest stable Caelo release tag from GitHub.
@@ -315,7 +392,22 @@ async function resolveAdminIapResource(meta, region, adminServiceName) {
     return name ? { kind: "backend-services", service: name } : null;
 }
 export async function upgradeCommand(opts = {}) {
-    const { installId, meta } = requireInstall();
+    if (opts.installId) {
+        const selected = selectInstall(listInstalls(), opts.installId);
+        if (!selected.ok)
+            return exitNoInstall(selected.message);
+        if (!(await upgradeInstall(selected.meta.installId, selected.meta, opts)))
+            process.exitCode = 1;
+        return;
+    }
+    await forEachInstall("upgrade", (installId, meta) => upgradeInstall(installId, meta, opts), 
+    // Each install has its own recorded region (#607); one --region can't
+    // match several, and finding out halfway would leave a partial batch.
+    (chosen) => opts.region && chosen.length > 1
+        ? "--region names one install's region; drop it when upgrading several installs."
+        : null);
+}
+async function upgradeInstall(installId, meta, opts) {
     // v0.5.15 — extended to cover gcp-firebase too. Both providers share
     // the identical admin + gateway shape on Cloud Run (Artifact
     // Registry image, `caelo-production-<slug>` service naming, the same
@@ -325,10 +417,10 @@ export async function upgradeCommand(opts = {}) {
     // provider adapters land.
     if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") {
         log.warn(`upgrade for provider ${meta.provider} not yet implemented.`);
-        return;
+        return false;
     }
     if (!meta.projectId)
-        return;
+        return false;
     // #607 — upgrade never moves an install: a `--region` other than the
     // recorded one is refused before anything rolls.
     const region = installRegion(meta, opts.region);
@@ -357,7 +449,7 @@ export async function upgradeCommand(opts = {}) {
         const serviceName = await resolveGcpResourceName("run-service", `caelo-production-${slug}`, meta.projectId, region);
         if (!serviceName) {
             sPre.stop(red(`Could not find caelo-production-${slug}* Cloud Run service`));
-            return;
+            return false;
         }
         // Resolve the tag via the public Docker Registry V2 API directly,
         // not `gcloud artifacts docker tags list`. The gcloud path requires
@@ -374,18 +466,18 @@ export async function upgradeCommand(opts = {}) {
             log.error(`Public registry returned ${digestRes.reason}.\n` +
                 `Verify the tag exists at https://${registryRegion}-docker.pkg.dev/${registryProject}/${registryRepo}/${slug}:${targetTag}\n` +
                 `(latest releases live at https://github.com/caelo-cms/caelo-cms/releases — pass --version vX.Y.Z to pin.)`);
-            return;
+            return false;
         }
         const digest = digestRes.digest;
         const priorRevision = await findCurrentRevision(meta.projectId, region, serviceName);
         if (!priorRevision) {
             sPre.stop(red(`Could not capture current revision for ${slug} — refusing to roll`));
-            return;
+            return false;
         }
         const serviceJson = await describeServiceJson(meta.projectId, region, serviceName);
         if (!serviceJson) {
             sPre.stop(red(`Could not read the ${slug} service's configuration — refusing to roll`));
-            return;
+            return false;
         }
         plans.push({
             slug,
@@ -410,7 +502,7 @@ export async function upgradeCommand(opts = {}) {
     if (!envPlan.ok) {
         sPre.stop(red("Pre-flight failed: the install's env contract can't be applied"));
         log.error(envPlan.error);
-        return;
+        return false;
     }
     // The admin's media lives in the media bucket, mounted as a Cloud Storage
     // volume. Without it every upload is lost on the next revision or
@@ -422,7 +514,7 @@ export async function upgradeCommand(opts = {}) {
     if (!mediaVolume.ok) {
         sPre.stop(red("Pre-flight failed: the admin's media volume can't be set up"));
         log.error(mediaVolume.error);
-        return;
+        return false;
     }
     let rolls = plans.map((p) => {
         // #553 — the admin runs the Lighthouse quality audit; raise it to the
@@ -464,7 +556,7 @@ export async function upgradeCommand(opts = {}) {
     if (!opts.skipVerify) {
         const verified = await verifyCosignAll(plans, registryRegion, registryProject, registryRepo);
         if (!verified)
-            return;
+            return false;
     }
     else {
         log.warn(yellow("--skip-verify set — image signatures NOT verified."));
@@ -496,7 +588,7 @@ export async function upgradeCommand(opts = {}) {
         for (const o of runtimeFailed)
             log.error(red(`  FAILED: ${o.id}\n    ${o.error ?? ""}`));
         log.warn("No traffic was shifted and no migrations ran. Fix the above and re-run.");
-        return;
+        return false;
     }
     sRt.stop(green("Gateway service account + runtime secrets ok"));
     for (const o of runtime.filter((o) => o.status === "applied"))
@@ -506,10 +598,11 @@ export async function upgradeCommand(opts = {}) {
     // from an image roll: IAM bindings + CDN policy added to the stacks after
     // this install was provisioned (stack-contract.ts). Additive + idempotent.
     // A failure the install can't work without aborts here, before
-    // migrations or any traffic shift; the rest warn.
+    // migrations or any traffic shift; the rest warn. Also deletes GCP's
+    // auto-created SSH/RDP-from-anywhere firewall rules (default-firewall.ts).
     // ────────────────────────────────────────────────────────────────
     const sInv = spinner();
-    sInv.start("Ensuring the IAM bindings + CDN settings the stack declares...");
+    sInv.start("Ensuring the IAM bindings + CDN settings the stack declares, removing default SSH/RDP firewall rules...");
     const invariants = await ensureStackInvariants({
         provider: meta.provider,
         projectId: meta.projectId,
@@ -536,7 +629,7 @@ export async function upgradeCommand(opts = {}) {
     }
     if (invariants.mustAbort) {
         log.warn("No traffic was shifted and no migrations ran. Fix the bindings above and re-run.");
-        return;
+        return false;
     }
     // ────────────────────────────────────────────────────────────────
     // P21 ship 3 — DB migrations BEFORE traffic shifts. Idempotent
@@ -565,10 +658,38 @@ export async function upgradeCommand(opts = {}) {
             "  gcloud logging read 'resource.type=cloud_run_job AND " +
             'resource.labels.job_name=~"caelo-migrate-.*"\' ' +
             `--project=${meta.projectId} --limit=50`);
-        return;
+        return false;
     }
     sMig.stop(green("Migrations applied"));
     // ────────────────────────────────────────────────────────────────
+    // #613 — the gateway's own database login. Migration 0248 created
+    // gateway_role without a password; give it the gateway-role-password
+    // secret's value now, before the gateway rolls onto it. Without it the
+    // new gateway could not reach cms_admin, so a failure aborts here.
+    // public_role moves onto its own password right before the gateway's
+    // roll (gateway-credentials.ts), so the old revision keeps working
+    // until then.
+    // ────────────────────────────────────────────────────────────────
+    const sRole = spinner();
+    sRole.start("Ensuring the gateway's database login (gateway_role)...");
+    const sqlInstance = await resolveGcpResourceName("sql-instance", "caelo-production-pg", meta.projectId);
+    if (!sqlInstance) {
+        sRole.stop(red("Could not find the Cloud SQL instance. Aborting upgrade."));
+        log.warn("Migrations ran; no traffic was shifted. Re-run upgrade once the instance is found.");
+        return false;
+    }
+    const roleTarget = { projectId: meta.projectId, env: GCP_STACK_ENV, sqlInstance };
+    const gatewayRole = await ensureGatewayRolePassword(roleTarget);
+    if (gatewayRole.status === "failed") {
+        sRole.stop(red("gateway_role could not be given its password. Aborting upgrade."));
+        log.error(red(`  ${gatewayRole.error ?? ""}`));
+        log.warn("Migrations ran (0248 creates gateway_role); no traffic was shifted. Fix the above and re-run upgrade.");
+        return false;
+    }
+    sRole.stop(green("gateway_role ready"));
+    const gatewayLive = plans.find((p) => p.slug === "gateway");
+    const publicRole = planPublicRoleSwitch(roleTarget, gatewayLive?.liveEnv ?? new Map());
+    // ────────────────────────────────────────────────────────────────
     // Issue #37 — MCP through IAP. Idempotently ensure the MCP service
     // account + its IAP/token-creator bindings; the admin learns the SA's
     // email from CAELO_MCP_IAP_SERVICE_ACCOUNT in the env contract. Installs
@@ -648,9 +769,27 @@ export async function upgradeCommand(opts = {}) {
     // service's run SA: the gateway moves to its own SA here.
     // ────────────────────────────────────────────────────────────────
     const rolled = [];
+    /** A failed gateway roll goes back to a revision that reads public_role's old password. */
+    const restorePublicRole = async (slug) => {
+        if (slug !== "gateway")
+            return;
+        const outcome = await publicRole.afterGatewayRollFailed();
+        if (outcome.startsWith("could NOT"))
+            log.error(red(`  ${outcome}`));
+        else
+            log.info(`  ${outcome}`);
+    };
     for (const plan of rolls) {
         const s = spinner();
         s.start(`Rolling ${plan.slug} → ${plan.digest.slice(0, 19)}...`);
+        if (plan.slug === "gateway") {
+            const moved = await publicRole.beforeGatewayRoll();
+            if (moved.status === "failed") {
+                s.stop(red(`public_role could not be moved onto its own password: ${moved.error ?? ""}`));
+                await rollbackPriorlyRolled(meta.projectId, region, rolled);
+                return false;
+            }
+        }
         const upd = await rollService(serviceRollArgs({
             serviceName: plan.serviceName,
             region,
@@ -665,8 +804,9 @@ export async function upgradeCommand(opts = {}) {
         }));
         if (!upd.ok) {
             s.stop(red(`Failed: ${upd.stderr.trim()}`));
+            await restorePublicRole(plan.slug);
             await rollbackPriorlyRolled(meta.projectId, region, rolled);
-            return;
+            return false;
         }
         // Force traffic onto the new revision. `update --image` only auto-
         // flips when the service has no explicit traffic config — but our
@@ -691,14 +831,16 @@ export async function upgradeCommand(opts = {}) {
         ]);
         if (!flip.ok) {
             s.stop(red(`Traffic flip to latest failed: ${flip.stderr.trim()}`));
+            await restorePublicRole(plan.slug);
             await rollbackPriorlyRolled(meta.projectId, region, rolled);
-            return;
+            return false;
         }
         if (!(await findServiceUrl(meta.projectId, region, plan.serviceName))) {
             s.stop(red(`Could not resolve service URL for ${plan.slug} — rolling back`));
             await rollbackTraffic(meta.projectId, region, plan.serviceName, plan.priorRevision);
+            await restorePublicRole(plan.slug);
             await rollbackPriorlyRolled(meta.projectId, region, rolled);
-            return;
+            return false;
         }
         // Cloud Run's `update --image` (with default --quiet) returns only
         // after the new revision passes its container-readiness probe AND
@@ -717,6 +859,29 @@ export async function upgradeCommand(opts = {}) {
         rolled.push(plan);
     }
     log.success(`Upgrade to ${bold(targetTag)} complete (admin + gateway revisions Ready).`);
+    // #613 — with both services on their new revisions, drop the bindings
+    // nothing reads through any more: above all the gateway SA's access to
+    // admin_role's password. Loud on failure — until it is gone, a
+    // compromised gateway could still read that password.
+    const retired = await removeRetiredIamBindings({
+        provider: meta.provider,
+        projectId: meta.projectId,
+        region,
+        env: GCP_STACK_ENV,
+        services: Object.fromEntries(rolls.map((r) => [r.slug, r.serviceName])),
+    });
+    for (const o of retired) {
+        if (o.status === "removed")
+            log.info(`  removed: ${o.id} ${dim(`(${o.why})`)}`);
+        if (o.status === "failed") {
+            log.error(red(`  NOT removed: ${o.id} — ${o.why}\n    ${o.error ?? ""}\n    Re-run upgrade with an account that may change IAM, or remove the binding by hand.`));
+        }
+    }
+    if (publicRole.moves) {
+        // Until this upgrade the gateway held admin_role's password; anyone who
+        // ever compromised it could have kept a copy.
+        log.warn(yellow(`The gateway no longer holds admin_role's password, but it did until now. Rotate it: ${bold("bunx @caelo-cms/provisioning rotate-secret postgres-password")}`));
+    }
     if (rolls.some((r) => liveEnvHasInlinePassword(r.liveEnv))) {
         // The services just moved their database password to Secret Manager,
         // but every earlier revision still shows it in its env.
@@ -732,6 +897,7 @@ export async function upgradeCommand(opts = {}) {
         admin: digestOf("admin"),
         gateway: digestOf("gateway"),
     });
+    return true;
 }
 /**
  * Record the digests just rolled so nothing re-deploys an older release:
@@ -909,19 +1075,21 @@ async function rollbackPriorlyRolled(projectId, region, rolled) {
 // backup — Cloud SQL on-demand backup + bundle Pulumi state
 // =========================================================================
 export async function backupCommand() {
-    const { meta } = requireInstall();
+    await forEachInstall("backup", (_installId, meta) => backupOf(meta));
+}
+async function backupOf(meta) {
     if (meta.provider !== "gcp") {
         log.warn(`backup for provider ${meta.provider} not yet implemented.`);
-        return;
+        return false;
     }
     if (!meta.projectId)
-        return;
+        return false;
     const s = spinner();
     s.start("Resolving Cloud SQL instance + triggering on-demand backup...");
     const sqlName = await resolveGcpResourceName("sql-instance", "caelo-production-pg", meta.projectId);
     if (!sqlName) {
         s.stop(red("Could not find caelo-production-pg* Cloud SQL instance"));
-        return;
+        return false;
     }
     const r = await gcloud([
         "sql",
@@ -936,9 +1104,10 @@ export async function backupCommand() {
     ]);
     if (!r.ok) {
         s.stop(red(`Failed: ${r.stderr.trim()}`));
-        return;
+        return false;
     }
     s.stop(green(`Backup created. List with \`gcloud sql backups list --instance=${sqlName}\`.`));
+    return true;
 }
 // =========================================================================
 // rotate-secret <name>
@@ -955,7 +1124,7 @@ export async function rotateSecretCommand(name) {
         process.exit(2);
     }
     const secret = name;
-    const { meta } = requireInstall();
+    const { meta } = await requireInstall("rotate-secret");
     if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") {
         log.warn(`rotate-secret for provider ${meta.provider} not yet implemented.`);
         return;
@@ -974,7 +1143,7 @@ export async function rotateSecretCommand(name) {
         }
         services[slug] = serviceName;
     }
-    const sqlInstance = secret === "postgres-password"
+    const sqlInstance = rolesOfSecret(secret).length > 0
         ? await resolveGcpResourceName("sql-instance", "caelo-production-pg", meta.projectId)
         : null;
     const report = await rotateRuntimeSecret({
@@ -1010,7 +1179,7 @@ export async function rotateSecretCommand(name) {
  * longer renders cleanly.
  */
 export async function truncateCommand() {
-    const { meta } = requireInstall();
+    const { meta } = await requireInstall("truncate");
     log.warn(yellow(`${bold("Truncate")} wipes ALL pages, modules, templates, layouts, redirects, nav menus, theme tokens, media assets, chat history, audit log, snapshots, and site memory on ${bold(meta.domain)}.\n` +
         `Preserved: users / roles / AI providers / domains / site_defaults / provisioning_outputs.`));
     const confirm1 = await confirm({
@@ -1054,7 +1223,7 @@ export async function truncateCommand() {
     log.warn(`Truncate not implemented for provider=${meta.provider}.`);
 }
 export async function destroyCommand() {
-    const { installId, meta } = requireInstall();
+    const { installId, meta } = await requireInstall("destroy");
     log.warn(red(`${bold("Destroy will PERMANENTLY delete")} the GCP project + every Caelo resource. This is irreversible after the 30-day undelete window.`));
     const confirm1 = await confirm({
         message: `Destroy the install for ${bold(meta.domain)} (${bold(meta.projectId ?? "self-hosted")})?`,