@caelo-cms/provisioning 0.10.46 → 0.10.48

package/dist/wizards/gcp.js

@@ -28,7 +28,7 @@ import { randomBytes } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { cancel, confirm, isCancel, log, note, select, spinner, text } from "@clack/prompts";
+import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
 import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
 import { checkAdcImpersonation } from "../adc-impersonation.js";
 import { CERT_WAIT_TIMEOUT_MS, lbCertProbe, waitForCert } from "../cert-wait.js";
@@ -39,20 +39,27 @@ import { formatDnsRecordGroups, groupDnsRecords } from "../dns/groups.js";
 import { pickDnsAdapter } from "../dns/index.js";
 import { ensureGatewayRolePassword, findSqlInstance } from "../gateway-credentials.js";
 import { activeAccount, createProject, createServiceAccount, enableApis, gcloud, gcloudEnv, grantProvisionerRoles, linkBilling, listBillingAccounts, PROVISIONER_ROLE_LIST, projectExists, REQUIRED_API_LIST, serviceAccountExists, } from "../gcloud.js";
-import { GCP_STACK_ENV } from "../gcp-names.js";
-import { adoptStateBackend, ensureInstallDir, installRoot, installStagingHost, isStepDone, markStepDone, readMetadata, readSecret, recordedImageDigests, recordImageDigests, writeMetadata, writeSecret, } from "../install-state.js";
+import { GCP_STACK_ENV, upgraderServiceAccountEmail } from "../gcp-names.js";
+import { INFRA_CONTRACT_VERSION } from "../infra-contract.js";
+import { INSTALL_JOB_NAME, INSTALL_JOB_ROLES, INSTALL_SETUP_LINK_SECRET, storeSetupLink, } from "../install-job.js";
+import { adoptStateBackend, ensureInstallDir, installRoot, installStagingHost, isStepDone, markStepDone, readLocalMetadata, readMetadata, readSecret, recordedImageDigests, recordImageDigests, recordInfraContractVersion, writeMetadata, writeSecret, } from "../install-state.js";
 import { ensureOperatorAccessSync, installIapAllowlist, resolveOperatorAccessTarget, } from "../operator-access.js";
 import { ensureOpsBackend } from "../ops-backend.js";
 import { convergeKeylessProvisioner, PROVISIONER_ACCOUNT_ID, provisionerServiceAccountEmail, } from "../provisioner-identity.js";
 import { decideRegion, detectCliRegion, installableRegions, suggestRegion } from "../regions.js";
 import { selfCommand } from "../self-command.js";
 import { ABSENT_DEFAULT_FIREWALL_RULES } from "../stack-contract.js";
-import { chooseInstallBackend, describeBackend, gcsBackendRecord, hasLocalPulumiState, resolveBackend, } from "../state-backend.js";
-import { GcsStateStore } from "../state-store.js";
+import { chooseInstallBackend, describeBackend, gcsBackendRecord, hasLocalPulumiState, installBackend, parseStateBackend, resolveBackend, } from "../state-backend.js";
+import { acquireStateLock, GcsStateStore } from "../state-store.js";
+import { releaseVersionForDigest, resolveOpsImageRef, verifyCosignAll, } from "../upgrade/release-images.js";
+import { ensureUpgraderJob, resolveUpgraderTarget } from "../upgrader-setup.js";
 import { estimateGcpCost } from "./gcp-cost.js";
+import { promptProjectId } from "./gcp-prompts.js";
 import { GCP_STACK_KNOB_DEFAULTS, pulumiUpGcp, readStackOutput } from "./gcp-pulumi.js";
 export async function runGcpWizard(opts) {
-    const { installId, domain, ownerEmail } = opts;
+    const { installId } = opts;
+    const provider = opts.provider ?? "gcp";
+    const runMode = opts.runMode ?? "local";
     const { secretsDir } = ensureInstallDir(installId);
     // === 1. gcloud auth ===
     await stepActiveGcloud();
@@ -75,7 +82,7 @@ export async function runGcpWizard(opts) {
     // === 4. Billing link ===
     await stepBillingLink(installId, projectId, opts.nonInteractive);
     // === 5. Enable APIs ===
-    await stepEnableApis(installId, projectId, opts.provider ?? "gcp");
+    await stepEnableApis(installId, projectId, provider);
     // === 6. Service account + roles + impersonation (no key, #696) ===
     // v0.3.12 — dropped the firebase-hosting-identity step. The
     // Service Usage GenerateServiceIdentity API doesn't support
@@ -89,19 +96,19 @@ export async function runGcpWizard(opts) {
     // didn't.
     const saEmail = provisionerServiceAccountEmail(projectId);
     await stepServiceAccount(installId, projectId, saEmail);
-    await stepGrantRoles(installId, projectId, saEmail);
-    await stepKeylessProvisioner(projectId, secretsDir, opts.nonInteractive);
+    await stepGrantRoles(installId, projectId, saEmail, runMode);
+    // Pulumi runs here only in local mode; the install job runs AS the
+    // provisioner, so this terminal needs no Application Default Credentials.
+    await stepKeylessProvisioner(projectId, secretsDir, {
+        nonInteractive: opts.nonInteractive,
+        checkAdc: runMode === "local",
+    });
     // === 8. Cost-estimate pre-flight ===
     // v0.3.3 — provider variant threads through so gcp-firebase
     // drops the LB / Cloud CDN / Cloud Armor lines (saves ~$19/mo
     // vs gcp). Default 'gcp' for backwards compatibility with
     // installs that don't set the field.
-    const costInputs = {
-        ...GCP_STACK_KNOB_DEFAULTS,
-        provider: opts.provider ?? "gcp",
-        region,
-    };
-    const estimate = estimateGcpCost(costInputs);
+    const estimate = estimateGcpCost({ ...GCP_STACK_KNOB_DEFAULTS, provider, region });
     note([
         bold("Estimated monthly cost (resource floor — actual usage adds AI calls + egress + storage growth)"),
         `  ${estimate.regionNote}`,
@@ -111,6 +118,11 @@ export async function runGcpWizard(opts) {
         `  ${bold("TOTAL".padEnd(40))} ${bold(`$${estimate.totalUsd}`.padStart(5))}/mo`,
         "",
         `  ${dim(`Also deletes the default network's pre-populated firewall rules ${ABSENT_DEFAULT_FIREWALL_RULES.map((r) => r.name).join(" + ")} (Caelo runs no VMs and does not use that network).`)}`,
+        ...(runMode === "job"
+            ? [
+                `  ${dim(`The install itself runs as a one-off Cloud Run job (${INSTALL_JOB_NAME}) in the project: about ${INSTALL_JOB_COST_NOTE} for one install run, nothing while idle.`)}`,
+            ]
+            : []),
     ].join("\n"), "Pre-flight");
     if (!opts.nonInteractive) {
         const proceed = await confirm({
@@ -126,7 +138,48 @@ export async function runGcpWizard(opts) {
     // A new install: the ops bucket + KMS key in its own project — after the
     // cost confirmation, which lists them (CLAUDE.md §11.C: nothing billable
     // before the y/N). An older install keeps its local state + passphrase.
-    const backend = await stepStateBackend(installId, projectId, region, saEmail);
+    const backend = await stepStateBackend(installId, projectId, region, saEmail, runMode);
+    // === 9. The rest runs where the run mode says (run-mode.ts) ===
+    if (runMode === "job") {
+        const { launchInstallJob } = await import("./gcp-install-job.js");
+        await launchInstallJob({
+            installId,
+            projectId,
+            region,
+            provisionerEmail: saEmail,
+            nonInteractive: opts.nonInteractive,
+        });
+        return;
+    }
+    await runGcpConverge({
+        installId,
+        projectId,
+        region,
+        provider,
+        impersonateServiceAccount: saEmail,
+        backend,
+        nonInteractive: opts.nonInteractive,
+        setupLink: "print",
+    });
+}
+/** One-off cost of an install-job run (2 vCPU + 4 GiB for up to ~2 h). */
+const INSTALL_JOB_COST_NOTE = "$0.20-0.40";
+/**
+ * The half of an install that needs no operator identity: image digests,
+ * `pulumi up`, DNS + certificate waits, migrations, database logins, the
+ * operator-access job, the placeholder page and the summary. Runs in the
+ * operator's terminal (local mode) or in the install job (install-job.ts).
+ * Every step is idempotent or checkpointed in progress.json, so a re-run —
+ * or a new job execution — resumes where the last one stopped.
+ */
+export async function runGcpConverge(opts) {
+    const { installId, projectId, region, provider, backend } = opts;
+    const phase = opts.onPhase ?? (() => { });
+    const meta = readMetadata(installId);
+    if (!meta)
+        throw new Error(`No install.json for install '${installId}'.`);
+    const { domain, ownerEmail } = meta;
+    const costInputs = { ...GCP_STACK_KNOB_DEFAULTS, provider, region };
     // === 9. Image digests to deploy ===
     // An install that already runs a recorded release (from an earlier
     // wizard run or from `upgrade`) keeps it: re-running the wizard converges
@@ -137,7 +190,10 @@ export async function runGcpWizard(opts) {
     // An install provisioned before the record existed keeps what its
     // services run. A new install resolves the floating `:latest` tag to a
     // fixed sha256 digest: Cloud Run keys revisions by image reference, so a
-    // tag would never trigger a new revision.
+    // tag would never trigger a new revision. (The install job's launcher
+    // records the digests of its signed release first, so a job install never
+    // reaches `:latest`.)
+    phase("Choosing the release images");
     const deployed = isStepDone(installId, `pulumi-up-${projectId}`);
     if (deployed) {
         // The kept release must be able to use the env the stack deploys.
@@ -148,7 +204,7 @@ export async function runGcpWizard(opts) {
         }
     }
     const choice = await chooseImageDigests({
-        recorded: meta ? recordedImageDigests(meta) : null,
+        recorded: recordedImageDigests(meta),
         deployed,
         readLive: () => readDeployedImageDigests({ projectId, region }),
         resolveLatest: resolveImageDigests,
@@ -174,17 +230,15 @@ export async function runGcpWizard(opts) {
     // Pre-built signed images live at the Caelo-team public AR repo
     // (caelo-website/caelo-cms-images by default). Cloud Run reads them
     // directly with no operator-side IAM binding — anonymous public pull.
+    phase("Provisioning the infrastructure (pulumi up, 8-15 min)");
     await stepPulumiUp(installId, {
         projectId,
         domain,
         // #650 — the wizard recorded it (or the install predates the setting).
-        stagingDomain: installStagingHost({
-            domain,
-            stagingDomain: readMetadata(installId)?.stagingDomain,
-        }),
+        stagingDomain: installStagingHost({ domain, stagingDomain: meta.stagingDomain }),
         ownerEmail,
         region,
-        impersonateServiceAccount: saEmail,
+        impersonateServiceAccount: opts.impersonateServiceAccount,
         backend,
         cloudSqlTier: costInputs.cloudSqlTier,
         cloudSqlHa: costInputs.cloudSqlHa,
@@ -197,7 +251,7 @@ export async function runGcpWizard(opts) {
         imageDigests,
         // v0.3.1 — route pulumi up at the right stack folder + use the
         // matching config namespace.
-        provider: opts.provider ?? "gcp",
+        provider,
     });
     recordImageDigests(installId, imageDigests);
     // === 10.1. Remove the default network's pre-populated firewall rules ===
@@ -220,8 +274,9 @@ export async function runGcpWizard(opts) {
     // The DNS records come first: the certificate only validates once the
     // admin host resolves to the load balancer. Only the "set now" records
     // are applied here — the main domain moves at go-live (dns/groups.ts).
-    if ((opts.provider ?? "gcp") === "gcp") {
-        await stepGcpDnsRecords(installId);
+    if (provider === "gcp") {
+        await stepGcpDnsRecords(installId, { nonInteractive: opts.nonInteractive, onPhase: phase });
+        phase(`Waiting for the TLS certificate of admin.${domain} (15-60 min)`);
         await stepWaitForCertActive(projectId, `admin.${domain}`);
     }
     // === 11. Apply DB migrations against Cloud SQL via a one-shot Job ===
@@ -233,6 +288,7 @@ export async function runGcpWizard(opts) {
     // P21 ship 3 — shared with `cms-provision upgrade`. Aborts the
     // wizard with a clear error if migrations fail (vs. silently
     // continuing with a half-migrated DB).
+    phase("Applying database migrations");
     {
         const { runMigrationsViaCloudRunJob } = await import("../migration-runner.js");
         const r = await runMigrationsViaCloudRunJob({ projectId, region });
@@ -246,9 +302,27 @@ export async function runGcpWizard(opts) {
     // gateway-role-password secret's value here. The gateway (deployed by
     // Pulumi before migrations) retries its database on every request, so it
     // starts serving /api/* as soon as this lands — no restart.
+    phase("Finishing: database logins, operator access, welcome page");
     await stepGatewayRolePassword(projectId);
+    // === 11.4. The upgrader job: Settings → Updates → Install update (#696) ===
+    // Before operator access, which keeps the upgrader's caelo-mcp grant.
+    const releaseVersion = await releaseVersionForDigest(gcloud, "admin", imageDigests.admin);
+    const upgraderReady = await stepUpgraderJob(projectId, region, releaseVersion);
+    const keepSigner = upgraderReady || (await upgraderExists(projectId));
     // === 11.5. Operator access: IAP follows the Caelo user list ===
-    await stepOperatorAccessSync(projectId, region, opts.provider ?? "gcp", ownerEmail);
+    await stepOperatorAccessSync(projectId, region, provider, ownerEmail, keepSigner ? [`serviceAccount:${upgraderServiceAccountEmail(projectId, GCP_STACK_ENV)}`] : []);
+    // A fresh install ran every converge step above, so it is at this CLI's
+    // infrastructure contract — recorded only when the upgrader is in place
+    // (the last step a contract bump added). With the release version, the
+    // admin can install the next update itself.
+    if (upgraderReady) {
+        const version = releaseVersion;
+        recordImageDigests(installId, imageDigests, version);
+        recordInfraContractVersion(installId, INFRA_CONTRACT_VERSION);
+        if (!version) {
+            log.warn(yellow("Could not tell which Caelo release the admin runs, so in-product updates stay off until the first `upgrade`."));
+        }
+    }
     // === 12. Upload the fresh-install placeholder to the static bucket ===
     // Without this, https://<domain>/ returns the raw GCS NoSuchKey XML
     // until the operator publishes their first deploy. Idempotent: skips
@@ -259,16 +333,13 @@ export async function runGcpWizard(opts) {
     // Hosting, not a GCS bucket; the static-publisher's first deploy
     // creates the initial Firebase Hosting version. Skip the GCS
     // placeholder upload on that provider.
-    if ((opts.provider ?? "gcp") === "gcp") {
+    if (provider === "gcp") {
         await stepUploadStaticPlaceholder(installId, projectId, domain);
     }
     // === 12. DNS records + bootstrap URL ===
     // IAP is enabled directly on the LB BackendService (Pulumi-managed);
     // no post-up gcloud step needed.
-    await stepFinalize(installId);
-    // Reference unused params to silence the linter.
-    void domain;
-    void ownerEmail;
+    await stepFinalize(installId, opts.setupLink);
 }
 /**
  * Resolve the floating `:latest` tag on each service's public AR image
@@ -343,25 +414,7 @@ async function stepProjectId(opts) {
         return opts.projectId;
     }
     const guess = opts.domain.split(".")[0]?.replace(/[^a-z0-9-]/g, "-") ?? "caelo";
-    const value = await text({
-        message: "GCP project id (will be created if absent)",
-        placeholder: guess,
-        defaultValue: guess,
-        validate: (v) => {
-            if (!v || v.length < 6 || v.length > 30) {
-                return "GCP project id must be 6–30 chars";
-            }
-            if (!/^[a-z][a-z0-9-]*[a-z0-9]$/.test(v)) {
-                return "GCP project id: lowercase letters, digits, hyphens; start with letter; no trailing hyphen";
-            }
-            return undefined;
-        },
-    });
-    if (isCancel(value)) {
-        cancel("Cancelled.");
-        process.exit(0);
-    }
-    return value;
+    return promptProjectId(guess);
 }
 /**
  * Settle the install region (#607). A recorded region — or, for an install
@@ -566,7 +619,35 @@ async function stepServiceAccount(installId, projectId, saEmail) {
     s.stop(green(`Provisioner SA ${bold(saEmail)} created`));
     markStepDone(installId, stepName, { created: true, saEmail });
 }
-async function stepGrantRoles(installId, projectId, saEmail) {
+async function stepGrantRoles(installId, projectId, saEmail, runMode) {
+    await stepGrantBaseRoles(installId, projectId, saEmail);
+    if (runMode === "job")
+        await stepGrantInstallJobRoles(installId, projectId, saEmail);
+}
+/**
+ * #696 — the roles the provisioner needs when it runs the install job
+ * itself (install-job.ts INSTALL_JOB_ROLES). Only for installs that use it.
+ */
+async function stepGrantInstallJobRoles(installId, projectId, saEmail) {
+    const stepName = `grant-install-job-roles-${projectId}`;
+    if (isStepDone(installId, stepName)) {
+        log.success(`Install-job roles granted ${dim("(checkpointed)")}`);
+        return;
+    }
+    const s = spinner();
+    s.start(`Granting the provisioner ${INSTALL_JOB_ROLES.length} more roles to run the install job...`);
+    const { granted, failed } = await grantProvisionerRoles(projectId, saEmail, {
+        roles: INSTALL_JOB_ROLES,
+    });
+    if (failed.length > 0) {
+        s.stop(red(`Failed: ${failed.map((f) => `${f.role}: ${f.error}`).join("; ")}`));
+        cancel("Aborted — re-run the installer (idempotent).");
+        process.exit(1);
+    }
+    s.stop(green(`${granted} install-job roles granted`));
+    markStepDone(installId, stepName, { granted });
+}
+async function stepGrantBaseRoles(installId, projectId, saEmail) {
     const stepName = `grant-roles-${projectId}`;
     if (isStepDone(installId, stepName)) {
         log.success(`IAM roles granted ${dim(`(${PROVISIONER_ROLE_LIST.length} roles, checkpointed)`)}`);
@@ -608,13 +689,69 @@ async function stepGatewayRolePassword(projectId) {
     }
     s.stop(green("gateway_role ready"));
 }
+/**
+ * Set up the upgrader job (upgrader-setup.ts, #696): what the admin's
+ * "Install update" starts. After migrations, because the job's database
+ * user gets the role migration 0267 creates. A failure does not stop the
+ * install — the site works; updates then go through the CLI until a later
+ * `upgrade` sets the job up — so it only warns. Idempotent.
+ *
+ * @returns whether the job is ready
+ */
+async function stepUpgraderJob(projectId, region, releaseVersion) {
+    // The upgrader of the release the install runs (a re-run keeps an older
+    // release — a newer upgrader would meet a schema it does not know yet).
+    const opsImageRef = releaseVersion ? await resolveOpsImageRef(releaseVersion) : null;
+    const digest = opsImageRef ? /@(sha256:[0-9a-f]{64})$/.exec(opsImageRef)?.[1] : undefined;
+    // The job runs with rights over the whole app: only a verified image.
+    if (digest && !(await verifyCosignAll([{ slug: "ops", digest }]))) {
+        log.warn(`The upgrader is not set up: its image's signature did not verify. The install works; ${bold(selfCommand("upgrade"))} sets it up later.`);
+        return false;
+    }
+    const s = spinner();
+    s.start("Setting up the upgrader job (install updates from the admin)...");
+    const resolved = opsImageRef
+        ? await resolveUpgraderTarget({ projectId, region, env: GCP_STACK_ENV, opsImageRef })
+        : {
+            ok: false,
+            error: releaseVersion
+                ? `release ${releaseVersion} has no caelo-ops image in the registry`
+                : "the release this install runs could not be told from its images",
+        };
+    const r = resolved.ok
+        ? await ensureUpgraderJob(resolved.target)
+        : { ok: false, done: [], error: resolved.error };
+    if (!r.ok) {
+        s.stop(yellow(`Upgrader job not set up: ${r.error}`));
+        log.warn(`The install works; Settings → Updates shows the upgrade command instead of an Install button until ${bold(selfCommand("upgrade"))} sets the job up.`);
+        return false;
+    }
+    s.stop(green(`Upgrader job ready (${r.done.length} steps)`));
+    return true;
+}
+/**
+ * Whether an upgrader set up by an earlier run exists, so a failed setup
+ * this time still keeps its caelo-mcp signing grant (the operator-access
+ * job would otherwise remove it, breaking its health check).
+ */
+async function upgraderExists(projectId) {
+    const r = await gcloud([
+        "iam",
+        "service-accounts",
+        "describe",
+        upgraderServiceAccountEmail(projectId, GCP_STACK_ENV),
+        `--project=${projectId}`,
+        "--format=value(email)",
+    ]);
+    return r.ok;
+}
 /**
  * Set up the operator-access sync job (operator-access.ts): the only
  * principal that may change who passes IAP, run as the operator's gcloud
  * identity. After migrations, because the job's database user gets the role
  * migration 0239 creates. Idempotent, so it runs on every wizard pass.
  */
-async function stepOperatorAccessSync(projectId, region, provider, ownerEmail) {
+async function stepOperatorAccessSync(projectId, region, provider, ownerEmail, mcpSigners) {
     const s = spinner();
     s.start("Setting up the operator-access sync job (Google IAP follows the user list)...");
     const resolved = await resolveOperatorAccessTarget({
@@ -625,7 +762,7 @@ async function stepOperatorAccessSync(projectId, region, provider, ownerEmail) {
         ownerEmail,
     });
     const r = resolved.ok
-        ? await ensureOperatorAccessSync(resolved.target)
+        ? await ensureOperatorAccessSync({ ...resolved.target, mcpSigners })
         : { ok: false, done: [], error: resolved.error };
     if (!r.ok) {
         s.stop(red(`Failed: ${r.error}`));
@@ -667,7 +804,7 @@ async function stepRemoveDefaultFirewallRules(projectId) {
  * Application Default Credentials impersonating the SA — is verified before
  * anything billable runs. Idempotent; runs on every wizard pass.
  */
-async function stepKeylessProvisioner(projectId, secretsDir, nonInteractive) {
+async function stepKeylessProvisioner(projectId, secretsDir, opts) {
     const sa = provisionerServiceAccountEmail(projectId);
     const account = await activeAccount();
     if (!account) {
@@ -702,6 +839,8 @@ async function stepKeylessProvisioner(projectId, secretsDir, nonInteractive) {
         for (const line of converged.lines)
             log.info(`  ${line}`);
     }
+    if (!opts.checkAdc)
+        return;
     const check = async () => checkAdcImpersonation({
         projectId,
         operatorAccount: account,
@@ -710,7 +849,7 @@ async function stepKeylessProvisioner(projectId, secretsDir, nonInteractive) {
     const sAdc = spinner();
     sAdc.start("Checking that your Application Default Credentials can impersonate it...");
     let adc = await check();
-    if (!adc.ok && adc.reason === "no-adc" && !nonInteractive) {
+    if (!adc.ok && adc.reason === "no-adc" && !opts.nonInteractive) {
         sAdc.stop(yellow("No Application Default Credentials yet"));
         const login = await confirm({
             message: `Run ${bold("gcloud auth application-default login")} now (opens your browser; sign in as ${account})?`,
@@ -745,15 +884,21 @@ async function stepKeylessProvisioner(projectId, secretsDir, nonInteractive) {
  * that install's record; this install's own copy from an interrupted run is
  * taken over.
  */
-async function stepStateBackend(installId, projectId, region, provisionerEmail) {
+async function stepStateBackend(installId, projectId, region, provisionerEmail, runMode) {
     const root = installRoot(installId);
     const meta = readMetadata(installId);
     if (!meta)
         throw new Error(`No install.json for install '${installId}'.`);
     const kind = chooseInstallBackend(meta, hasLocalPulumiState(root), isStepDone(installId, `pulumi-up-${projectId}`));
+    if (kind === "local" && runMode === "job") {
+        // The job reads the state from the project; a passphrase-encrypted
+        // state on this machine is out of its reach.
+        cancel(`This install's Pulumi state is on this machine (${describeBackend(meta, root)}), so the install job can't run it. Move it into project ${projectId} first — ${selfCommand("state", "migrate", "--install", installId)} — or re-run with --local.`);
+        process.exit(1);
+    }
     if (kind === "local") {
         stepPulumiPassphrase(installId);
-        log.info(`Pulumi state: ${dim(describeBackend(meta, root))} (this install predates state in project ${projectId}; it stays where it is).`);
+        log.info(`Pulumi state: ${dim(describeBackend(meta, root))} (this install predates state in project ${projectId}). Move it there, so any machine you sign in on can manage the install: ${bold(selfCommand("state", "migrate", "--install", installId))}`);
         return resolveBackend(meta, {
             installRoot: root,
             readPassphrase: () => readSecret(installId, "pulumi-passphrase"),
@@ -791,7 +936,7 @@ async function stepStateBackend(installId, projectId, region, provisionerEmail)
         // run, interrupted between writing the bucket and the local copy:
         // finishing the move is safe. Anything else is another install.
         if (other && !(other.installId === meta.installId && other.createdAt === meta.createdAt)) {
-            cancel(`Project ${projectId} already has a Caelo install (${other.domain ?? "unknown domain"}) whose state lives in gs://${record.bucket}. This run would overwrite that install's record; run the installer from the machine that set it up instead.`);
+            cancel(`Project ${projectId} already has a Caelo install (${other.domain ?? "unknown domain"}) whose state lives in gs://${record.bucket}. This run would overwrite that install's record. To manage it from this machine run: ${selfCommand("attach", "--project", projectId)}`);
             process.exit(1);
         }
         adoptStateBackend(installId, record);
@@ -815,33 +960,77 @@ function stepPulumiPassphrase(installId) {
 }
 async function stepPulumiUp(installId, opts) {
     const stepName = `pulumi-up-${opts.projectId}`;
-    if (isStepDone(installId, stepName)) {
+    const deployed = isStepDone(installId, stepName);
+    if (deployed) {
         log.success(`Pulumi up ${dim("(checkpointed — re-running for drift refresh)")}`);
     }
     const { secretsDir } = ensureInstallDir(installId);
     const root = installRoot(installId);
+    // #696 — hold the install's state lock through `pulumi up` (`state
+    // migrate` takes the same one), then re-check, uncached, that the state
+    // still lives where this run resolved it: the bucket's install.json for a
+    // bucket-backed install (another machine may have moved it back), the
+    // file on disk for a local one.
+    const opsBucket = gcsBackendRecord(opts.projectId, opts.region).bucket;
+    let releaseLock;
+    try {
+        releaseLock = acquireStateLock(opts.backend.kind === "gcs" ? { kind: "gcs", bucket: opsBucket } : { kind: "local" }, root, "installer pulumi up");
+    }
+    catch (e) {
+        cancel(e instanceof Error ? e.message : String(e));
+        process.exit(1);
+    }
+    let moved;
+    try {
+        const nowText = opts.backend.kind === "gcs"
+            ? new GcsStateStore(opsBucket, root).peek("install.json")
+            : JSON.stringify(readLocalMetadata(installId));
+        const now = nowText ? JSON.parse(nowText) : null;
+        moved =
+            !now ||
+                installBackend({ stateBackend: parseStateBackend(now.stateBackend) }).kind !==
+                    opts.backend.kind;
+    }
+    catch (e) {
+        releaseLock();
+        throw e;
+    }
+    if (moved) {
+        releaseLock();
+        cancel("The install's Pulumi state has moved (state migrate) since this run started. Re-run the installer.");
+        process.exit(1);
+    }
     log.info(`Pulumi up — wall-clock 8–15 min (Cloud SQL is the long pole). Streaming progress...`);
     let resourceCount = 0;
-    const result = await pulumiUpGcp({
-        installId,
-        installRoot: root,
-        secretsDir,
-        ...opts,
-        provider: opts.provider ?? "gcp",
-    }, (kind, message) => {
-        if (kind === "resource") {
-            resourceCount++;
-            if (resourceCount % 5 === 0) {
-                process.stdout.write(`\r${dim(`  ${resourceCount} resources updated`)}`);
+    let result;
+    try {
+        result = await pulumiUpGcp({
+            installId,
+            installRoot: root,
+            secretsDir,
+            ...opts,
+            // A deployed install's stack must exist: a missing one means the
+            // state is elsewhere, and a fresh stack would try to recreate it all.
+            stackMustExist: deployed,
+            provider: opts.provider ?? "gcp",
+        }, (kind, message) => {
+            if (kind === "resource") {
+                resourceCount++;
+                if (resourceCount % 5 === 0) {
+                    process.stdout.write(`\r${dim(`  ${resourceCount} resources updated`)}`);
+                }
             }
-        }
-        else if (kind === "error") {
-            process.stdout.write("\n");
-            log.error(red(redactCredentials(message)));
-        }
-        // logs: silent — too noisy for a live progress UI; pulumi
-        // diagnostics will print on failure via the SDK's onOutput.
-    });
+            else if (kind === "error") {
+                process.stdout.write("\n");
+                log.error(red(redactCredentials(message)));
+            }
+            // logs: silent — too noisy for a live progress UI; pulumi
+            // diagnostics will print on failure via the SDK's onOutput.
+        });
+    }
+    finally {
+        releaseLock();
+    }
     process.stdout.write("\n");
     log.success(green(`Pulumi up complete — ${result.resourceCount.created} created, ${result.resourceCount.updated} updated, ${result.resourceCount.deleted} deleted`));
     markStepDone(installId, stepName, { outputs: result.outputs });
@@ -1228,13 +1417,14 @@ const ADAPTER_RECORD_TYPES = new Set(["A", "AAAA", "CNAME", "TXT"]);
  * live site are never applied here — the summary lists them for go-live.
  * Idempotent: a finished step is skipped on re-runs.
  */
-async function stepGcpDnsRecords(installId) {
+async function stepGcpDnsRecords(installId, opts) {
     const meta = readMetadata(installId);
     if (!meta?.projectId)
         return;
     const stepName = `dns-${meta.projectId}`;
     if (isStepDone(installId, stepName))
         return;
+    opts.onPhase("Setting up DNS");
     const records = stackDnsRecords(await recordedStackOutputs(installId, meta.projectId));
     if (records.length === 0) {
         cancel("The stack reported no DNS records (dnsRecordsRequiredOut). Re-run the installer to refresh its outputs.");
@@ -1245,8 +1435,12 @@ async function stepGcpDnsRecords(installId) {
         log.warn(yellow(`Not touching ${meta.domain} itself: its records switch a live website to Caelo and are listed at the end for go-live.`));
     }
     const toApply = setNow.filter((r) => ADAPTER_RECORD_TYPES.has(r.type));
-    const adapter = await pickDnsAdapter({ domain: meta.domain });
+    const adapter = await pickDnsAdapter({
+        domain: meta.domain,
+        nonInteractive: opts.nonInteractive,
+    });
     log.info(`DNS adapter: ${bold(adapter.name)}`);
+    opts.onPhase("Waiting for DNS", `Set at your registrar: ${toApply.map((r) => `${r.type} ${r.hostname} → ${r.value}`).join("; ")}`);
     try {
         await adapter.applyRecords(toApply.map((r) => ({ hostname: r.hostname, type: r.type, value: r.value })));
         markStepDone(installId, stepName, { adapter: adapter.name });
@@ -1258,7 +1452,7 @@ async function stepGcpDnsRecords(installId) {
         process.exit(1);
     }
 }
-async function stepFinalize(installId) {
+async function stepFinalize(installId, setupLink) {
     const meta = readMetadata(installId);
     if (!meta?.projectId)
         return;
@@ -1280,6 +1474,20 @@ async function stepFinalize(installId) {
     const bootstrapUrl = provider === "gcp-firebase" && adminCloudRunUrl.length > 0
         ? `${adminCloudRunUrl}/setup?token=${bootstrapToken}`
         : stackBootstrapUrl;
+    // #696 — the install job's output lands in Cloud Logging: the link goes to
+    // Secret Manager, where the operator's terminal reads it (install-job.ts).
+    if (setupLink === "secret") {
+        if (!meta.region)
+            throw new Error(`install.json of ${installId} records no region.`);
+        const stored = await storeSetupLink(gcloud, { projectId: meta.projectId, region: meta.region }, bootstrapUrl);
+        if (!stored.ok) {
+            cancel(`Could not store the Owner setup link: ${stored.error}`);
+            process.exit(1);
+        }
+    }
+    const setupLinkLine = setupLink === "print"
+        ? `     ${cyan(String(bootstrapUrl))}`
+        : `     ${dim(`(shown in your Cloud Shell terminal; or: gcloud secrets versions access latest --secret=${INSTALL_SETUP_LINK_SECRET} --project=${meta.projectId})`)}`;
     // Both providers report their records in `dnsRecordsRequiredOut`; the
     // summary lists them in two groups so the main-domain records that take a
     // live site offline are not set during the install (dns/groups.ts). On gcp
@@ -1319,7 +1527,7 @@ async function stepFinalize(installId) {
         "",
         bold("Next steps:"),
         `  1. Owner setup — open in your browser (sign in with ${meta.ownerEmail}):`,
-        `     ${cyan(String(bootstrapUrl))}`,
+        setupLinkLine,
         `  2. AI provider — visit ${cyan(`${adminBase}/settings/ai`)} and paste your Anthropic API key`,
         `     (encrypted under the project KEK and stored in the database, no env vars).`,
         `  3. Optional: connect Claude Code or another MCP client — at ${cyan(`${adminBase}/settings/mcp`)}`,