@caelo-cms/provisioning 0.10.46 → 0.10.48

package/src/state-backend.ts

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * Where an install's Pulumi state and install metadata live (#696, D2).
+ *
+ * Two kinds:
+ *
+ *   - `local` — the original layout: Pulumi's DIY backend at
+ *     `file://~/.caelo-<id>/state`, secrets encrypted with the passphrase in
+ *     `~/.caelo-<id>/secrets/pulumi-passphrase`. Only the machine that
+ *     installed can upgrade or re-run the install. Self-hosted always stays
+ *     here: its state is on the host, next to the install.
+ *   - `gcs` — the install's own GCP project holds everything: a versioned,
+ *     uniform-access, public-access-prevented bucket `<project>-caelo-ops`
+ *     (`pulumi/` for Pulumi's DIY backend, `install/` for install.json and
+ *     progress.json) and a Cloud KMS key `caelo/pulumi-state` as Pulumi's
+ *     secrets provider. No passphrase exists. Any machine (laptop, Cloud
+ *     Shell, an in-cloud job) the operator is signed in on can run the
+ *     install's lifecycle commands after `cms-provision attach`.
+ *
+ * install.json records the kind in `stateBackend`; an install without the
+ * field predates #696 and is `local`. Every Pulumi call resolves its backend
+ * URL, secrets provider and environment through {@link resolveBackend} — no
+ * other code builds a `file://` URL or reads the passphrase.
+ */
+
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { selfCommand } from "./self-command.js";
+
+/** Name of the Cloud KMS key ring holding Caelo's keys in the install project. */
+export const OPS_KEY_RING = "caelo";
+/** Name of the Cloud KMS key Pulumi encrypts the stack's secrets with. */
+export const OPS_STATE_KEY = "pulumi-state";
+/** Prefix inside the ops bucket that Pulumi's DIY backend uses. */
+export const OPS_PULUMI_PREFIX = "pulumi";
+/** Prefix inside the ops bucket that holds install.json + progress.json. */
+export const OPS_INSTALL_PREFIX = "install";
+
+/** What install.json records about the install's state backend. */
+export type StateBackendRecord =
+  | { readonly kind: "local" }
+  | {
+      readonly kind: "gcs";
+      /** The ops bucket, without `gs://`. */
+      readonly bucket: string;
+      /** Full resource name of the KMS key (`projects/…/cryptoKeys/pulumi-state`). */
+      readonly kmsKey: string;
+    };
+
+/** The ops-bucket variant of {@link StateBackendRecord}. */
+export type GcsBackendRecord = Extract<StateBackendRecord, { kind: "gcs" }>;
+
+/** The backend a Pulumi command runs against, ready to pass to the CLI. */
+export interface ResolvedBackend {
+  readonly kind: StateBackendRecord["kind"];
+  /** `PULUMI_BACKEND_URL`. */
+  readonly url: string;
+  /** What `pulumi stack init/migrate --secrets-provider` takes. */
+  readonly secretsProvider: string;
+  /** The environment Pulumi needs for this backend (and nothing else). */
+  readonly envVars: Readonly<Record<string, string>>;
+}
+
+/** `<project>-caelo-ops` — the install's ops bucket (project ids are globally unique). */
+export function opsBucketName(projectId: string): string {
+  return `${projectId}-caelo-ops`;
+}
+
+/** Full resource name of the install's Pulumi state key. */
+export function opsKmsKeyName(projectId: string, region: string): string {
+  return `projects/${projectId}/locations/${region}/keyRings/${OPS_KEY_RING}/cryptoKeys/${OPS_STATE_KEY}`;
+}
+
+/** The `gcs` record for an install in `projectId`, its ops resources in `region`. */
+export function gcsBackendRecord(projectId: string, region: string): GcsBackendRecord {
+  return {
+    kind: "gcs",
+    bucket: opsBucketName(projectId),
+    kmsKey: opsKmsKeyName(projectId, region),
+  };
+}
+
+/** The install's recorded backend; installs that predate #696 are `local`. */
+export function installBackend(meta: { readonly stateBackend?: StateBackendRecord }) {
+  return meta.stateBackend ?? ({ kind: "local" } as const);
+}
+
+const BUCKET = /^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$/;
+const KMS_KEY =
+  /^projects\/[a-z][a-z0-9.:-]{0,61}[a-z0-9]\/locations\/[a-z0-9-]+\/keyRings\/[A-Za-z0-9_-]+\/cryptoKeys\/[A-Za-z0-9_-]+$/;
+
+/**
+ * Validate a `stateBackend` value read from install.json (on disk or in the
+ * bucket). A malformed record would point Pulumi at the wrong place, so it
+ * throws instead of guessing.
+ */
+export function parseStateBackend(value: unknown): StateBackendRecord | undefined {
+  if (value === undefined) return undefined;
+  const v = value as Record<string, unknown> | null;
+  if (v && v.kind === "local") return { kind: "local" };
+  if (
+    v &&
+    v.kind === "gcs" &&
+    typeof v.bucket === "string" &&
+    BUCKET.test(v.bucket) &&
+    typeof v.kmsKey === "string" &&
+    KMS_KEY.test(v.kmsKey)
+  ) {
+    return { kind: "gcs", bucket: v.bucket, kmsKey: v.kmsKey };
+  }
+  throw new Error(
+    `install.json stateBackend is malformed (${JSON.stringify(value)}). Expected {"kind":"local"} or {"kind":"gcs","bucket":"<project>-caelo-ops","kmsKey":"projects/…/cryptoKeys/${OPS_STATE_KEY}"}.`,
+  );
+}
+
+/** Where a `local` install keeps Pulumi's state. */
+export function localStateDir(installRoot: string): string {
+  return join(installRoot, "state");
+}
+
+/**
+ * Resolve the backend URL, secrets provider and environment for an install.
+ *
+ * `local` needs the passphrase and fails loudly without it — Pulumi would
+ * otherwise prompt for it on a terminal nobody sees. `gcs` needs no secret:
+ * Pulumi reaches the bucket and the KMS key with the operator's Application
+ * Default Credentials. Both set `PULUMI_FALLBACK_TO_STATE_SECRETS_MANAGER`:
+ * Pulumi then takes the secrets provider (KMS key or passphrase salt) from
+ * the install's own state when the stack's config file in the package's
+ * stack dir names none (wizards/gcp-pulumi.ts alignedStackSettings).
+ *
+ * @param meta the install (its `stateBackend`, absent = local)
+ * @param opts.installRoot `~/.caelo-<id>`
+ * @param opts.readPassphrase reads `secrets/pulumi-passphrase`; called for local only
+ */
+export function resolveBackend(
+  meta: { readonly installId: string; readonly stateBackend?: StateBackendRecord },
+  opts: { readonly installRoot: string; readonly readPassphrase: () => string | null },
+): ResolvedBackend {
+  const record = installBackend(meta);
+  if (record.kind === "gcs") {
+    const url = `gs://${record.bucket}/${OPS_PULUMI_PREFIX}`;
+    return {
+      kind: "gcs",
+      url,
+      secretsProvider: `gcpkms://${record.kmsKey}`,
+      envVars: {
+        PULUMI_BACKEND_URL: url,
+        PULUMI_FALLBACK_TO_STATE_SECRETS_MANAGER: "true",
+      },
+    };
+  }
+  const passphrase = opts.readPassphrase();
+  if (!passphrase) {
+    throw new Error(
+      `~/.caelo-${meta.installId}/secrets/pulumi-passphrase is missing, so the install's local Pulumi state can't be decrypted. Restore it from your backup of ~/.caelo-${meta.installId}/.`,
+    );
+  }
+  const url = `file://${localStateDir(opts.installRoot)}`;
+  return {
+    kind: "local",
+    url,
+    secretsProvider: "passphrase",
+    envVars: {
+      PULUMI_BACKEND_URL: url,
+      PULUMI_CONFIG_PASSPHRASE: passphrase,
+      PULUMI_FALLBACK_TO_STATE_SECRETS_MANAGER: "true",
+    },
+  };
+}
+
+/**
+ * The shell prefix that points a manual `pulumi …` command at the install's
+ * backend, for recovery instructions the CLI prints. Never contains the
+ * passphrase itself: a local install reads it from its file at run time.
+ */
+export function backendShellEnv(
+  meta: { readonly stateBackend?: StateBackendRecord },
+  installRoot: string,
+): string {
+  const record = installBackend(meta);
+  if (record.kind === "gcs") {
+    return `PULUMI_BACKEND_URL=gs://${record.bucket}/${OPS_PULUMI_PREFIX} PULUMI_FALLBACK_TO_STATE_SECRETS_MANAGER=true`;
+  }
+  return `PULUMI_BACKEND_URL=file://${localStateDir(installRoot)} PULUMI_CONFIG_PASSPHRASE="$(cat ${join(installRoot, "secrets", "pulumi-passphrase")})" PULUMI_FALLBACK_TO_STATE_SECRETS_MANAGER=true`;
+}
+
+/** Human-readable location of the install's state, for status and progress output. */
+export function describeBackend(
+  meta: { readonly stateBackend?: StateBackendRecord },
+  installRoot: string,
+): string {
+  const record = installBackend(meta);
+  return record.kind === "gcs"
+    ? `gs://${record.bucket} (secrets: Cloud KMS ${record.kmsKey.split("/").slice(-3).join("/")})`
+    : `${localStateDir(installRoot)} on this machine (secrets: passphrase file)`;
+}
+
+/**
+ * Whether a local Pulumi backend dir holds any stack (Pulumi's DIY layout
+ * keeps them under `.pulumi/stacks/`).
+ */
+export function hasLocalPulumiState(installRoot: string): boolean {
+  const stacks = join(localStateDir(installRoot), ".pulumi", "stacks");
+  if (!existsSync(stacks)) return false;
+  // An unreadable dir throws: guessing "no state" would move a deployed
+  // install onto an empty backend.
+  return readdirSync(stacks).length > 0;
+}
+
+/**
+ * Which backend the GCP wizard runs an install on (#696). A recorded
+ * backend wins. An install without one whose state already sits on this
+ * machine stays local — moving state needs a preflight, a backup and a
+ * verification, never a side effect of a re-run.
+ * Everything else (a new GCP install) gets the ops bucket. Self-hosted, AWS
+ * and Azure stay local until their adapters have an equivalent.
+ */
+export function chooseInstallBackend(
+  meta: {
+    readonly installId?: string;
+    readonly provider: string;
+    readonly stateBackend?: StateBackendRecord;
+  },
+  localStateExists: boolean,
+  deployed = false,
+): StateBackendRecord["kind"] {
+  if (meta.stateBackend) return meta.stateBackend.kind;
+  if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") return "local";
+  if (localStateExists) return "local";
+  if (deployed) {
+    // Deployed, yet no state anywhere: a fresh backend would make Pulumi
+    // recreate (and collide with) every resource.
+    throw new Error(
+      `Install ${meta.installId ?? ""} was deployed, but its Pulumi state is neither recorded in install.json nor in ~/.caelo-${meta.installId ?? "<install-id>"}/state. Restore that directory from your backup and re-run.`,
+    );
+  }
+  return "gcs";
+}
+
+/**
+ * What `upgrade` tells the operator about an install whose Pulumi state is
+ * still on this machine (#696), or null when it lives in the install's own
+ * cloud. `upgrade` never moves state itself: the move has its own
+ * preflight, backup and verification (`state migrate`).
+ */
+export function localStateNotice(meta: {
+  readonly installId: string;
+  readonly provider: string;
+  readonly stateBackend?: StateBackendRecord;
+}): string | null {
+  if (installBackend(meta).kind !== "local") return null;
+  if (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") return null;
+  return [
+    `This install's Pulumi state and passphrase live only on this machine (~/.caelo-${meta.installId}/).`,
+    "Lose this machine and the install's infrastructure can no longer be converged from anywhere.",
+    "Move them into the install's own GCP project (versioned bucket + Cloud KMS key, < $0.10/mo):",
+    `  ${selfCommand("state", "migrate", "--install", meta.installId, "--dry-run")}`,
+    `  ${selfCommand("state", "migrate", "--install", meta.installId)}`,
+  ].join("\n");
+}

package/src/state-command.ts

@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * CLI wiring for `cms-provision state migrate` and `cms-provision attach`
+ * (#696): parses the flags, picks the install, and hands the engines
+ * (state-migrate.ts, state-attach.ts) the real gcloud, Pulumi and file
+ * system.
+ */
+
+import { spawnSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { confirm, isCancel, log, note } from "@clack/prompts";
+import { bold, red } from "kleur/colors";
+import { activeAccount } from "./gcloud.js";
+import {
+  adoptStateBackend,
+  currentMetadata,
+  ensureInstallDir,
+  installFlag,
+  installRoot,
+  installStagingHost,
+  listInstalls,
+  readSecret,
+  recordedImageDigests,
+  selectInstall,
+  writeSecret,
+} from "./install-state.js";
+import { installIapAllowlist } from "./operator-access.js";
+import { ensureOpsBackend } from "./ops-backend.js";
+import { provisionerServiceAccountEmail } from "./provisioner-identity.js";
+import { selfCommand } from "./self-command.js";
+import { attachInstall } from "./state-attach.js";
+import {
+  gcsBackendRecord,
+  hasLocalPulumiState,
+  installBackend,
+  localStateDir,
+  parseStateBackend,
+  resolveBackend,
+} from "./state-backend.js";
+import { migrateState } from "./state-migrate.js";
+import { realStatePulumiOps } from "./state-pulumi.js";
+import { acquireStateLock, GcsStateStore, realSyncGcloud } from "./state-store.js";
+import { clackReporter } from "./upgrade/reporter.js";
+import { GCP_STACK_KNOB_DEFAULTS } from "./wizards/gcp-pulumi.js";
+
+/** Where install-dir archives go: `~/.caelo-backups/` (mode 700). */
+export function backupsDir(home: string = homedir()): string {
+  return join(home, ".caelo-backups");
+}
+
+/**
+ * Archive `~/.caelo-<id>/` (secrets and state included) to
+ * `~/.caelo-backups/<id>-<time>.tar.zst`, or `.tar.gz` when `zstd` is not
+ * installed. The archive and its directory are private (600 / 700): it
+ * holds the passphrase. Throws when the archive can't be made.
+ *
+ * @returns the archive's path
+ */
+export function backupInstallDir(
+  installId: string,
+  opts: { home?: string; now?: Date; hasZstd?: boolean } = {},
+): string {
+  const home = opts.home ?? homedir();
+  const dir = backupsDir(home);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true, mode: 0o700 });
+  chmodSync(dir, 0o700);
+  const stamp = (opts.now ?? new Date()).toISOString().replace(/[:.]/g, "-");
+  const base = join(dir, `${installId}-${stamp}`);
+  const zstd = opts.hasZstd ?? spawnSync("zstd", ["--version"], { stdio: "ignore" }).status === 0;
+  const source = ["-C", home, `.caelo-${installId}`];
+  const run = (cmd: string, args: string[]) => {
+    const r = spawnSync(cmd, args, { stdio: ["ignore", "ignore", "pipe"], encoding: "utf8" });
+    if (r.status !== 0) {
+      throw new Error(
+        `${cmd} failed while archiving ~/.caelo-${installId}: ${(r.stderr ?? "").trim()}`,
+      );
+    }
+  };
+  if (zstd) {
+    const tar = `${base}.tar`;
+    try {
+      run("tar", ["-cf", tar, ...source]);
+      chmodSync(tar, 0o600);
+      run("zstd", ["-q", "--rm", "-o", `${tar}.zst`, tar]);
+    } finally {
+      rmSync(tar, { force: true });
+    }
+    chmodSync(`${tar}.zst`, 0o600);
+    return `${tar}.zst`;
+  }
+  const out = `${base}.tar.gz`;
+  run("tar", ["-czf", out, ...source]);
+  chmodSync(out, 0o600);
+  return out;
+}
+
+/** The value of `--<name> <v>` / `--<name>=<v>` on argv. */
+function flagValue(argv: readonly string[], name: string): string | undefined {
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i] ?? "";
+    if (a === `--${name}`) return argv[i + 1];
+    if (a.startsWith(`--${name}=`)) return a.slice(name.length + 3);
+  }
+  return undefined;
+}
+
+const USAGE =
+  "Usage: cms-provision state migrate [--install <install-id or domain>] [--to remote|local] [--dry-run] [--yes]";
+
+/** `cms-provision state migrate …`. */
+export async function stateCommand(argv: readonly string[]): Promise<void> {
+  if (argv[3] !== "migrate") {
+    console.log(USAGE);
+    process.exit(2);
+  }
+  const to = flagValue(argv, "to") ?? "remote";
+  if (to !== "remote" && to !== "local") {
+    console.error(`--to must be remote or local.\n${USAGE}`);
+    process.exit(2);
+  }
+  const dryRun = argv.includes("--dry-run");
+  const selected = selectInstall(listInstalls(), installFlag(argv));
+  if (!selected.ok) {
+    console.error(selected.message);
+    process.exit(1);
+  }
+  const meta = currentMetadata(selected.meta);
+  const { installId } = meta;
+  const root = installRoot(installId);
+  if (
+    (meta.provider !== "gcp" && meta.provider !== "gcp-firebase") ||
+    !meta.projectId ||
+    !meta.region
+  ) {
+    log.error(
+      red(
+        `${installId} (${meta.provider}) keeps its state on the host or has no project/region recorded; state migrate is for gcp and gcp-firebase installs.`,
+      ),
+    );
+    process.exit(1);
+  }
+  const { projectId, region, provider } = meta;
+  const imageDigests = recordedImageDigests(meta);
+  if (!imageDigests) {
+    log.error(
+      red(
+        `install.json of ${installId} records no image digests. Run ${bold(selfCommand("upgrade", "--install", installId))} first, then migrate.`,
+      ),
+    );
+    process.exit(1);
+  }
+  const operator = await activeAccount();
+  if (!operator) {
+    log.error(red("No active gcloud account. Run `gcloud auth login`, then re-run."));
+    process.exit(1);
+  }
+  const provisioner = provisionerServiceAccountEmail(projectId);
+  note(
+    [
+      `Install:   ${bold(meta.domain)} (${installId}, project ${projectId}, ${region})`,
+      `Direction: ${to === "remote" ? `this machine → gs://${gcsBackendRecord(projectId, region).bucket} + Cloud KMS` : "the install's bucket → this machine"}`,
+      dryRun
+        ? "Dry run:   nothing will be changed"
+        : "The source stays in use until the copy is verified.",
+    ].join("\n"),
+    "state migrate",
+  );
+
+  const result = await migrateState(
+    meta,
+    { to, dryRun },
+    {
+      pulumi: realStatePulumiOps({
+        provider,
+        installRoot: root,
+        impersonateServiceAccount: provisioner,
+        stackConfig: {
+          ...GCP_STACK_KNOB_DEFAULTS,
+          provider,
+          projectId,
+          domain: meta.domain,
+          stagingDomain: installStagingHost(meta),
+          ownerEmail: meta.ownerEmail,
+          region,
+          iapAllowlist: installIapAllowlist(meta.ownerEmail),
+          imageDigests,
+        },
+      }),
+      report: clackReporter(),
+      resolve: (record) =>
+        resolveBackend(
+          { installId, stateBackend: record },
+          { installRoot: root, readPassphrase: () => readSecret(installId, "pulumi-passphrase") },
+        ),
+      confirmCost: async (costNote) => {
+        log.info(costNote);
+        if (argv.includes("--yes")) return true;
+        if (!process.stdin.isTTY) {
+          log.error(red("Not a terminal: pass --yes to accept the cost above."));
+          return false;
+        }
+        const yes = await confirm({ message: "Create them and continue?", initialValue: true });
+        return !isCancel(yes) && yes === true;
+      },
+      ensureRemote: (dry) =>
+        ensureOpsBackend(
+          { projectId, region, operatorAccount: operator, provisionerEmail: provisioner },
+          { dryRun: dry },
+        ),
+      backup: async () => backupInstallDir(installId),
+      localStateEmpty: () => !hasLocalPulumiState(root),
+      prepareLocalTarget: () => {
+        mkdirSync(localStateDir(root), { recursive: true, mode: 0o700 });
+        if (!readSecret(installId, "pulumi-passphrase")) {
+          writeSecret(installId, "pulumi-passphrase", randomBytes(32).toString("hex"));
+        }
+      },
+      lock: async () => {
+        // The source's lock (local file or bucket object) keeps installer
+        // runs out; the bucket's also keeps out a migration started on
+        // another machine when the bucket is the target.
+        const source = meta.stateBackend ?? { kind: "local" as const };
+        const releases = [acquireStateLock(source, root, "state migrate")];
+        if (source.kind === "local" && to === "remote") {
+          try {
+            releases.push(
+              acquireStateLock(gcsBackendRecord(projectId, region), root, "state migrate"),
+            );
+          } catch (e) {
+            for (const r of releases) r();
+            throw e;
+          }
+        }
+        return async () => {
+          for (const r of releases.reverse()) r();
+        };
+      },
+      remoteRecordedBackend: () => {
+        const text = new GcsStateStore(gcsBackendRecord(projectId, region).bucket, root).peek(
+          "install.json",
+        );
+        if (text === null) return null;
+        return installBackend({ stateBackend: parseStateBackend(JSON.parse(text).stateBackend) })
+          .kind;
+      },
+      adopt: (record) => {
+        adoptStateBackend(installId, record);
+      },
+      retireLocalState: () => {
+        const dir = localStateDir(root);
+        if (!existsSync(dir)) return null;
+        const moved = `${dir}.migrated-${new Date().toISOString().replace(/[:.]/g, "-")}`;
+        renameSync(dir, moved);
+        return moved;
+      },
+    },
+  );
+  if (!result.ok) {
+    log.error(red(result.error));
+    process.exit(1);
+  }
+}
+
+/** `cms-provision attach --project <id>`. */
+export async function attachCommand(argv: readonly string[]): Promise<void> {
+  const projectId = flagValue(argv, "project");
+  if (!projectId || projectId.startsWith("-")) {
+    console.log("Usage: cms-provision attach --project <gcp-project-id>");
+    process.exit(2);
+  }
+  const r = attachInstall(projectId, {
+    run: realSyncGcloud,
+    rootFor: installRoot,
+    hasLocalState: hasLocalPulumiState,
+  });
+  if (!r.ok) {
+    log.error(red(r.error));
+    process.exit(1);
+  }
+  ensureInstallDir(r.installId);
+  log.success(
+    r.status === "refreshed"
+      ? `${r.domain} was already attached; its local copy of install.json is up to date.`
+      : `Attached ${bold(r.domain)} (${r.installId}). status, upgrade, backup and re-running the installer now work from this machine.`,
+  );
+  log.info(
+    "Infrastructure runs need Application Default Credentials once on this machine: gcloud auth application-default login",
+  );
+}