@caelo-cms/provisioning 0.11.9 → 0.11.11

package/src/converge-db-trust.ts

@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #1373 — where the converge job (#819) opens its database queue, and whom
+ * it trusts there, BEFORE the install's database TLS step (#1338) ran.
+ *
+ * The job writes the update timeline and the request's end through that
+ * queue. Its code connects with verify-full only, which needs the
+ * instance's DNS name, and on an install whose services still dial the
+ * instance's IP the DNS name exists only once the TLS step has run. When
+ * the job ran that step before opening the queue, a failure of the step
+ * reached only the job's log: Settings → Updates showed "Interrupted" and
+ * nothing else. So the job first opens the queue on the host the install
+ * dials today and runs the TLS step as a timeline step of its own
+ * (converge-job.ts `secureDatabase`).
+ *
+ * Dialling an IP, the host-name half of verify-full cannot hold (the
+ * certificate names no IP). The connection must still authenticate the
+ * server: the job's password is the provisioner's access token, worth far
+ * more than a database password, so `sslmode=require` (encrypted, server
+ * unchecked) is not an option. The trust comes from the Cloud SQL Admin
+ * API, over an authenticated channel, about this one instance:
+ *
+ *   - per-instance CA (`GOOGLE_MANAGED_INTERNAL_CA`, the default before
+ *     #1338): the instance's own server CA (`serverCaCert`). Nobody else
+ *     holds a certificate from it, so a verified chain is this instance.
+ *   - shared CA (`GOOGLE_MANAGED_CAS_CA`): always pinned, because the
+ *     shared CA signs other customers' instances too. With its certificate
+ *     active: Google's shared-CA bundle (the one every image ships) and the
+ *     fingerprints `listServerCertificates` names; not rotated in yet: the
+ *     one certificate the API names active. A listing that does not name
+ *     the active certificate with a well-formed fingerprint is refused —
+ *     fail closed, never a guess.
+ *   - any other CA mode: refused.
+ *
+ * No input but the metadata server's placement and the APIs: the job's
+ * scrubbed-environment contract (§11.C) is unchanged.
+ */
+
+import { readFileSync } from "node:fs";
+import { CLOUD_SQL_CA_BUNDLE, isIpLiteral } from "@caelo-cms/shared";
+import { z } from "zod";
+import { describeInstance } from "./cloudsql-tls.js";
+import { findSqlInstance } from "./gateway-credentials.js";
+import { gcloud } from "./gcloud.js";
+import type { GcloudRunner } from "./gcloud-retry.js";
+import { GCP_STACK_ENV } from "./gcp-names.js";
+import type { HttpFetch } from "./runtime-secrets.js";
+import { CLOUD_SQL_SERVER_CA_MODE } from "./stack-contract.js";
+import { type UpgraderPlacement, upgraderDatabaseHost } from "./upgrader-gcp.js";
+
+/** How the converge job reaches the database. */
+export type ConvergeDatabaseTarget =
+  /** The services dial the DNS name already: verify-full, nothing to set up. */
+  | { readonly kind: "dns"; readonly host: string }
+  /**
+   * The services dial the instance's IP: connect there, verify the chain
+   * against `caPem` and, when `pinnedSha1` is set, the certificate's
+   * fingerprint (lower-case hex, no colons) against it.
+   */
+  | {
+      readonly kind: "ip";
+      readonly host: string;
+      readonly caPem: string;
+      readonly pinnedSha1: readonly string[] | null;
+    };
+
+export interface ConvergeDatabaseTargetDeps {
+  readonly run?: GcloudRunner;
+  readonly http?: HttpFetch;
+  /** The shipped shared-CA bundle (tests pass their own). */
+  readonly readSharedCaBundle?: () => string;
+}
+
+/** A SHA-1 fingerprint as hex without separators, lower case (`AB:CD…` and `abcd…` compare equal). */
+export function normalizedSha1(fingerprint: string): string {
+  return fingerprint.replace(/[^0-9a-f]/gi, "").toLowerCase();
+}
+
+/**
+ * The host the install dials today and, when it is the instance's IP, the
+ * server identity the converge job's connection must verify there. Throws
+ * with what could not be read: without it the job must not connect.
+ */
+export async function convergeDatabaseTarget(
+  placement: Pick<UpgraderPlacement, "projectId" | "region">,
+  deps: ConvergeDatabaseTargetDeps = {},
+): Promise<ConvergeDatabaseTarget> {
+  const run = deps.run ?? gcloud;
+  const host = await upgraderDatabaseHost(placement, run);
+  if (!isIpLiteral(host)) return { kind: "dns", host };
+  const install = { projectId: placement.projectId, env: GCP_STACK_ENV };
+  const lookup = await findSqlInstance(install, { run });
+  if (lookup.status !== "found") {
+    throw new Error(`could not find the install's Cloud SQL instance (${lookup.status})`);
+  }
+  const target = { ...install, sqlInstance: lookup.name };
+  const described = await describeInstance(target, run);
+  if (!described.ok) throw new Error(described.error);
+  const d = described.d;
+  const mode = d.settings?.ipConfiguration?.serverCaMode;
+  const ownCa = d.serverCaCert?.cert?.trim() || null;
+  if (mode === CLOUD_SQL_SERVER_CA_MODE) {
+    const listing = await serverCertificateListing(
+      target,
+      run,
+      deps.http ?? (fetch as unknown as HttpFetch),
+    );
+    // Fail closed: once the shared CA may be active, every connection is
+    // pinned to a certificate the API names — the shared CA alone would
+    // also vouch for other customers' instances.
+    const sharedBundle = (
+      deps.readSharedCaBundle ?? (() => readFileSync(CLOUD_SQL_CA_BUNDLE, "utf8"))
+    )();
+    if (listing.fingerprints.includes(listing.active)) {
+      return { kind: "ip", host, caPem: sharedBundle, pinnedSha1: listing.fingerprints };
+    }
+    // Moved to the shared CA, its certificate not rotated in yet: the
+    // server still presents the active one from its own CA. Pinned to
+    // exactly that certificate, so which CA vouches for it does not widen
+    // what is accepted.
+    const caPem = [ownCa, sharedBundle].filter((c): c is string => Boolean(c)).join("\n");
+    return { kind: "ip", host, caPem, pinnedSha1: [listing.active] };
+  }
+  // Per-instance CA (the default, which older instances do not name):
+  // nobody else holds a certificate from it, so the verified chain is this
+  // instance. Any other mode (a customer-managed CA) is not one this job
+  // can reason about.
+  if (mode !== undefined && mode !== PER_INSTANCE_CA_MODE) {
+    throw new Error(
+      `${lookup.name} uses the server CA mode ${mode}, which the converge job cannot verify a connection to ${host} for`,
+    );
+  }
+  if (!ownCa) {
+    throw new Error(
+      `${lookup.name} names no server CA certificate (serverCaCert), so the connection to ${host} could not be verified`,
+    );
+  }
+  return { kind: "ip", host, caPem: ownCa, pinnedSha1: null };
+}
+
+/** Cloud SQL's default per-instance server CA mode. */
+const PER_INSTANCE_CA_MODE = "GOOGLE_MANAGED_INTERNAL_CA";
+
+const SHA1_HEX = /^[0-9a-f]{40}$/;
+
+/**
+ * The Admin API's `listServerCertificates` answer, validated: the active
+ * certificate's fingerprint and every listed one, normalized. Throws on
+ * anything else — a missing `activeVersion`, a certificate without a
+ * well-formed fingerprint — since a guessed listing would be a guessed
+ * trust decision.
+ */
+async function serverCertificateListing(
+  t: { readonly projectId: string; readonly sqlInstance: string },
+  run: GcloudRunner,
+  http: HttpFetch,
+): Promise<{ readonly active: string; readonly fingerprints: readonly string[] }> {
+  const token = await run(["auth", "print-access-token"]);
+  const bearer = token.ok ? token.stdout.trim() : "";
+  if (!bearer) throw new Error(`gcloud auth print-access-token: ${token.stderr.trim()}`);
+  const res = await http(
+    `https://sqladmin.googleapis.com/v1/projects/${encodeURIComponent(t.projectId)}/instances/${encodeURIComponent(t.sqlInstance)}/listServerCertificates`,
+    { method: "GET", headers: { Authorization: `Bearer ${bearer}` } },
+  );
+  const text = await res.text();
+  if (!res.ok) {
+    throw new Error(`list server certificates: HTTP ${res.status} ${text.slice(0, 300)}`);
+  }
+  let body: unknown;
+  try {
+    body = JSON.parse(text);
+  } catch {
+    throw new Error("list server certificates: the Cloud SQL Admin API answered no JSON");
+  }
+  const parsed = ListingSchema.safeParse(body);
+  if (!parsed.success) {
+    throw new Error(
+      `list server certificates: the answer does not name the active certificate and the listed ones with SHA-1 fingerprints (${parsed.error.issues[0]?.message ?? "invalid"}), so the connection is not trusted`,
+    );
+  }
+  return {
+    active: parsed.data.activeVersion,
+    fingerprints: parsed.data.serverCerts.map((c) => c.sha1Fingerprint),
+  };
+}
+
+const Sha1 = z
+  .string()
+  .transform(normalizedSha1)
+  .refine((f) => SHA1_HEX.test(f), "a SHA-1 fingerprint");
+
+const ListingSchema = z.object({
+  activeVersion: Sha1,
+  // Before the first shared-CA certificate exists the API may list none.
+  serverCerts: z.array(z.object({ sha1Fingerprint: Sha1 })).default([]),
+});
+
+/**
+ * Why a server certificate with `fingerprint` (as `node:tls` prints it) is
+ * not the instance `target` names, or null when it is. Only for an `ip`
+ * target; the chain itself is checked against `caPem` by the TLS layer.
+ */
+export function serverIdentityProblem(
+  target: Extract<ConvergeDatabaseTarget, { kind: "ip" }>,
+  fingerprint: string,
+): string | null {
+  if (target.pinnedSha1 === null) return null;
+  return target.pinnedSha1.includes(normalizedSha1(fingerprint))
+    ? null
+    : `the database at ${target.host} presented a certificate (SHA-1 ${normalizedSha1(fingerprint)}) that the Cloud SQL Admin API does not list for this install's instance`;
+}

package/src/converge-job-gcp.ts

@@ -42,7 +42,7 @@ import {
 } from "./install-state.js";
 import { runMigrationsViaCloudRunJob } from "./migration-runner.js";
 import { installIapAllowlist } from "./operator-access.js";
-import { fetchVerifiedReleaseManifest } from "./release-manifest.js";
+import { fetchVerifiedReleaseManifest, manifestServiceImages } from "./release-manifest.js";
 import type { HttpFetch } from "./runtime-secrets.js";
 import { publicSiteUrl, STAGING_HOST_ENV } from "./stack-contract.js";
 import { attachInstall } from "./state-attach.js";
@@ -203,10 +203,14 @@ export function startedExecutionName(stdout: string, stderr: string): string | n
   return /^[a-z0-9-]+$/.test(name) ? name : null;
 }
 
-/** Every dependency of {@link runConvergeJob} except the database and the log. */
+/**
+ * Every dependency of {@link runConvergeJob} except the database (the queue
+ * and how it is secured, #1373) and the log.
+ */
 export function gcpConvergeDeps(
   placement: UpgraderPlacement,
   queue: ConvergeQueue,
+  secureDatabase: ConvergeJobDeps["secureDatabase"],
   log: JobLog,
 ): ConvergeJobDeps {
   const run: GcloudRunner = gcloud;
@@ -300,6 +304,9 @@ export function gcpConvergeDeps(
         domain: meta.domain,
         stagingHost: installStagingHost(meta),
         ownerEmail: meta.ownerEmail,
+        ...(meta.schedulerIntervalMinutes !== undefined
+          ? { schedulerIntervalMinutes: meta.schedulerIntervalMinutes }
+          : {}),
         targetTag: m.version,
         // This code is the target release's (stage 2), so its contract is it.
         infraContractVersion: INFRA_CONTRACT_VERSION,
@@ -314,16 +321,7 @@ export function gcpConvergeDeps(
         sleep: realSleep,
         reporter,
         // The signed manifest's digests — never a registry tag lookup.
-        resolveImages: async () => ({
-          ok: true,
-          images: {
-            admin: { digest: m.digests.admin, imageRef: releaseImageRef("admin", m.digests.admin) },
-            gateway: {
-              digest: m.digests.gateway,
-              imageRef: releaseImageRef("gateway", m.digests.gateway),
-            },
-          },
-        }),
+        resolveImages: async () => manifestServiceImages(m),
         verifyImages: verifyCosignAll,
         migrate: (imageRef) =>
           runMigrationsViaCloudRunJob(
@@ -364,6 +362,7 @@ export function gcpConvergeDeps(
 
   return {
     queue,
+    secureDatabase,
     projectId,
     region,
     iam: gcloudIamTester(run, http),

package/src/converge-job-preflight.test.ts

@@ -43,6 +43,11 @@ const PREDEFINED: Record<string, readonly string[]> = {
     "cloudscheduler.jobs.get",
     "cloudscheduler.jobs.update",
   ],
+  "roles/workflows.editor": [
+    "workflows.workflows.create",
+    "workflows.workflows.get",
+    "workflows.workflows.update",
+  ],
   "roles/logging.logWriter": ["logging.logEntries.create"],
 };
 

package/src/converge-job-preflight.ts

@@ -62,6 +62,14 @@ export const INSTALL_ROLE_USES: Record<
       "cloudscheduler.jobs.update",
     ],
   },
+  "roles/workflows.editor": {
+    need: "deploy and update the workflow the scheduler tick runs (#1140)",
+    permissions: [
+      "workflows.workflows.create",
+      "workflows.workflows.get",
+      "workflows.workflows.update",
+    ],
+  },
   "roles/logging.logWriter": {
     need: "write the update's progress to Cloud Logging",
     permissions: ["logging.logEntries.create"],

package/src/converge-job.test.ts

@@ -19,6 +19,7 @@ import {
   type ConvergeQueue,
   type ConvergeUpgradeOutcome,
   runConvergeJob,
+  type SecureDatabaseResult,
 } from "./converge-job.js";
 import type { ExecutionState } from "./install-job.js";
 import type { InstallMetadata } from "./install-state.js";
@@ -147,6 +148,8 @@ function harness(
     imageSigned?: boolean;
     /** What IAM answers the self-check; every permission asked for by default. */
     iam?: IamTester;
+    /** #1373 — the database TLS step's answer; null: the queue dials the DNS name already. */
+    secureDatabase?: SecureDatabaseResult | null;
   } = {},
 ): Harness {
   const calls: string[] = [];
@@ -156,6 +159,13 @@ function harness(
   let stored = opts.checkpoint ?? null;
   const deps: ConvergeJobDeps = {
     queue: q.queue,
+    secureDatabase:
+      opts.secureDatabase === undefined || opts.secureDatabase === null
+        ? null
+        : async () => {
+            calls.push("secure database");
+            return opts.secureDatabase as SecureDatabaseResult;
+          },
     projectId: "acme",
     region: "europe-west1",
     iam: opts.iam ?? {
@@ -556,6 +566,66 @@ describe("runConvergeJob — resume after a killed run", () => {
   });
 });
 
+describe("runConvergeJob — the database TLS step (#1373)", () => {
+  const DNS = "abc.p-label.europe-west1.sql-psa.goog";
+
+  it("a failed TLS step reaches the timeline and ends the request, before any roll", async () => {
+    const h = harness({
+      secureDatabase: {
+        ok: false,
+        error:
+          "database TLS could not be set up: caelo-production-pg-1 reports no PRIVATE_SERVICES_ACCESS DNS name",
+      },
+    });
+    const r = await runConvergeJob(h.deps);
+    expect(r.exitCode).toBe(1);
+    expect(r.status).toBe("failed");
+    expect(h.upgrades).toEqual([]);
+    // Under the lock, after the resumable checkpoint, released at the end.
+    expect(h.calls).toEqual([
+      "manifest 0.11.0",
+      "lock",
+      "checkpoint running",
+      "secure database",
+      "checkpoint failed",
+      "unlock",
+    ]);
+    const end = h.q.ends[0];
+    expect(end?.codes).toEqual(["database-tls"]);
+    expect(end?.reason).toContain("reports no PRIVATE_SERVICES_ACCESS DNS name");
+    // The step's start and its error are timeline events of the request.
+    const converge = h.q.events.filter((e) => e.phase === "converge");
+    expect(converge[0]?.message).toStartWith("Setting up verified encryption");
+    const error = h.q.events.find((e) => e.level === "error");
+    expect(error?.message).toContain("reports no PRIVATE_SERVICES_ACCESS DNS name");
+  });
+
+  it("a successful TLS step runs right before the upgrade and names the verified host", async () => {
+    const h = harness({ secureDatabase: { ok: true, host: DNS } });
+    const r = await runConvergeJob(h.deps);
+    expect(r.status).toBe("succeeded");
+    const at = h.calls.indexOf("secure database");
+    expect(at).toBeGreaterThan(h.calls.indexOf("lock"));
+    expect(h.calls[at + 1]).toBe("upgrade");
+    expect(h.q.events.some((e) => e.level === "success" && e.message.includes(DNS))).toBe(true);
+  });
+
+  it("stage 1 never runs it: the release's own installer changes the database", async () => {
+    const h = harness({ ownVersion: "0.10.53", secureDatabase: { ok: true, host: DNS } });
+    await runConvergeJob(h.deps);
+    expect(h.calls.includes("secure database")).toBe(false);
+  });
+
+  it("a stop before the lock leaves the database alone", async () => {
+    const h = harness({
+      secureDatabase: { ok: true, host: DNS },
+      manifest: { ok: false, error: { code: "signature", message: "no valid signature" } },
+    });
+    await runConvergeJob(h.deps);
+    expect(h.calls.includes("secure database")).toBe(false);
+  });
+});
+
 describe("runConvergeJob — nothing to do", () => {
   it("exits cleanly without a converge request", async () => {
     const h = harness({ claim: null });

package/src/converge-job.ts

@@ -116,8 +116,22 @@ export type ConvergeUpgradeOutcome =
       readonly unrecorded: boolean;
     };
 
+/** How {@link ConvergeJobDeps.secureDatabase} ended. */
+export type SecureDatabaseResult =
+  | { readonly ok: true; readonly host: string }
+  | { readonly ok: false; readonly error: string };
+
 export interface ConvergeJobDeps {
   readonly queue: ConvergeQueue;
+  /**
+   * #1373 — the install's database TLS step (#1338, cloudsql-tls.ts), and
+   * then {@link queue} moved onto the database's DNS name with verify-full.
+   * Null when the queue already dials that name. Until it ran the queue
+   * dials the instance's IP (converge-db-trust.ts), so a failure of the
+   * step reaches the timeline and the request's end like any other step;
+   * when it fails the queue is reopened on whatever the install dials then.
+   */
+  readonly secureDatabase: (() => Promise<SecureDatabaseResult>) | null;
   /** From the metadata server — never from env or argv. */
   readonly projectId: string;
   /** The region this job runs in (metadata server). */
@@ -463,6 +477,24 @@ export async function runConvergeJob(deps: ConvergeJobDeps): Promise<UpgraderRun
     save({});
 
     r.phase("converge");
+    // #1373 — stage 2 only, under the lock and after every check: it
+    // changes the database instance, its DNS and the services' URLs.
+    if (deps.secureDatabase) {
+      r.info(
+        "Setting up verified encryption for the database connection (the database's certificate and its private DNS name).",
+      );
+      await r.flush();
+      const secured = await deps.secureDatabase();
+      if (!secured.ok) {
+        save({ state: "failed", phase: "stopped" });
+        return stopWith(
+          "failed",
+          ["database-tls"],
+          `The update to ${m.version} stopped at the database's TLS step, before it rolled anything: ${secured.error}. The previous release keeps serving. Fix what it names and start the update again; it continues where it stopped.`,
+        );
+      }
+      r.success(`The database connection is verified (${secured.host}).`);
+    }
     const outcome = await deps.upgrade({
       manifest: m,
       reporter: r,

package/src/database-tls-builders.test.ts

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * #1338 — every database URL a provider adapter builds asks for
+ * `sslmode=verify-full`, and no adapter still builds a weaker one
+ * (CLAUDE.md §2 no fallbacks: a downgrade is never silent).
+ */
+
+import { describe, expect, it } from "bun:test";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { databaseTlsProblem } from "@caelo-cms/shared";
+import { generateDockerCompose } from "./compose.js";
+import { databaseUrls } from "./stack-contract.js";
+
+const DNS = "abc123.p-label.europe-west1.sql-psa.goog";
+
+describe("database URL builders (#1338)", () => {
+  it("Cloud SQL: verify-full on the instance's DNS name, never on its IP", () => {
+    for (const url of Object.values(databaseUrls(DNS))) {
+      expect(databaseTlsProblem(url)).toBeNull();
+      expect(new URL(url).hostname).toBe(DNS);
+    }
+    expect(() => databaseUrls("10.20.0.3")).toThrow(/DNS name/);
+  });
+
+  it("self-hosted Compose: every service URL is verify-full against the install's CA", () => {
+    const yaml = generateDockerCompose({
+      domain: "example.com",
+      postgresPassword: "superpw",
+      rolePasswords: { admin: "adminpw", public: "publicpw", publicStaging: "ps" },
+      caeloSecretKek: "0".repeat(64),
+      dbTlsServerCertSha256: "abababababababababababababababababababababababababababababababab",
+      internalSecret: "i".repeat(48),
+      diskSize: "20Gi",
+      images: {
+        mode: "release",
+        admin: `ghcr.io/caelo-cms/admin@sha256:${"a".repeat(64)}`,
+        gateway: `ghcr.io/caelo-cms/gateway@sha256:${"b".repeat(64)}`,
+      },
+      ports: { http: 80, https: 443 },
+      project: "caelo-example-com",
+    });
+    const urls = [...yaml.matchAll(/_DATABASE_URL: "([^"]+)"/g)].map((m) => m[1] as string);
+    // admin: cms_admin, cms_public, cms_public_staging (#1329); gateway: cms_public.
+    expect(urls).toHaveLength(4);
+    for (const url of urls) expect(databaseTlsProblem(url)).toBeNull();
+  });
+
+  for (const stack of ["gcp", "gcp-firebase", "aws", "azure"]) {
+    it(`stacks/${stack} builds no URL weaker than verify-full`, () => {
+      const src = readFileSync(resolve(import.meta.dir, `../stacks/${stack}/index.ts`), "utf8");
+      expect(src).not.toMatch(/sslmode=(disable|allow|prefer|require|verify-ca)\b/);
+    });
+  }
+});