@caelo-cms/provisioning 0.10.55 → 0.10.58

package/src/stack-converge.ts

@@ -18,6 +18,9 @@
  *   - {@link planMediaVolume}: the admin's media bucket volume (gen2 + Cloud
  *     Storage volume + mount), added in the same `services update` when
  *     missing.
+ *   - {@link planSecretFiles} / {@link planRequestTimeout} (#806): the
+ *     gateway's secret file (the remote-MCP OAuth client) and its request
+ *     timeout, in the same `services update`.
  */
 
 import { removeDefaultFirewallRules } from "./default-firewall.js";
@@ -40,11 +43,15 @@ import {
   type MediaVolumeContract,
   memoryQuantityMiB,
   RETIRED_IAM_BINDINGS,
+  RETIRED_RUNTIME_SECRETS,
   RETIRED_SERVICE_ENV,
   type RetiredIamBinding,
+  type RetiredSecretFile,
   type RuntimeEnvInputs,
+  type SecretFileContract,
   type ServiceEnvInputs,
   STATIC_CDN_POLICY,
+  secretFilePath,
   stackIamInvariants,
   staticBackendBucketPrefix,
 } from "./stack-contract.js";
@@ -285,6 +292,12 @@ export function planContractEnv(
   opts: {
     /** Contract vars to leave exactly as the services have them. */
     readonly leaveUntouched?: readonly string[];
+    /**
+     * Vars beyond the contract to set as well (#806: the remote-MCP identity
+     * env, resolved by the converge). A name the contract also sets is
+     * refused rather than silently doubled.
+     */
+    readonly extraEnv?: Readonly<Partial<Record<CloudRunSlug, readonly CloudRunEnvVar[]>>>;
   } = {},
 ):
   | {
@@ -320,16 +333,25 @@ export function planContractEnv(
     inputs = { ...runtime, provider: "gcp" };
   }
   const skip = new Set(opts.leaveUntouched ?? []);
-  const owned = (contract: readonly CloudRunEnvVar[]) => contract.filter((e) => !skip.has(e.name));
-  const admin = planEnvUpdate(
-    services.admin.liveEnv,
-    owned(adminEnvContract(inputs)),
-    RETIRED_SERVICE_ENV.admin,
-  );
+  const desired = (slug: CloudRunSlug, contract: readonly CloudRunEnvVar[]) => {
+    const extra = opts.extraEnv?.[slug] ?? [];
+    const clash = extra.find((e) => contract.some((c) => c.name === e.name));
+    if (clash) throw new Error(`${slug}: ${clash.name} is both in the contract and extra env`);
+    return [...contract, ...extra].filter((e) => !skip.has(e.name));
+  };
+  let adminDesired: CloudRunEnvVar[];
+  let gatewayDesired: CloudRunEnvVar[];
+  try {
+    adminDesired = desired("admin", adminEnvContract(inputs));
+    gatewayDesired = desired("gateway", gatewayEnvContract(runtime));
+  } catch (e) {
+    return { ok: false, error: e instanceof Error ? e.message : String(e) };
+  }
+  const admin = planEnvUpdate(services.admin.liveEnv, adminDesired, RETIRED_SERVICE_ENV.admin);
   if (!admin.ok) return { ok: false, error: `admin: ${admin.error}` };
   const gateway = planEnvUpdate(
     services.gateway.liveEnv,
-    owned(gatewayEnvContract(runtime)),
+    gatewayDesired,
     RETIRED_SERVICE_ENV.gateway,
   );
   if (!gateway.ok) return { ok: false, error: `gateway: ${gateway.error}` };
@@ -662,6 +684,53 @@ export async function removeRetiredIamBindings(
   return outcomes;
 }
 
+/**
+ * Delete the secrets older contracts created and nothing reads any more
+ * ({@link RETIRED_RUNTIME_SECRETS}) — run by upgrade AFTER both services
+ * rolled onto revisions that no longer reference them (Cloud Run would
+ * refuse a revision whose secret is gone). Deleting a secret drops its IAM
+ * bindings with it. An absent secret is `absent`; never throws for a
+ * gcloud failure.
+ */
+export async function deleteRetiredSecrets(
+  install: Pick<InstallTarget, "projectId" | "env">,
+  deps: {
+    run?: GcloudRunner;
+    retired?: readonly { readonly name: string; readonly why: string }[];
+  } = {},
+): Promise<RetiredBindingOutcome[]> {
+  const run = deps.run ?? defaultGcloud;
+  const outcomes: RetiredBindingOutcome[] = [];
+  for (const s of deps.retired ?? RETIRED_RUNTIME_SECRETS) {
+    const id = `secret ${gcpSecretId(install.env, s.name)}`;
+    const project = `--project=${install.projectId}`;
+    const describe = await run([
+      "secrets",
+      "describe",
+      gcpSecretId(install.env, s.name),
+      project,
+      "--format=value(name)",
+    ]);
+    if (!describe.ok && RESOURCE_MISSING.test(describe.stderr)) {
+      outcomes.push({ id, status: "absent", why: s.why });
+      continue;
+    }
+    const del = await run([
+      "secrets",
+      "delete",
+      gcpSecretId(install.env, s.name),
+      project,
+      "--quiet",
+    ]);
+    outcomes.push(
+      del.ok || RESOURCE_MISSING.test(del.stderr)
+        ? { id, status: del.ok ? "removed" : "absent", why: s.why }
+        : { id, status: "failed", why: s.why, error: del.stderr.trim() },
+    );
+  }
+  return outcomes;
+}
+
 /** The memory limit of the service's container (`512Mi`, `2Gi`), or null
  *  when the service declares none (Cloud Run then runs it with 512Mi). */
 export function liveContainerMemory(serviceJson: string): string | null {
@@ -713,6 +782,11 @@ const EXECUTION_ENVIRONMENT_ANNOTATION = "run.googleapis.com/execution-environme
 
 interface KnativeVolume {
   readonly name: string;
+  /** A Secret Manager volume (#806 secret files). */
+  readonly secret?: {
+    readonly secretName?: string;
+    readonly items?: readonly { readonly key?: string; readonly path?: string }[];
+  };
   readonly csi?: {
     readonly driver?: string;
     readonly readOnly?: boolean;
@@ -843,6 +917,87 @@ export function planMediaVolume(live: LiveVolumes, desired: MediaVolumeContract)
   return { ok: true, flags, changes };
 }
 
+// ===========================================================================
+// Secret files + request timeout (#806)
+// ===========================================================================
+
+export type RollAdditionsPlan =
+  | { readonly ok: true; readonly flags: string[]; readonly changes: string[] }
+  | { readonly ok: false; readonly error: string };
+
+/** `projects/<p>/secrets/<id>` or `<id>` → `<id>`. */
+function bareSecretId(name: string | undefined): string {
+  return (name ?? "").slice((name ?? "").lastIndexOf("/") + 1);
+}
+
+/**
+ * The flags that mount each of `files` (stack-contract.ts
+ * SERVICE_SECRET_FILES) where the stacks mount it, adding only what is
+ * missing — `--update-secrets=<dir>/<file>=<secret>:latest`, the form
+ * cloud.google.com/run/docs/configuring/services/secrets documents (gcloud
+ * names the volume itself). A file already mounted from the same secret at
+ * `latest` is converged whatever its volume is called. Anything else at that
+ * directory is refused with what was found: replacing it could feed the
+ * gateway a different credential.
+ */
+export function planSecretFiles(
+  live: LiveVolumes,
+  files: readonly SecretFileContract[],
+  env: string,
+  retired: readonly RetiredSecretFile[] = [],
+): RollAdditionsPlan {
+  const flags: string[] = [];
+  const changes: string[] = [];
+  // #914 — mounts an older contract made and nothing reads any more
+  // (RETIRED_SECRET_FILES) go in the same revision that mounts their
+  // replacement. `--remove-secrets=<dir>/<file>` drops the file, and the
+  // mount with it when it was the volume's only file (gcloud
+  // `_PruneManagedVolumeMapping`). Repeated `--remove-secrets` /
+  // `--update-secrets` flags merge (gcloud map flags use UpdateAction).
+  for (const old of retired) {
+    if (!live.mounts.some((m) => m.mountPath === old.mountDir)) continue;
+    flags.push(`--remove-secrets=${old.mountDir}/${old.fileName}`);
+    changes.push(`secret file ${old.mountDir}/${old.fileName}: removed (${old.why})`);
+  }
+  for (const file of files) {
+    const secretId = gcpSecretId(env, file.secret);
+    const mount = live.mounts.find((m) => m.mountPath === file.mountDir);
+    if (mount) {
+      const volume = live.volumes.find((v) => v.name === mount.name);
+      const item = volume?.secret?.items?.find((i) => i.path === file.fileName);
+      const inShape =
+        bareSecretId(volume?.secret?.secretName) === secretId && item?.key === file.version;
+      if (inShape) continue;
+      return {
+        ok: false,
+        error: `the service mounts volume "${mount.name}" at ${file.mountDir}, where ${secretId}:${file.version} belongs as ${file.fileName}. Remove it (gcloud run services update <service> --remove-volume-mount=${file.mountDir}) only if nothing else needs it, then re-run upgrade.`,
+      };
+    }
+    flags.push(`--update-secrets=${secretFilePath(file)}=${secretId}:${file.version}`);
+    changes.push(`secret file ${secretFilePath(file)}: (none) → ${secretId}:${file.version}`);
+  }
+  return { ok: true, flags, changes };
+}
+
+/** The request timeout of a described service, in seconds (null: Cloud Run's default, 300). */
+export function liveTimeoutSeconds(serviceJson: string): number | null {
+  const svc = JSON.parse(serviceJson) as {
+    spec?: { template?: { spec?: { timeoutSeconds?: number | string } } };
+  };
+  const t = svc.spec?.template?.spec?.timeoutSeconds;
+  return t === undefined ? null : Number(t);
+}
+
+/** `--timeout=<desired>` when the live timeout differs; nothing when it matches. */
+export function planRequestTimeout(live: number | null, desired: number): RollAdditionsPlan {
+  if (live === desired) return { ok: true, flags: [], changes: [] };
+  return {
+    ok: true,
+    flags: [`--timeout=${desired}`],
+    changes: [`request timeout: ${live === null ? "(default 300)" : live}s → ${desired}s`],
+  };
+}
+
 /**
  * The single `gcloud run services update` that rolls a service to a new
  * image, its run SA and its env changes, so all land in one new revision.

package/src/state-attach.test.ts

@@ -186,12 +186,18 @@ describe("state locks", () => {
       }
       if (args[1] === "cat")
         return objects.has(url) ? ok(objects.get(url)) : fail("matched no objects");
+      if (args[1] === "objects") {
+        const target = args[3] ?? "";
+        return objects.has(target)
+          ? ok(JSON.stringify({ generation: "1" }))
+          : fail("matched no objects");
+      }
       if (args[1] === "rm") return objects.delete(url) ? ok() : fail("matched no objects");
       return fail("unexpected");
     };
     const release = acquireBucketStateLock("acme-site-caelo-ops", "state migrate", run);
     expect(() => acquireBucketStateLock("acme-site-caelo-ops", "installer pulumi up", run)).toThrow(
-      /gcloud storage rm gs:\/\/acme-site-caelo-ops\/install\/state.lock/,
+      /held by state migrate .*lease runs until/,
     );
     release();
     acquireBucketStateLock("acme-site-caelo-ops", "installer pulumi up", run)();

package/src/state-lock-lease.test.ts

@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The state lock's lease (state-lock-lease.ts, #815): when a held lock may
+ * be broken, and how Cloud Run's execution status is read.
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { GcloudResult } from "./gcloud.js";
+import {
+  type CloudRunExecutionRef,
+  type ExecutionState,
+  judgeHeldLock,
+  LEASE_SKEW_MS,
+  type LocalHost,
+  lockBody,
+  machineId,
+  PROCESS_LEASE_MS,
+  parseExecutionState,
+  parseLockBody,
+  renewedLockBody,
+} from "./state-lock-lease.js";
+
+const ok = (stdout = ""): GcloudResult => ({ ok: true, stdout, stderr: "", exitCode: 0 });
+const fail = (stderr: string): GcloudResult => ({ ok: false, stdout: "", stderr, exitCode: 1 });
+
+const EXEC: CloudRunExecutionRef = {
+  projectId: "acme",
+  region: "europe-west1",
+  name: "caelo-production-upgrader-x1",
+};
+const START = new Date("2026-10-16T10:00:00Z");
+const HOUR = 60 * 60_000;
+/** The lease of a lock taken at START by a one-hour job. */
+const LEASE_END = START.getTime() + HOUR + LEASE_SKEW_MS;
+
+const jobLock = lockBody("caelo upgrader job", START, { execution: EXEC, maxRuntimeMs: HOUR });
+const at = (ms: number) => new Date(ms);
+const state =
+  (s: ExecutionState) =>
+  (ref: CloudRunExecutionRef): ExecutionState => {
+    expect(ref).toEqual(EXEC);
+    return s;
+  };
+
+describe("lockBody", () => {
+  it("an execution's lock names the execution and a lease of task timeout + skew", () => {
+    const body = parseLockBody(jobLock);
+    expect(body?.execution).toEqual(EXEC);
+    expect(body?.leaseUntil).toBe(new Date(LEASE_END).toISOString());
+  });
+
+  it("a machine-local lock (no holder) carries neither", () => {
+    const body = parseLockBody(lockBody("cms-provision upgrade", START));
+    expect(body?.purpose).toBe("cms-provision upgrade");
+    expect(body?.execution).toBeUndefined();
+    expect(body?.leaseUntil).toBeUndefined();
+  });
+
+  it("a CLI process's lock names host + pid, a unique id and a lease of leaseMs + skew", () => {
+    const a = parseLockBody(
+      lockBody("cms-provision upgrade", START, { leaseMs: PROCESS_LEASE_MS }),
+    );
+    const b = parseLockBody(
+      lockBody("cms-provision upgrade", START, { leaseMs: PROCESS_LEASE_MS }),
+    );
+    expect(a?.pid).toBe(process.pid);
+    expect(a?.host).toBeTruthy();
+    expect(a?.execution).toBeUndefined();
+    expect(a?.leaseUntil).toBe(
+      new Date(START.getTime() + PROCESS_LEASE_MS + LEASE_SKEW_MS).toISOString(),
+    );
+    expect(a?.id).toBeTruthy();
+    expect(a?.id).not.toBe(b?.id);
+  });
+
+  it("a renewed body keeps its id and start and moves only the lease", () => {
+    const body = parseLockBody(
+      lockBody("cms-provision upgrade", START, { leaseMs: PROCESS_LEASE_MS }, "id-1"),
+    );
+    if (!body) throw new Error("unparsable");
+    const later = at(START.getTime() + 3 * HOUR);
+    const renewed = parseLockBody(renewedLockBody(body, later, PROCESS_LEASE_MS));
+    expect(renewed).toMatchObject({ id: "id-1", startedAt: START.toISOString() });
+    expect(renewed?.leaseUntil).toBe(
+      new Date(later.getTime() + PROCESS_LEASE_MS + LEASE_SKEW_MS).toISOString(),
+    );
+  });
+});
+
+describe("judgeHeldLock — a CLI process holder (#815 review)", () => {
+  const HERE: LocalHost = { host: "laptop", alive: (pid) => pid === 4242 };
+  const cli = (host: string, pid: number) =>
+    JSON.stringify({
+      purpose: "cms-provision upgrade",
+      host,
+      pid,
+      startedAt: START.toISOString(),
+      leaseUntil: new Date(START.getTime() + PROCESS_LEASE_MS + LEASE_SKEW_MS).toISOString(),
+    });
+  const LEASE = START.getTime() + PROCESS_LEASE_MS + LEASE_SKEW_MS;
+  const never = () => {
+    throw new Error("Cloud Run must not be asked about a CLI holder");
+  };
+
+  it("its own host's lock whose pid is dead is broken at once, inside the lease", () => {
+    const v = judgeHeldLock(cli("laptop", 9999), never, at(START.getTime() + 1), HERE);
+    expect(v.kind).toBe("break");
+    expect(v.why).toContain("process is gone");
+  });
+
+  it("its own host's lock whose pid lives is held until the lease ends", () => {
+    expect(judgeHeldLock(cli("laptop", 4242), never, at(LEASE), HERE)).toMatchObject({
+      kind: "held",
+      manualRemoval: false,
+    });
+    expect(judgeHeldLock(cli("laptop", 4242), never, at(LEASE + 1), HERE).kind).toBe("break");
+  });
+
+  it("another host's lock (a Cloud Shell) is held within the lease, broken after it", () => {
+    expect(judgeHeldLock(cli("cs-1", 9999), never, at(LEASE), HERE).kind).toBe("held");
+    const v = judgeHeldLock(cli("cs-1", 9999), never, at(LEASE + 1), HERE);
+    expect(v.kind).toBe("break");
+    expect(v.why).toContain("stopped renewing");
+  });
+
+  it("#815 review: a dead pid is only judged on the machine that took the lock", () => {
+    const at1 = at(START.getTime() + 1);
+    const withMachine = (machine: string | null) =>
+      JSON.stringify({ ...JSON.parse(cli("localhost", 9999)), machine });
+    const here = (machine: string | null): LocalHost => ({
+      host: "localhost",
+      machine,
+      alive: () => false,
+    });
+    // Same host name, another machine (two Cloud Run containers, two
+    // laptops called alike): the pid is someone else's, the lease decides.
+    expect(judgeHeldLock(withMachine("m-a"), never, at1, here("m-b")).kind).toBe("held");
+    // A lock from Cloud Run names no machine: never judged by its pid.
+    expect(judgeHeldLock(withMachine(null), never, at1, here("m-b")).kind).toBe("held");
+    // A judge in Cloud Run has no machine id: it never trusts a pid, not
+    // even one of a lock written before machine ids.
+    expect(judgeHeldLock(cli("localhost", 9999), never, at1, here(null)).kind).toBe("held");
+    // The same machine: the dead pid ends it at once.
+    expect(judgeHeldLock(withMachine("m-a"), never, at1, here("m-a")).kind).toBe("break");
+    // Whatever the machine, the lease still ends it.
+    expect(judgeHeldLock(withMachine("m-a"), never, at(LEASE + 1), here("m-b")).kind).toBe("break");
+  });
+
+  it("#815 review: machineId is null inside Cloud Run and stable outside", () => {
+    expect(machineId({ CLOUD_RUN_EXECUTION: "caelo-upgrader-abc" })).toBeNull();
+    expect(machineId({ K_SERVICE: "caelo-production-admin" })).toBeNull();
+    expect(machineId({})).toBe(machineId({}));
+    const body = parseLockBody(lockBody("cms-provision upgrade", START, { leaseMs: 1 }));
+    expect(body && "machine" in body).toBe(true);
+  });
+
+  it("an unreadable lease is held for a person to remove", () => {
+    const body = JSON.stringify({
+      purpose: "cms-provision upgrade",
+      host: "cs-1",
+      pid: 1,
+      leaseUntil: "soon",
+    });
+    expect(judgeHeldLock(body, never, at(LEASE + HOUR), HERE)).toMatchObject({
+      kind: "held",
+      manualRemoval: true,
+    });
+  });
+});
+
+describe("judgeHeldLock", () => {
+  it("breaks the lock of an execution that has finished — succeeded, failed or cancelled", () => {
+    for (const how of ["succeeded", "failed", "cancelled"]) {
+      const v = judgeHeldLock(
+        jobLock,
+        state({ kind: "finished", how }),
+        at(START.getTime() + 60_000),
+      );
+      expect(v.kind).toBe("break");
+      expect(v.why).toContain(`has ${how}`);
+    }
+  });
+
+  it("never breaks the lock of a running execution, even long past its lease", () => {
+    for (const now of [START.getTime() + 1, LEASE_END + 1, LEASE_END + 24 * HOUR]) {
+      const v = judgeHeldLock(jobLock, state({ kind: "running" }), at(now));
+      expect(v).toMatchObject({ kind: "held", manualRemoval: false });
+      expect(v.why).toContain("still running");
+    }
+  });
+
+  it("an execution Cloud Run does not know: held within the lease, broken after it", () => {
+    const gone = state({ kind: "not-found" });
+    expect(judgeHeldLock(jobLock, gone, at(START.getTime())).kind).toBe("held");
+    // Exactly at the lease's end it still holds; one millisecond later it is dead.
+    expect(judgeHeldLock(jobLock, gone, at(LEASE_END)).kind).toBe("held");
+    expect(judgeHeldLock(jobLock, gone, at(LEASE_END + 1)).kind).toBe("break");
+  });
+
+  it("a lock whose lease is unreadable is never broken on 'not found'", () => {
+    const body = JSON.stringify({
+      purpose: "caelo upgrader job",
+      execution: EXEC,
+      leaseUntil: "soon",
+    });
+    const v = judgeHeldLock(body, state({ kind: "not-found" }), at(LEASE_END + 10 * HOUR));
+    expect(v).toMatchObject({ kind: "held", manualRemoval: true });
+  });
+
+  it("a holder Cloud Run cannot be asked about is undecided, never dead", () => {
+    const v = judgeHeldLock(
+      jobLock,
+      state({ kind: "unknown", error: "PERMISSION_DENIED" }),
+      at(LEASE_END + HOUR),
+    );
+    expect(v.kind).toBe("unknown");
+    expect(v.why).toContain("PERMISSION_DENIED");
+  });
+
+  it("a pre-#815 lock (no lease) from another host is never broken automatically, and Cloud Run is not asked", () => {
+    const describe = () => {
+      throw new Error("must not be asked");
+    };
+    const cli = judgeHeldLock(
+      lockBody("cms-provision upgrade", START),
+      describe,
+      at(START.getTime() + 100 * HOUR),
+    );
+    expect(cli).toMatchObject({ kind: "held", manualRemoval: true });
+    const legacy = JSON.stringify({
+      purpose: "caelo upgrader job",
+      host: "localhost",
+      pid: 1,
+      startedAt: START.toISOString(),
+    });
+    expect(
+      judgeHeldLock(legacy, describe, at(START.getTime() + 100 * HOUR), {
+        host: "laptop",
+        alive: () => false,
+      }).kind,
+    ).toBe("held");
+  });
+
+  it("an unreadable lock body is held, with the body quoted", () => {
+    const v = judgeHeldLock("not json", state({ kind: "running" }), START);
+    expect(v).toMatchObject({ kind: "held", manualRemoval: true });
+    expect(v.why).toContain("not json");
+  });
+});
+
+describe("parseExecutionState", () => {
+  const exec = (status: unknown) => ok(JSON.stringify({ status }));
+
+  it("reads the Completed condition: True succeeded, False failed or cancelled", () => {
+    expect(
+      parseExecutionState(exec({ conditions: [{ type: "Completed", status: "True" }] })),
+    ).toEqual({
+      kind: "finished",
+      how: "succeeded",
+    });
+    expect(
+      parseExecutionState(
+        exec({ conditions: [{ type: "Completed", status: "False", reason: "Cancelled" }] }),
+      ),
+    ).toEqual({ kind: "finished", how: "cancelled" });
+    expect(
+      parseExecutionState(exec({ conditions: [{ type: "Completed", status: "False" }] })),
+    ).toEqual({
+      kind: "finished",
+      how: "failed",
+    });
+  });
+
+  it("a completion time without a decided condition is finished too", () => {
+    expect(parseExecutionState(exec({ completionTime: "2026-10-16T11:00:00Z" })).kind).toBe(
+      "finished",
+    );
+  });
+
+  it("an undecided or not yet started execution is running", () => {
+    expect(
+      parseExecutionState(exec({ conditions: [{ type: "Completed", status: "Unknown" }] })).kind,
+    ).toBe("running");
+    expect(parseExecutionState(exec({})).kind).toBe("running");
+    expect(parseExecutionState(ok("{}")).kind).toBe("running");
+  });
+
+  it("not found vs. any other gcloud failure", () => {
+    expect(
+      parseExecutionState(fail("ERROR: (gcloud.run.jobs.executions.describe) NOT_FOUND")).kind,
+    ).toBe("not-found");
+    expect(parseExecutionState(fail("PERMISSION_DENIED: run.executions.get"))).toEqual({
+      kind: "unknown",
+      error: "PERMISSION_DENIED: run.executions.get",
+    });
+    expect(parseExecutionState(ok("garbage")).kind).toBe("unknown");
+    expect(parseExecutionState(ok(JSON.stringify({ status: { conditions: "x" } }))).kind).toBe(
+      "unknown",
+    );
+  });
+});