@rizom/ops 0.2.0-alpha.29 → 0.2.0-alpha.291

package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts

@@ -15,13 +15,52 @@ const decrypter = new Decrypter();
 decrypter.addIdentity(ageSecretKey);
 
 const plaintext = await decrypter.decrypt(decoded, "text");
-const secrets = parseFlatYaml(plaintext);
-const pilot = parseFlatYaml(readFileSync("pilot.yaml", "utf8"));
-
-writeGitHubEnv("AI_API_KEY", secrets["aiApiKey"] ?? "");
-writeGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"] ?? "");
-writeGitHubEnv("MCP_AUTH_TOKEN", secrets["mcpAuthToken"] ?? "");
-writeGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"] ?? "");
+const secrets = parseStringYamlMapping(plaintext);
+const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
+
+writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
+writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
+writeSecretGitHubEnv(
+  "CMS_CONTENT_REPO_PAT",
+  secrets["cmsContentRepoPat"] ?? secrets["gitSyncToken"],
+);
+writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
+writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
+writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
+// Per-user AT Protocol publishing credential (optional; from the user's
+// encrypted secrets file). The scaffold wires this so a pilot can publish its
+// brain's agent card to its PDS; a deployment that doesn't publish simply
+// leaves it unset.
+writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
+// Per-user custom-domain TLS overrides the shared fleet certificate when a
+// complete certificate/key pair is present in the encrypted user secrets.
+const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
+const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
+if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
+  throw new Error(
+    "Custom-domain TLS secrets require both certificatePem and privateKeyPem",
+  );
+}
+// A corrupted stored value would otherwise only surface as a kamal-proxy
+// "unable to load certificate" failure after the container has already booted.
+assertPemShape("certificatePem", certificatePem);
+assertPemShape("privateKeyPem", privateKeyPem);
+writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
+writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
+
+function assertPemShape(name: string, value: string | undefined): void {
+  if (value === undefined || value.trim().length === 0) {
+    return;
+  }
+  if (
+    !/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
+    !/-----END [A-Z0-9 ]+-----/.test(value)
+  ) {
+    throw new Error(
+      `Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
+    );
+  }
+}
 
 writeGitHubOutput(
   "shared_ai_api_key_secret_name",
@@ -32,10 +71,29 @@ writeGitHubOutput(
   requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
 );
 writeGitHubOutput(
-  "shared_mcp_auth_token_secret_name",
-  requireFlatValue(pilot, "mcpAuthToken", "pilot.yaml"),
+  "shared_content_repo_admin_token_secret_name",
+  requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
 );
 
+function writeSecretGitHubEnv(name: string, value: string | undefined): void {
+  if (!value || value.trim().length === 0) {
+    return;
+  }
+
+  maskGitHubSecret(value);
+  writeGitHubEnv(name, value);
+}
+
+function maskGitHubSecret(value: string): void {
+  const escaped = value
+    .replace(/%/g, "%25")
+    .replace(/\r/g, "%0D")
+    .replace(/\n/g, "%0A");
+  if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
+    console.log(`::add-mask::${escaped}`);
+  }
+}
+
 function extractAgeIdentity(contents: string): string {
   const line = contents
     .split(/\r?\n/)
@@ -49,27 +107,25 @@ function extractAgeIdentity(contents: string): string {
   return line;
 }
 
-function parseFlatYaml(contents: string): Record<string, string> {
-  const result: Record<string, string> = {};
-
-  for (const rawLine of contents.split(/\r?\n/)) {
-    const line = rawLine.trim();
-    if (!line || line.startsWith("#")) {
-      continue;
-    }
+function parseStringYamlMapping(contents: string): Record<string, string> {
+  const parsed: unknown = Bun.YAML.parse(contents);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Expected a YAML mapping");
+  }
 
-    const match = line.match(/^([A-Za-z0-9_]+):\s*(.*)$/);
-    if (!match) {
-      continue;
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string") {
+      result[key] = value;
     }
-
-    const [, key, rawValue] = match;
-    result[key] = rawValue.replace(/^['"]|['"]$/g, "");
   }
-
   return result;
 }
 
+function decodeEscapedSecret(value: string | undefined): string | undefined {
+  return value?.replace(/\\n/g, "\n");
+}
+
 function requireFlatValue(
   values: Record<string, string>,
   key: string,

package/templates/rover-pilot/deploy/scripts/helpers.ts

@@ -6,5 +6,8 @@ export {
   requireEnv,
   writeGitHubOutput,
   writeGitHubEnv,
+  siteImageTag,
+  sitePackagesFor,
+  runResolveMissingImages,
 } from "@rizom/ops/deploy";
 export type { EnvSchemaEntry } from "@rizom/ops/deploy";

package/templates/rover-pilot/deploy/scripts/install-health-watchdog.ts

@@ -0,0 +1,144 @@
+import { requireEnv } from "./helpers";
+
+const WATCHDOG_PATH = "/usr/local/sbin/brains-health-watchdog";
+const SERVICE_PATH = "/etc/systemd/system/brains-health-watchdog.service";
+const TIMER_PATH = "/etc/systemd/system/brains-health-watchdog.timer";
+
+export const BRAIN_WATCHDOG_LABEL = "ai.rizom.brain.watchdog";
+export const BRAIN_WATCHDOG_LABEL_VALUE = "true";
+export const BRAIN_WATCHDOG_LABEL_FILTER: string = `${BRAIN_WATCHDOG_LABEL}=${BRAIN_WATCHDOG_LABEL_VALUE}`;
+
+export const healthWatchdogScript: string = `#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+lock_path="\${BRAIN_WATCHDOG_LOCK_PATH:-/run/brains-health-watchdog.lock}"
+exec 9>"$lock_path"
+flock -n 9 || exit 0
+
+incident_dir="\${BRAIN_WATCHDOG_INCIDENT_DIR:-/var/log/brains-health-watchdog}"
+state_dir="\${BRAIN_WATCHDOG_STATE_DIR:-/var/lib/brains-health-watchdog}"
+MAX_RESTARTS=3
+WINDOW_SECONDS=3600
+mkdir -p "$incident_dir" "$state_dir"
+
+write_safe_inspect() {
+  docker inspect --format '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State}},"restartPolicy":{{json .HostConfig.RestartPolicy}}}' "$1"
+}
+
+while read -r container_id; do
+  [ -n "$container_id" ] || continue
+  service_name=$(docker inspect --format '{{ index .Config.Labels "service" }}' "$container_id" 2>/dev/null || true)
+  container_name=$(docker inspect --format '{{ .Name }}' "$container_id" 2>/dev/null | tr -d '/' || true)
+  state_key=$(printf '%s-%s-%s' "$service_name" "$container_name" "$container_id" | tr -c 'A-Za-z0-9_.-' '_')
+  state_path="$state_dir/$state_key.restarts"
+  recent_path="$state_path.tmp"
+  now_epoch=$(date -u +%s)
+  cutoff_epoch=$((now_epoch - WINDOW_SECONDS))
+  touch "$state_path"
+  awk -v cutoff="$cutoff_epoch" '$1 >= cutoff' "$state_path" >"$recent_path"
+  restart_count=$(wc -l <"$recent_path")
+
+  timestamp=$(date -u +%Y%m%dT%H%M%SZ)
+  incident_path="$incident_dir/$timestamp-$state_key.log"
+  temporary_path="$incident_path.tmp"
+
+  if [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
+    {
+      printf '{"timestamp":"%s","event":"restart-suppressed","reason":"restart-budget-exhausted","service":"%s","container":"%s","restartsInWindow":%s}\\n' "$timestamp" "$service_name" "$container_name" "$restart_count"
+      write_safe_inspect "$container_id"
+      docker logs --timestamps --tail 1000 "$container_id"
+    } >"$temporary_path" 2>&1
+    mv "$temporary_path" "$incident_path"
+    mv "$recent_path" "$state_path"
+    logger -t brains-health-watchdog "restart budget exhausted container=$container_name service=$service_name incident=$incident_path"
+    continue
+  fi
+
+  {
+    printf '{"timestamp":"%s","event":"container-restart","reason":"docker-health-unhealthy","service":"%s","container":"%s"}\\n' "$timestamp" "$service_name" "$container_name"
+    write_safe_inspect "$container_id"
+    docker logs --timestamps --tail 1000 "$container_id"
+  } >"$temporary_path" 2>&1
+  mv "$temporary_path" "$incident_path"
+  printf '%s\\n' "$now_epoch" >>"$recent_path"
+  mv "$recent_path" "$state_path"
+
+  logger -t brains-health-watchdog "restarting unhealthy container=$container_name service=$service_name incident=$incident_path"
+  docker restart "$container_id"
+done < <(docker ps --filter health=unhealthy --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}')
+`;
+
+export const healthWatchdogServiceUnit: string = `[Unit]
+Description=Restart persistently unhealthy Brain containers
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=${WATCHDOG_PATH}
+`;
+
+export const healthWatchdogTimerUnit: string = `[Unit]
+Description=Monitor Brain container liveness
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=30s
+RandomizedDelaySec=5s
+Persistent=true
+Unit=brains-health-watchdog.service
+
+[Install]
+WantedBy=timers.target
+`;
+
+function installCommand(path: string, content: string, mode: string): string {
+  const encoded = Buffer.from(content, "utf8").toString("base64");
+  return `printf '%s' '${encoded}' | $SUDO base64 --decode | $SUDO tee '${path}' >/dev/null\n$SUDO chmod ${mode} '${path}'`;
+}
+
+export function buildHealthWatchdogInstallScript(): string {
+  return `#!/usr/bin/env bash
+set -euo pipefail
+SUDO=""
+if [ "$(id -u)" -ne 0 ]; then
+  SUDO=sudo
+fi
+
+${installCommand(WATCHDOG_PATH, healthWatchdogScript, "0755")}
+${installCommand(SERVICE_PATH, healthWatchdogServiceUnit, "0644")}
+${installCommand(TIMER_PATH, healthWatchdogTimerUnit, "0644")}
+
+$SUDO systemctl daemon-reload
+$SUDO systemctl enable --now brains-health-watchdog.timer
+
+labelled_containers=""
+if ! labelled_containers=$($SUDO docker ps --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}' 2>/dev/null); then
+  echo "Warning: unable to verify running Brain containers with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+elif [ -z "$labelled_containers" ]; then
+  echo "Warning: no running Brain containers found with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+fi
+`;
+}
+
+export async function installHealthWatchdog(): Promise<void> {
+  const serverIp = requireEnv("SERVER_IP");
+  const sshUser = requireEnv("SSH_USER").trim();
+  if (!sshUser) throw new Error("Missing SSH_USER");
+  const child = Bun.spawn(["ssh", `${sshUser}@${serverIp}`, "bash -s"], {
+    stdin: new Blob([buildHealthWatchdogInstallScript()]),
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+  const exitCode = await child.exited;
+  if (exitCode !== 0) {
+    throw new Error(
+      `Health watchdog installation failed with exit ${exitCode}`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  await installHealthWatchdog();
+}

package/templates/rover-pilot/deploy/scripts/resolve-deploy-handles.ts

@@ -9,14 +9,23 @@ if (eventName === "workflow_dispatch") {
   process.exit(0);
 }
 
-if (eventName !== "push") {
+if (eventName !== "push" && eventName !== "workflow_run") {
   throw new Error(`Unsupported GITHUB_EVENT_NAME: ${eventName}`);
 }
 
 const beforeSha = requireEnv("BEFORE_SHA");
-const currentSha = requireEnv("GITHUB_SHA");
+const currentSha =
+  eventName === "workflow_run"
+    ? execFileSync("git", ["rev-parse", "HEAD"], {
+        encoding: "utf8",
+      }).trim()
+    : requireEnv("GITHUB_SHA");
 
-if (!isUsableGitRevision(beforeSha) || !isUsableGitRevision(currentSha)) {
+if (
+  !isUsableGitRevision(beforeSha) ||
+  !isUsableGitRevision(currentSha) ||
+  beforeSha === currentSha
+) {
   writeGitHubOutput("handles_json", JSON.stringify([]));
   process.exit(0);
 }

package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts

@@ -0,0 +1,13 @@
+import {
+  requireEnv,
+  runResolveMissingImages,
+  writeGitHubOutput,
+} from "./helpers";
+
+// The Build workflow's resolve step — the logic lives in @rizom/ops; this
+// pilot repo only supplies its own paths and repository.
+await runResolveMissingImages({
+  rootDir: process.cwd(),
+  imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
+  writeOutput: writeGitHubOutput,
+});

package/templates/rover-pilot/deploy/scripts/resolve-user-config.ts

@@ -1,6 +1,14 @@
 import { readFileSync } from "node:fs";
 
-import { parseEnvFile, requireEnv, writeGitHubOutput } from "./helpers";
+import { derivePreviewDomain, loadPilotRegistry } from "@rizom/ops";
+
+import {
+  parseEnvFile,
+  requireEnv,
+  sitePackagesFor,
+  siteImageTag,
+  writeGitHubOutput,
+} from "./helpers";
 
 const handle = requireEnv("HANDLE");
 const envPath = `users/${handle}/.env`;
@@ -17,23 +25,42 @@ if (!brainDomain) {
   throw new Error(`Missing domain in ${brainYamlPath}`);
 }
 
-const zone =
-  brainDomain.startsWith(`${handle}.`) && brainDomain.length > handle.length + 1
-    ? brainDomain.slice(handle.length + 1)
-    : "";
-if (!zone) {
-  throw new Error(`Could not derive preview domain from ${brainDomain}`);
+const registry = await loadPilotRegistry(process.cwd());
+const user = registry.users.find((entry) => entry.handle === handle);
+if (!user) {
+  throw new Error(`Unknown user handle: ${handle}`);
 }
-const previewDomain = `${handle}-preview.${zone}`;
+const previewDomain = derivePreviewDomain(brainDomain, {
+  sharedDomain: registry.pilot.domainSuffix,
+});
+const wwwDomain = isFleetDomain(
+  handle,
+  brainDomain,
+  registry.pilot.domainSuffix,
+)
+  ? ""
+  : `www.${brainDomain}`;
+
+const brainVersion = envEntries["BRAIN_VERSION"] ?? "";
+
+// The image tag is a pure function of this instance's own config: plain
+// `brain-{version}` for a default instance, or its own `brain-{version}-sites-
+// {hash}` when it declares a siteOverride. Resolved through the same helper the
+// build uses so the tag we wait for and run matches exactly what was pushed.
+const sitePackages = sitePackagesFor(user.siteOverride);
+const imageTag = siteImageTag(brainVersion, sitePackages);
 
 const outputs: Record<string, string> = {
-  brain_version: envEntries["BRAIN_VERSION"] ?? "",
+  brain_version: brainVersion,
   content_repo: envEntries["CONTENT_REPO"] ?? "",
   brain_domain: brainDomain,
   preview_domain: previewDomain,
+  www_domain: wwwDomain,
+  cloudflare_zone_id: user.cloudflareZoneId ?? process.env["CF_ZONE_ID"] ?? "",
   brain_yaml_path: brainYamlPath,
   instance_name: `rover-${handle}`,
   image_repository: `ghcr.io/${repository}`,
+  image_tag: imageTag,
   registry_username: repositoryOwner,
 };
 
@@ -47,3 +74,11 @@ for (const key of required) {
 for (const [key, value] of Object.entries(outputs)) {
   writeGitHubOutput(key, value);
 }
+
+function isFleetDomain(
+  userHandle: string,
+  domain: string,
+  pilotDomainSuffix: string,
+): boolean {
+  return domain === `${userHandle}${pilotDomainSuffix}`;
+}

package/templates/rover-pilot/deploy/scripts/sync-content-repo.ts

@@ -5,53 +5,6 @@ import { dirname, join } from "node:path";
 import { execFileSync } from "node:child_process";
 import { readJsonResponse, requireEnv } from "./helpers";
 
-const handle = requireEnv("HANDLE");
-const contentRepo = requireEnv("CONTENT_REPO");
-const token = requireEnv("GIT_SYNC_TOKEN");
-const sourceDir = join("users", handle, "content");
-
-if (!existsSync(sourceDir)) {
-  process.exit(0);
-}
-
-const { owner, repo } = parseRepoSlug(contentRepo);
-await ensureGitHubRepo({ owner, repo, token });
-
-const tempRoot = await mkdtemp(join(tmpdir(), "brains-ops-content-"));
-const checkoutDir = join(tempRoot, "repo");
-const remoteUrl = buildAuthenticatedRemoteUrl(owner, repo, token);
-
-runGit(["clone", remoteUrl, checkoutDir]);
-runGit(["-C", checkoutDir, "checkout", "-B", "main"]);
-
-const copiedFiles = await copyMissingFiles(sourceDir, checkoutDir);
-if (copiedFiles === 0) {
-  process.exit(0);
-}
-
-runGit(["-C", checkoutDir, "config", "user.name", "brains-ops[bot]"]);
-runGit([
-  "-C",
-  checkoutDir,
-  "config",
-  "user.email",
-  "41898282+github-actions[bot]@users.noreply.github.com",
-]);
-runGit(["-C", checkoutDir, "add", "."]);
-
-if (hasNoStagedChanges(checkoutDir)) {
-  process.exit(0);
-}
-
-runGit([
-  "-C",
-  checkoutDir,
-  "commit",
-  "-m",
-  `chore(content): seed ${handle} anchor profile`,
-]);
-runGit(["-C", checkoutDir, "push", "origin", "HEAD:main"]);
-
 const STALE_ANCHOR_PROFILE_MARKERS = [
   "name: Your Name Here",
   "Delete this and write your own",
@@ -68,6 +21,55 @@ interface GitHubRepoResponse {
   private?: boolean;
 }
 
+async function main(): Promise<void> {
+  const handle = requireEnv("HANDLE");
+  const contentRepo = requireEnv("CONTENT_REPO");
+  const token = requireEnv("GIT_SYNC_TOKEN");
+  const sourceDir = join("users", handle, "content");
+
+  if (!existsSync(sourceDir)) {
+    return;
+  }
+
+  const { owner, repo } = parseRepoSlug(contentRepo);
+  await ensureGitHubRepo({ owner, repo, token });
+
+  const tempRoot = await mkdtemp(join(tmpdir(), "brains-ops-content-"));
+  const checkoutDir = join(tempRoot, "repo");
+  const remoteUrl = buildAuthenticatedRemoteUrl(owner, repo, token);
+
+  runGit(["clone", remoteUrl, checkoutDir]);
+  runGit(["-C", checkoutDir, "checkout", "-B", "main"]);
+
+  const copiedFiles = await copyMissingFiles(sourceDir, checkoutDir);
+  if (copiedFiles === 0) {
+    return;
+  }
+
+  runGit(["-C", checkoutDir, "config", "user.name", "brains-ops[bot]"]);
+  runGit([
+    "-C",
+    checkoutDir,
+    "config",
+    "user.email",
+    "41898282+github-actions[bot]@users.noreply.github.com",
+  ]);
+  runGit(["-C", checkoutDir, "add", "."]);
+
+  if (hasNoStagedChanges(checkoutDir)) {
+    return;
+  }
+
+  runGit([
+    "-C",
+    checkoutDir,
+    "commit",
+    "-m",
+    `chore(content): seed ${handle} anchor profile`,
+  ]);
+  runGit(["-C", checkoutDir, "push", "origin", "HEAD:main"]);
+}
+
 async function ensureGitHubRepo(
   options: EnsureGitHubRepoOptions,
 ): Promise<void> {
@@ -177,3 +179,5 @@ function isStaleAnchorProfile(content: string): boolean {
     content.includes(marker),
   );
 }
+
+await main();

package/templates/rover-pilot/deploy/scripts/update-dns.ts

@@ -16,8 +16,11 @@ interface CloudflareResult {
   result?: Array<{ id: string }>;
 }
 
-async function upsertRecord(name: string): Promise<void> {
-  const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=A&name=${encodeURIComponent(name)}`;
+async function findRecordId(
+  name: string,
+  type: "A" | "CNAME",
+): Promise<string | undefined> {
+  const lookupUrl = `${baseUrl}/zones/${zoneId}/dns_records?type=${type}&name=${encodeURIComponent(name)}`;
   const lookup = await fetch(lookupUrl, { headers });
   const payload = (await readJsonResponse(
     lookup,
@@ -27,9 +30,16 @@ async function upsertRecord(name: string): Promise<void> {
     throw new Error(`Cloudflare DNS lookup failed: ${JSON.stringify(payload)}`);
   }
 
-  const existing = payload.result?.[0];
+  return payload.result?.[0]?.id;
+}
+
+async function upsertRecord(name: string): Promise<void> {
+  // Prefer an existing A record. If the hostname currently has a CNAME,
+  // replace that CNAME in-place so deploys can claim legacy www aliases.
+  const existing =
+    (await findRecordId(name, "A")) ?? (await findRecordId(name, "CNAME"));
   const url = existing
-    ? `${baseUrl}/zones/${zoneId}/dns_records/${existing.id}`
+    ? `${baseUrl}/zones/${zoneId}/dns_records/${existing}`
     : `${baseUrl}/zones/${zoneId}/dns_records`;
 
   const response = await fetch(url, {

package/templates/rover-pilot/deploy/scripts/validate-secrets.ts

@@ -4,7 +4,7 @@ import { parseEnvSchema } from "./helpers";
 const envSchemaPath = ".env.schema";
 const schema = parseEnvSchema(readFileSync(envSchemaPath, "utf8"));
 const requiredKeys = schema
-  .filter((entry) => entry.required)
+  .filter((entry) => entry.required && entry.key !== "BWS_ACCESS_TOKEN")
   .map((entry) => entry.key);
 
 const missing: string[] = [];
@@ -14,6 +14,17 @@ for (const key of requiredKeys) {
   }
 }
 
+const discordKeys = [
+  "DISCORD_BOT_TOKEN",
+  "DISCORD_PUBLIC_KEY",
+  "DISCORD_APPLICATION_ID",
+];
+if (discordKeys.some((key) => process.env[key])) {
+  for (const key of discordKeys) {
+    if (!process.env[key]) missing.push(key);
+  }
+}
+
 if (missing.length > 0) {
   throw new Error(`Missing required secrets: ${missing.join(", ")}`);
 }