@rizom/ops 0.2.0-alpha.4 → 0.2.0-alpha.400

package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts

@@ -0,0 +1,139 @@
+import { readFileSync } from "node:fs";
+
+import { Decrypter, armor } from "age-encryption";
+
+import { requireEnv, writeGitHubEnv, writeGitHubOutput } from "./helpers";
+
+const handle = process.argv[2] ?? requireEnv("HANDLE");
+const ageSecretKey = extractAgeIdentity(requireEnv("AGE_SECRET_KEY"));
+const encryptedPath = `users/${handle}.secrets.yaml.age`;
+
+const armored = readFileSync(encryptedPath, "utf8");
+const decoded = armor.decode(armored);
+
+const decrypter = new Decrypter();
+decrypter.addIdentity(ageSecretKey);
+
+const plaintext = await decrypter.decrypt(decoded, "text");
+const secrets = parseStringYamlMapping(plaintext);
+const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
+
+writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
+writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
+writeSecretGitHubEnv(
+  "STUDIO_CONTENT_REPO_PAT",
+  secrets["studioContentRepoPat"] ?? secrets["gitSyncToken"],
+);
+writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
+writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
+writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
+// Per-user AT Protocol publishing credential (optional; from the user's
+// encrypted secrets file). The scaffold wires this so a pilot can publish its
+// brain's agent card to its PDS; a deployment that doesn't publish simply
+// leaves it unset.
+writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
+// Per-user custom-domain TLS overrides the shared fleet certificate when a
+// complete certificate/key pair is present in the encrypted user secrets.
+const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
+const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
+if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
+  throw new Error(
+    "Custom-domain TLS secrets require both certificatePem and privateKeyPem",
+  );
+}
+// A corrupted stored value would otherwise only surface as a kamal-proxy
+// "unable to load certificate" failure after the container has already booted.
+assertPemShape("certificatePem", certificatePem);
+assertPemShape("privateKeyPem", privateKeyPem);
+writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
+writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
+
+function assertPemShape(name: string, value: string | undefined): void {
+  if (value === undefined || value.trim().length === 0) {
+    return;
+  }
+  if (
+    !/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
+    !/-----END [A-Z0-9 ]+-----/.test(value)
+  ) {
+    throw new Error(
+      `Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
+    );
+  }
+}
+
+writeGitHubOutput(
+  "shared_ai_api_key_secret_name",
+  requireFlatValue(pilot, "aiApiKey", "pilot.yaml"),
+);
+writeGitHubOutput(
+  "shared_git_sync_token_secret_name",
+  requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
+);
+writeGitHubOutput(
+  "shared_content_repo_admin_token_secret_name",
+  requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
+);
+
+function writeSecretGitHubEnv(name: string, value: string | undefined): void {
+  if (!value || value.trim().length === 0) {
+    return;
+  }
+
+  maskGitHubSecret(value);
+  writeGitHubEnv(name, value);
+}
+
+function maskGitHubSecret(value: string): void {
+  const escaped = value
+    .replace(/%/g, "%25")
+    .replace(/\r/g, "%0D")
+    .replace(/\n/g, "%0A");
+  if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
+    console.log(`::add-mask::${escaped}`);
+  }
+}
+
+function extractAgeIdentity(contents: string): string {
+  const line = contents
+    .split(/\r?\n/)
+    .map((entry) => entry.trim())
+    .find((entry) => entry.startsWith("AGE-SECRET-KEY-"));
+
+  if (!line) {
+    throw new Error("Missing AGE-SECRET-KEY in AGE_SECRET_KEY");
+  }
+
+  return line;
+}
+
+function parseStringYamlMapping(contents: string): Record<string, string> {
+  const parsed: unknown = Bun.YAML.parse(contents);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Expected a YAML mapping");
+  }
+
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string") {
+      result[key] = value;
+    }
+  }
+  return result;
+}
+
+function decodeEscapedSecret(value: string | undefined): string | undefined {
+  return value?.replace(/\\n/g, "\n");
+}
+
+function requireFlatValue(
+  values: Record<string, string>,
+  key: string,
+  label: string,
+): string {
+  const value = values[key];
+  if (!value) {
+    throw new Error(`Missing ${key} in ${label}`);
+  }
+  return value;
+}

package/templates/rover-pilot/deploy/scripts/helpers.ts

@@ -6,5 +6,7 @@ export {
   requireEnv,
   writeGitHubOutput,
   writeGitHubEnv,
+  runtimeImageTag,
+  runResolveMissingImages,
 } from "@rizom/ops/deploy";
 export type { EnvSchemaEntry } from "@rizom/ops/deploy";

package/templates/rover-pilot/deploy/scripts/install-health-watchdog.ts

@@ -0,0 +1,144 @@
+import { requireEnv } from "./helpers";
+
+const WATCHDOG_PATH = "/usr/local/sbin/brains-health-watchdog";
+const SERVICE_PATH = "/etc/systemd/system/brains-health-watchdog.service";
+const TIMER_PATH = "/etc/systemd/system/brains-health-watchdog.timer";
+
+export const BRAIN_WATCHDOG_LABEL = "ai.rizom.brain.watchdog";
+export const BRAIN_WATCHDOG_LABEL_VALUE = "true";
+export const BRAIN_WATCHDOG_LABEL_FILTER: string = `${BRAIN_WATCHDOG_LABEL}=${BRAIN_WATCHDOG_LABEL_VALUE}`;
+
+export const healthWatchdogScript: string = `#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+lock_path="\${BRAIN_WATCHDOG_LOCK_PATH:-/run/brains-health-watchdog.lock}"
+exec 9>"$lock_path"
+flock -n 9 || exit 0
+
+incident_dir="\${BRAIN_WATCHDOG_INCIDENT_DIR:-/var/log/brains-health-watchdog}"
+state_dir="\${BRAIN_WATCHDOG_STATE_DIR:-/var/lib/brains-health-watchdog}"
+MAX_RESTARTS=3
+WINDOW_SECONDS=3600
+mkdir -p "$incident_dir" "$state_dir"
+
+write_safe_inspect() {
+  docker inspect --format '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State}},"restartPolicy":{{json .HostConfig.RestartPolicy}}}' "$1"
+}
+
+while read -r container_id; do
+  [ -n "$container_id" ] || continue
+  service_name=$(docker inspect --format '{{ index .Config.Labels "service" }}' "$container_id" 2>/dev/null || true)
+  container_name=$(docker inspect --format '{{ .Name }}' "$container_id" 2>/dev/null | tr -d '/' || true)
+  state_key=$(printf '%s-%s-%s' "$service_name" "$container_name" "$container_id" | tr -c 'A-Za-z0-9_.-' '_')
+  state_path="$state_dir/$state_key.restarts"
+  recent_path="$state_path.tmp"
+  now_epoch=$(date -u +%s)
+  cutoff_epoch=$((now_epoch - WINDOW_SECONDS))
+  touch "$state_path"
+  awk -v cutoff="$cutoff_epoch" '$1 >= cutoff' "$state_path" >"$recent_path"
+  restart_count=$(wc -l <"$recent_path")
+
+  timestamp=$(date -u +%Y%m%dT%H%M%SZ)
+  incident_path="$incident_dir/$timestamp-$state_key.log"
+  temporary_path="$incident_path.tmp"
+
+  if [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
+    {
+      printf '{"timestamp":"%s","event":"restart-suppressed","reason":"restart-budget-exhausted","service":"%s","container":"%s","restartsInWindow":%s}\\n' "$timestamp" "$service_name" "$container_name" "$restart_count"
+      write_safe_inspect "$container_id"
+      docker logs --timestamps --tail 1000 "$container_id"
+    } >"$temporary_path" 2>&1
+    mv "$temporary_path" "$incident_path"
+    mv "$recent_path" "$state_path"
+    logger -t brains-health-watchdog "restart budget exhausted container=$container_name service=$service_name incident=$incident_path"
+    continue
+  fi
+
+  {
+    printf '{"timestamp":"%s","event":"container-restart","reason":"docker-health-unhealthy","service":"%s","container":"%s"}\\n' "$timestamp" "$service_name" "$container_name"
+    write_safe_inspect "$container_id"
+    docker logs --timestamps --tail 1000 "$container_id"
+  } >"$temporary_path" 2>&1
+  mv "$temporary_path" "$incident_path"
+  printf '%s\\n' "$now_epoch" >>"$recent_path"
+  mv "$recent_path" "$state_path"
+
+  logger -t brains-health-watchdog "restarting unhealthy container=$container_name service=$service_name incident=$incident_path"
+  docker restart "$container_id"
+done < <(docker ps --filter health=unhealthy --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}')
+`;
+
+export const healthWatchdogServiceUnit: string = `[Unit]
+Description=Restart persistently unhealthy Brain containers
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=${WATCHDOG_PATH}
+`;
+
+export const healthWatchdogTimerUnit: string = `[Unit]
+Description=Monitor Brain container liveness
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=30s
+RandomizedDelaySec=5s
+Persistent=true
+Unit=brains-health-watchdog.service
+
+[Install]
+WantedBy=timers.target
+`;
+
+function installCommand(path: string, content: string, mode: string): string {
+  const encoded = Buffer.from(content, "utf8").toString("base64");
+  return `printf '%s' '${encoded}' | $SUDO base64 --decode | $SUDO tee '${path}' >/dev/null\n$SUDO chmod ${mode} '${path}'`;
+}
+
+export function buildHealthWatchdogInstallScript(): string {
+  return `#!/usr/bin/env bash
+set -euo pipefail
+SUDO=""
+if [ "$(id -u)" -ne 0 ]; then
+  SUDO=sudo
+fi
+
+${installCommand(WATCHDOG_PATH, healthWatchdogScript, "0755")}
+${installCommand(SERVICE_PATH, healthWatchdogServiceUnit, "0644")}
+${installCommand(TIMER_PATH, healthWatchdogTimerUnit, "0644")}
+
+$SUDO systemctl daemon-reload
+$SUDO systemctl enable --now brains-health-watchdog.timer
+
+labelled_containers=""
+if ! labelled_containers=$($SUDO docker ps --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}' 2>/dev/null); then
+  echo "Warning: unable to verify running Brain containers with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+elif [ -z "$labelled_containers" ]; then
+  echo "Warning: no running Brain containers found with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+fi
+`;
+}
+
+export async function installHealthWatchdog(): Promise<void> {
+  const serverIp = requireEnv("SERVER_IP");
+  const sshUser = requireEnv("SSH_USER").trim();
+  if (!sshUser) throw new Error("Missing SSH_USER");
+  const child = Bun.spawn(["ssh", `${sshUser}@${serverIp}`, "bash -s"], {
+    stdin: new Blob([buildHealthWatchdogInstallScript()]),
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+  const exitCode = await child.exited;
+  if (exitCode !== 0) {
+    throw new Error(
+      `Health watchdog installation failed with exit ${exitCode}`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  await installHealthWatchdog();
+}

package/templates/rover-pilot/deploy/scripts/provision-server.ts

@@ -30,19 +30,54 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/**
+ * These scripts are copied verbatim into generated projects, so they stay
+ * dependency-free: the API payloads are checked by hand rather than with a
+ * schema library. readJsonResponse returns unknown; asserting the Hetzner
+ * response shape onto it meant a changed or error payload surfaced as an
+ * undefined field later instead of a failed lookup here.
+ */
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isHetznerServer(value: unknown): value is HetznerServer {
+  if (!isRecord(value)) return false;
+  if (typeof value["id"] !== "number") return false;
+  if (typeof value["status"] !== "string") return false;
+  const publicNet = value["public_net"];
+  if (publicNet === undefined) return true;
+  if (!isRecord(publicNet)) return false;
+  const ipv4 = publicNet["ipv4"];
+  if (ipv4 === undefined) return true;
+  if (!isRecord(ipv4)) return false;
+  const ip = ipv4["ip"];
+  return ip === undefined || typeof ip === "string";
+}
+
+function readServer(payload: unknown): HetznerServer | undefined {
+  if (!isRecord(payload)) return undefined;
+  const server = payload["server"];
+  return isHetznerServer(server) ? server : undefined;
+}
+
+function readServers(payload: unknown): HetznerServer[] | undefined {
+  if (!isRecord(payload)) return undefined;
+  const servers = payload["servers"];
+  return Array.isArray(servers) && servers.every(isHetznerServer)
+    ? servers
+    : undefined;
+}
+
 async function listServers(): Promise<HetznerServer[]> {
   const url = `${baseUrl}/servers?label_selector=${encodeURIComponent(labelSelector)}`;
   const response = await fetch(url, { headers });
-  const payload = (await readJsonResponse(
-    response,
-    "Hetzner server lookup",
-  )) as {
-    servers?: HetznerServer[];
-  };
-  if (!response.ok || !payload.servers) {
+  const payload = await readJsonResponse(response, "Hetzner server lookup");
+  const servers = readServers(payload);
+  if (!response.ok || !servers) {
     throw new Error(`Hetzner server lookup failed: ${JSON.stringify(payload)}`);
   }
-  return payload.servers;
+  return servers;
 }
 
 async function createServer(): Promise<HetznerServer> {
@@ -58,27 +93,22 @@ async function createServer(): Promise<HetznerServer> {
       labels: { brain: instanceName },
     }),
   });
-  const payload = (await readJsonResponse(
-    response,
-    "Hetzner server create",
-  )) as {
-    server?: HetznerServer;
-  };
-  if (!response.ok || !payload.server) {
+  const payload = await readJsonResponse(response, "Hetzner server create");
+  const server = readServer(payload);
+  if (!response.ok || !server) {
     throw new Error(`Hetzner server create failed: ${JSON.stringify(payload)}`);
   }
-  return payload.server;
+  return server;
 }
 
 async function getServer(id: number): Promise<HetznerServer> {
   const response = await fetch(`${baseUrl}/servers/${id}`, { headers });
-  const payload = (await readJsonResponse(response, "Hetzner server poll")) as {
-    server?: HetznerServer;
-  };
-  if (!response.ok || !payload.server) {
+  const payload = await readJsonResponse(response, "Hetzner server poll");
+  const server = readServer(payload);
+  if (!response.ok || !server) {
     throw new Error(`Hetzner server poll failed: ${JSON.stringify(payload)}`);
   }
-  return payload.server;
+  return server;
 }
 
 let server: HetznerServer | undefined = (await listServers())[0];
@@ -101,7 +131,7 @@ while (server.status !== "running" || !server.public_net?.ipv4?.ip) {
   server = await getServer(server.id);
 }
 
-const serverIp = server.public_net?.ipv4?.ip;
+const serverIp = server.public_net.ipv4.ip;
 if (!serverIp) {
   throw new Error(`Server ${server.id} running but has no IPv4 address`);
 }

package/templates/rover-pilot/deploy/scripts/resolve-deploy-handles.ts

@@ -1,4 +1,5 @@
 import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
 import { requireEnv, writeGitHubOutput } from "./helpers";
 
 const eventName = requireEnv("GITHUB_EVENT_NAME");
@@ -9,14 +10,23 @@ if (eventName === "workflow_dispatch") {
   process.exit(0);
 }
 
-if (eventName !== "push") {
+if (eventName !== "push" && eventName !== "workflow_run") {
   throw new Error(`Unsupported GITHUB_EVENT_NAME: ${eventName}`);
 }
 
 const beforeSha = requireEnv("BEFORE_SHA");
-const currentSha = requireEnv("GITHUB_SHA");
+const currentSha =
+  eventName === "workflow_run"
+    ? execFileSync("git", ["rev-parse", "HEAD"], {
+        encoding: "utf8",
+      }).trim()
+    : requireEnv("GITHUB_SHA");
 
-if (!isUsableGitRevision(beforeSha) || !isUsableGitRevision(currentSha)) {
+if (
+  !isUsableGitRevision(beforeSha) ||
+  !isUsableGitRevision(currentSha) ||
+  beforeSha === currentSha
+) {
   writeGitHubOutput("handles_json", JSON.stringify([]));
   process.exit(0);
 }
@@ -32,10 +42,15 @@ const handles = [
     diffOutput
       .split(/\r?\n/)
       .map((path) => {
-        const match = path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml)$/);
+        const match =
+          path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml|content\/.*)$/) ??
+          path.match(/^users\/([^/]+)\.secrets\.yaml\.age$/);
         return match?.[1] ?? null;
       })
       .filter((handle): handle is string => handle !== null)
+      // Deleted generated files belong to retired users and must not be sent
+      // through the onboarding/deploy path after an offboarding commit.
+      .filter((handle) => existsSync(`users/${handle}.yaml`))
       .sort((left, right) => left.localeCompare(right)),
   ),
 ];

package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts

@@ -0,0 +1,13 @@
+import {
+  requireEnv,
+  runResolveMissingImages,
+  writeGitHubOutput,
+} from "./helpers";
+
+// The Build workflow's resolve step — the logic lives in @rizom/ops; this
+// pilot repo only supplies its own paths and repository.
+await runResolveMissingImages({
+  rootDir: process.cwd(),
+  imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
+  writeOutput: writeGitHubOutput,
+});

package/templates/rover-pilot/deploy/scripts/resolve-user-config.ts

@@ -1,5 +1,13 @@
 import { readFileSync } from "node:fs";
-import { parseEnvFile, requireEnv, writeGitHubOutput } from "./helpers";
+
+import { derivePreviewDomain, loadPilotRegistry } from "@rizom/ops";
+
+import {
+  parseEnvFile,
+  requireEnv,
+  runtimeImageTag,
+  writeGitHubOutput,
+} from "./helpers";
 
 const handle = requireEnv("HANDLE");
 const envPath = `users/${handle}/.env`;
@@ -12,32 +20,47 @@ const repositoryOwner = repository.split("/")[0] ?? "";
 const brainYaml = readFileSync(brainYamlPath, "utf8");
 const domainMatch = brainYaml.match(/^domain:\s*(.+)$/m);
 const brainDomain = domainMatch?.[1]?.trim().replace(/^['"]|['"]$/g, "") ?? "";
-
 if (!brainDomain) {
   throw new Error(`Missing domain in ${brainYamlPath}`);
 }
 
+const registry = await loadPilotRegistry(process.cwd());
+const user = registry.users.find((entry) => entry.handle === handle);
+if (!user) {
+  throw new Error(`Unknown user handle: ${handle}`);
+}
+const previewDomain = derivePreviewDomain(brainDomain, {
+  sharedDomain: registry.pilot.domainSuffix,
+});
+const wwwDomain = isFleetDomain(
+  handle,
+  brainDomain,
+  registry.pilot.domainSuffix,
+)
+  ? ""
+  : `www.${brainDomain}`;
+
+const brainVersion = envEntries["BRAIN_VERSION"] ?? "";
+
+// Build and Deploy share one tag function: every instance on a Brain version
+// runs the same immutable fleet image.
+const imageTag = runtimeImageTag(brainVersion);
+
 const outputs: Record<string, string> = {
-  brain_version: envEntries["BRAIN_VERSION"] ?? "",
-  ai_api_key_secret_name: envEntries["AI_API_KEY_SECRET"] ?? "",
-  git_sync_token_secret_name: envEntries["GIT_SYNC_TOKEN_SECRET"] ?? "",
-  mcp_auth_token_secret_name: envEntries["MCP_AUTH_TOKEN_SECRET"] ?? "",
-  discord_bot_token_secret_name: envEntries["DISCORD_BOT_TOKEN_SECRET"] ?? "",
+  brain_version: brainVersion,
   content_repo: envEntries["CONTENT_REPO"] ?? "",
   brain_domain: brainDomain,
+  preview_domain: previewDomain,
+  www_domain: wwwDomain,
+  cloudflare_zone_id: user.cloudflareZoneId ?? process.env["CF_ZONE_ID"] ?? "",
   brain_yaml_path: brainYamlPath,
   instance_name: `rover-${handle}`,
   image_repository: `ghcr.io/${repository}`,
+  image_tag: imageTag,
   registry_username: repositoryOwner,
 };
 
-const required = [
-  "brain_version",
-  "ai_api_key_secret_name",
-  "git_sync_token_secret_name",
-  "mcp_auth_token_secret_name",
-  "registry_username",
-];
+const required = ["brain_version", "registry_username"];
 for (const key of required) {
   if (!outputs[key]) {
     throw new Error(`Missing ${key} (derived from ${envPath})`);
@@ -47,3 +70,11 @@ for (const key of required) {
 for (const [key, value] of Object.entries(outputs)) {
   writeGitHubOutput(key, value);
 }
+
+function isFleetDomain(
+  userHandle: string,
+  domain: string,
+  pilotDomainSuffix: string,
+): boolean {
+  return domain === `${userHandle}${pilotDomainSuffix}`;
+}