@rizom/ops 0.2.0-alpha.3 → 0.2.0-alpha.300

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,11 +35,31 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
+
+          git fetch origin "${{ github.ref_name }}"
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
+            exit 0
+          fi
+
+          patch_file="$(mktemp)"
+          git diff --binary -- users > "$patch_file"
+          git reset --hard "origin/${{ github.ref_name }}"
+          git apply --3way --index "$patch_file"
+
+          if git diff --cached --quiet -- users; then
+            exit 0
+          fi
+
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add views users
           git commit -m "chore(ops): reconcile pilot outputs"
-          git push
+          git push origin HEAD:${{ github.ref_name }}

package/templates/rover-pilot/.github/workflows/upgrade.yml

@@ -0,0 +1,104 @@
+name: Upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Explicit @rizom/ops version (empty means latest)
+        required: false
+        type: string
+  schedule:
+    # Weekday morning check for new operator tooling releases.
+    - cron: "17 6 * * 1-5"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: upgrade
+  cancel-in-progress: false
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      # Check only presence here; the private key itself is never exposed to
+      # this step. A missing private-key secret fails before checkout.
+      - name: Require upgrade App configuration
+        env:
+          APP_ID: ${{ vars.OPS_UPGRADE_APP_ID }}
+          HAS_PRIVATE_KEY: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY != '' }}
+        run: |
+          missing=""
+          if [ -z "$APP_ID" ]; then
+            missing="OPS_UPGRADE_APP_ID"
+          fi
+          if [ "$HAS_PRIVATE_KEY" != "true" ]; then
+            missing="${missing:+$missing and }OPS_UPGRADE_APP_PRIVATE_KEY"
+          fi
+          if [ -n "$missing" ]; then
+            echo "$missing is not set for this repository." >&2
+            echo "Configure the upgrade GitHub App as described in docs/operator-playbook.md." >&2
+            exit 1
+          fi
+
+      # No credential is persisted in .git/config while the workflow installs
+      # and executes a freshly published @rizom/ops package.
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Upgrade operator tooling
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: bunx brains-ops upgrade . --to "${VERSION_INPUT:-latest}"
+
+      - name: Detect upgrade changes
+        id: upgrade-changes
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "Already up to date."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create repository-scoped upgrade token
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        id: upgrade-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.OPS_UPGRADE_APP_ID }}
+          private-key: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write
+
+      - name: Open upgrade PR
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ steps.upgrade-token.outputs.token }}
+        run: |
+          VERSION=$(bun -e 'console.log(JSON.parse(await Bun.file("package.json").text()).devDependencies["@rizom/ops"])')
+          BRANCH="ops/upgrade-rizom-ops-$VERSION"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A
+          git commit -m "chore(ops): upgrade @rizom/ops to $VERSION"
+          git push --force \
+            "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \
+            "$BRANCH"
+          if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            echo "PR already open for $BRANCH"
+            exit 0
+          fi
+          gh pr create \
+            --title "chore(ops): upgrade @rizom/ops to $VERSION" \
+            --body "Automated operator tooling upgrade. Review the scaffold diff, then merge; deploys stay operator-dispatched." \
+            --head "$BRANCH"

package/templates/rover-pilot/README.md

@@ -1,6 +1,6 @@
-# rover-pilot
+# private brain pilot
 
-Private desired-state repo for the rover pilot.
+Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
 
 This is a single operator-owned repo. Pilot users do not get their own brain repos.
 Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
@@ -29,15 +29,28 @@ The repo also checks in its deploy contract:
 - `.github/workflows/*`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
-The shared pilot image tag is `brain-${brainVersion}` end to end.
-When `pilot.yaml.brainVersion` changes and you push, CI rebuilds the shared tag, refreshes generated user env files, and redeploys affected users.
+Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
+The default pilot image tag is `brain-${brainVersion}` end to end. A user with `siteOverride` gets an isolated `brain-${brainVersion}-sites-${packageHash}` image instead.
+When the effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI rebuilds the required default/site tags, refreshes generated user env files, and redeploys affected users. Every external site and theme package uses its own required exact version pin; ops never infers either version from the brain or from another package.
 When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
 
 ## Commands
 
 - `brains-ops init <repo>`
-- `brains-ops render <repo>`
-- `brains-ops onboard <repo> <handle>`
-- `brains-ops secrets:push <repo> <handle>`
+- `brains-ops upgrade <repo>` — bumps `@rizom/ops` (or `--to <version>`) and refreshes the generated scaffold; the Upgrade workflow runs this on a schedule and opens a PR
+- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
+- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
+- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
+- `brains-ops age-key:bootstrap <repo>`
+- `brains-ops ssh-key:bootstrap <repo>`
+- `brains-ops cert:bootstrap <repo>`
+- `brains-ops secrets:encrypt <repo> <handle>`
 - `brains-ops reconcile-cohort <repo> <cohort>`
 - `brains-ops reconcile-all <repo>`
+- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
+
+`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
+
+The Upgrade workflow uses a repository-scoped GitHub App token because an Actions `GITHUB_TOKEN` cannot update generated files under `.github/workflows/`. Checkout persists no credential, and the App token is minted only after the freshly published tooling finishes and only when there is a change to push. Configure the App and its `OPS_UPGRADE_APP_ID` Actions variable and `OPS_UPGRADE_APP_PRIVATE_KEY` Actions secret as described in `docs/operator-playbook.md`. If token creation or the upgrade push fails, stop; do not substitute an operator's personal credentials.
+
+Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.

package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts

@@ -0,0 +1,139 @@
+import { readFileSync } from "node:fs";
+
+import { Decrypter, armor } from "age-encryption";
+
+import { requireEnv, writeGitHubEnv, writeGitHubOutput } from "./helpers";
+
+const handle = process.argv[2] ?? requireEnv("HANDLE");
+const ageSecretKey = extractAgeIdentity(requireEnv("AGE_SECRET_KEY"));
+const encryptedPath = `users/${handle}.secrets.yaml.age`;
+
+const armored = readFileSync(encryptedPath, "utf8");
+const decoded = armor.decode(armored);
+
+const decrypter = new Decrypter();
+decrypter.addIdentity(ageSecretKey);
+
+const plaintext = await decrypter.decrypt(decoded, "text");
+const secrets = parseStringYamlMapping(plaintext);
+const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
+
+writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
+writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
+writeSecretGitHubEnv(
+  "CMS_CONTENT_REPO_PAT",
+  secrets["cmsContentRepoPat"] ?? secrets["gitSyncToken"],
+);
+writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
+writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
+writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
+// Per-user AT Protocol publishing credential (optional; from the user's
+// encrypted secrets file). The scaffold wires this so a pilot can publish its
+// brain's agent card to its PDS; a deployment that doesn't publish simply
+// leaves it unset.
+writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
+// Per-user custom-domain TLS overrides the shared fleet certificate when a
+// complete certificate/key pair is present in the encrypted user secrets.
+const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
+const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
+if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
+  throw new Error(
+    "Custom-domain TLS secrets require both certificatePem and privateKeyPem",
+  );
+}
+// A corrupted stored value would otherwise only surface as a kamal-proxy
+// "unable to load certificate" failure after the container has already booted.
+assertPemShape("certificatePem", certificatePem);
+assertPemShape("privateKeyPem", privateKeyPem);
+writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
+writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
+
+function assertPemShape(name: string, value: string | undefined): void {
+  if (value === undefined || value.trim().length === 0) {
+    return;
+  }
+  if (
+    !/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
+    !/-----END [A-Z0-9 ]+-----/.test(value)
+  ) {
+    throw new Error(
+      `Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
+    );
+  }
+}
+
+writeGitHubOutput(
+  "shared_ai_api_key_secret_name",
+  requireFlatValue(pilot, "aiApiKey", "pilot.yaml"),
+);
+writeGitHubOutput(
+  "shared_git_sync_token_secret_name",
+  requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
+);
+writeGitHubOutput(
+  "shared_content_repo_admin_token_secret_name",
+  requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
+);
+
+function writeSecretGitHubEnv(name: string, value: string | undefined): void {
+  if (!value || value.trim().length === 0) {
+    return;
+  }
+
+  maskGitHubSecret(value);
+  writeGitHubEnv(name, value);
+}
+
+function maskGitHubSecret(value: string): void {
+  const escaped = value
+    .replace(/%/g, "%25")
+    .replace(/\r/g, "%0D")
+    .replace(/\n/g, "%0A");
+  if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
+    console.log(`::add-mask::${escaped}`);
+  }
+}
+
+function extractAgeIdentity(contents: string): string {
+  const line = contents
+    .split(/\r?\n/)
+    .map((entry) => entry.trim())
+    .find((entry) => entry.startsWith("AGE-SECRET-KEY-"));
+
+  if (!line) {
+    throw new Error("Missing AGE-SECRET-KEY in AGE_SECRET_KEY");
+  }
+
+  return line;
+}
+
+function parseStringYamlMapping(contents: string): Record<string, string> {
+  const parsed: unknown = Bun.YAML.parse(contents);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Expected a YAML mapping");
+  }
+
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string") {
+      result[key] = value;
+    }
+  }
+  return result;
+}
+
+function decodeEscapedSecret(value: string | undefined): string | undefined {
+  return value?.replace(/\\n/g, "\n");
+}
+
+function requireFlatValue(
+  values: Record<string, string>,
+  key: string,
+  label: string,
+): string {
+  const value = values[key];
+  if (!value) {
+    throw new Error(`Missing ${key} in ${label}`);
+  }
+  return value;
+}

package/templates/rover-pilot/deploy/scripts/helpers.ts

@@ -6,5 +6,8 @@ export {
   requireEnv,
   writeGitHubOutput,
   writeGitHubEnv,
+  siteImageTag,
+  sitePackagesFor,
+  runResolveMissingImages,
 } from "@rizom/ops/deploy";
 export type { EnvSchemaEntry } from "@rizom/ops/deploy";

package/templates/rover-pilot/deploy/scripts/install-health-watchdog.ts

@@ -0,0 +1,144 @@
+import { requireEnv } from "./helpers";
+
+const WATCHDOG_PATH = "/usr/local/sbin/brains-health-watchdog";
+const SERVICE_PATH = "/etc/systemd/system/brains-health-watchdog.service";
+const TIMER_PATH = "/etc/systemd/system/brains-health-watchdog.timer";
+
+export const BRAIN_WATCHDOG_LABEL = "ai.rizom.brain.watchdog";
+export const BRAIN_WATCHDOG_LABEL_VALUE = "true";
+export const BRAIN_WATCHDOG_LABEL_FILTER: string = `${BRAIN_WATCHDOG_LABEL}=${BRAIN_WATCHDOG_LABEL_VALUE}`;
+
+export const healthWatchdogScript: string = `#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+lock_path="\${BRAIN_WATCHDOG_LOCK_PATH:-/run/brains-health-watchdog.lock}"
+exec 9>"$lock_path"
+flock -n 9 || exit 0
+
+incident_dir="\${BRAIN_WATCHDOG_INCIDENT_DIR:-/var/log/brains-health-watchdog}"
+state_dir="\${BRAIN_WATCHDOG_STATE_DIR:-/var/lib/brains-health-watchdog}"
+MAX_RESTARTS=3
+WINDOW_SECONDS=3600
+mkdir -p "$incident_dir" "$state_dir"
+
+write_safe_inspect() {
+  docker inspect --format '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State}},"restartPolicy":{{json .HostConfig.RestartPolicy}}}' "$1"
+}
+
+while read -r container_id; do
+  [ -n "$container_id" ] || continue
+  service_name=$(docker inspect --format '{{ index .Config.Labels "service" }}' "$container_id" 2>/dev/null || true)
+  container_name=$(docker inspect --format '{{ .Name }}' "$container_id" 2>/dev/null | tr -d '/' || true)
+  state_key=$(printf '%s-%s-%s' "$service_name" "$container_name" "$container_id" | tr -c 'A-Za-z0-9_.-' '_')
+  state_path="$state_dir/$state_key.restarts"
+  recent_path="$state_path.tmp"
+  now_epoch=$(date -u +%s)
+  cutoff_epoch=$((now_epoch - WINDOW_SECONDS))
+  touch "$state_path"
+  awk -v cutoff="$cutoff_epoch" '$1 >= cutoff' "$state_path" >"$recent_path"
+  restart_count=$(wc -l <"$recent_path")
+
+  timestamp=$(date -u +%Y%m%dT%H%M%SZ)
+  incident_path="$incident_dir/$timestamp-$state_key.log"
+  temporary_path="$incident_path.tmp"
+
+  if [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
+    {
+      printf '{"timestamp":"%s","event":"restart-suppressed","reason":"restart-budget-exhausted","service":"%s","container":"%s","restartsInWindow":%s}\\n' "$timestamp" "$service_name" "$container_name" "$restart_count"
+      write_safe_inspect "$container_id"
+      docker logs --timestamps --tail 1000 "$container_id"
+    } >"$temporary_path" 2>&1
+    mv "$temporary_path" "$incident_path"
+    mv "$recent_path" "$state_path"
+    logger -t brains-health-watchdog "restart budget exhausted container=$container_name service=$service_name incident=$incident_path"
+    continue
+  fi
+
+  {
+    printf '{"timestamp":"%s","event":"container-restart","reason":"docker-health-unhealthy","service":"%s","container":"%s"}\\n' "$timestamp" "$service_name" "$container_name"
+    write_safe_inspect "$container_id"
+    docker logs --timestamps --tail 1000 "$container_id"
+  } >"$temporary_path" 2>&1
+  mv "$temporary_path" "$incident_path"
+  printf '%s\\n' "$now_epoch" >>"$recent_path"
+  mv "$recent_path" "$state_path"
+
+  logger -t brains-health-watchdog "restarting unhealthy container=$container_name service=$service_name incident=$incident_path"
+  docker restart "$container_id"
+done < <(docker ps --filter health=unhealthy --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}')
+`;
+
+export const healthWatchdogServiceUnit: string = `[Unit]
+Description=Restart persistently unhealthy Brain containers
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=${WATCHDOG_PATH}
+`;
+
+export const healthWatchdogTimerUnit: string = `[Unit]
+Description=Monitor Brain container liveness
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=30s
+RandomizedDelaySec=5s
+Persistent=true
+Unit=brains-health-watchdog.service
+
+[Install]
+WantedBy=timers.target
+`;
+
+function installCommand(path: string, content: string, mode: string): string {
+  const encoded = Buffer.from(content, "utf8").toString("base64");
+  return `printf '%s' '${encoded}' | $SUDO base64 --decode | $SUDO tee '${path}' >/dev/null\n$SUDO chmod ${mode} '${path}'`;
+}
+
+export function buildHealthWatchdogInstallScript(): string {
+  return `#!/usr/bin/env bash
+set -euo pipefail
+SUDO=""
+if [ "$(id -u)" -ne 0 ]; then
+  SUDO=sudo
+fi
+
+${installCommand(WATCHDOG_PATH, healthWatchdogScript, "0755")}
+${installCommand(SERVICE_PATH, healthWatchdogServiceUnit, "0644")}
+${installCommand(TIMER_PATH, healthWatchdogTimerUnit, "0644")}
+
+$SUDO systemctl daemon-reload
+$SUDO systemctl enable --now brains-health-watchdog.timer
+
+labelled_containers=""
+if ! labelled_containers=$($SUDO docker ps --filter label=${BRAIN_WATCHDOG_LABEL_FILTER} --format '{{.ID}}' 2>/dev/null); then
+  echo "Warning: unable to verify running Brain containers with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+elif [ -z "$labelled_containers" ]; then
+  echo "Warning: no running Brain containers found with label ${BRAIN_WATCHDOG_LABEL_FILTER}." >&2
+fi
+`;
+}
+
+export async function installHealthWatchdog(): Promise<void> {
+  const serverIp = requireEnv("SERVER_IP");
+  const sshUser = requireEnv("SSH_USER").trim();
+  if (!sshUser) throw new Error("Missing SSH_USER");
+  const child = Bun.spawn(["ssh", `${sshUser}@${serverIp}`, "bash -s"], {
+    stdin: new Blob([buildHealthWatchdogInstallScript()]),
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+  const exitCode = await child.exited;
+  if (exitCode !== 0) {
+    throw new Error(
+      `Health watchdog installation failed with exit ${exitCode}`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  await installHealthWatchdog();
+}

package/templates/rover-pilot/deploy/scripts/provision-server.ts

@@ -101,7 +101,7 @@ while (server.status !== "running" || !server.public_net?.ipv4?.ip) {
   server = await getServer(server.id);
 }
 
-const serverIp = server.public_net?.ipv4?.ip;
+const serverIp = server.public_net.ipv4.ip;
 if (!serverIp) {
   throw new Error(`Server ${server.id} running but has no IPv4 address`);
 }

package/templates/rover-pilot/deploy/scripts/resolve-deploy-handles.ts

@@ -9,14 +9,23 @@ if (eventName === "workflow_dispatch") {
   process.exit(0);
 }
 
-if (eventName !== "push") {
+if (eventName !== "push" && eventName !== "workflow_run") {
   throw new Error(`Unsupported GITHUB_EVENT_NAME: ${eventName}`);
 }
 
 const beforeSha = requireEnv("BEFORE_SHA");
-const currentSha = requireEnv("GITHUB_SHA");
+const currentSha =
+  eventName === "workflow_run"
+    ? execFileSync("git", ["rev-parse", "HEAD"], {
+        encoding: "utf8",
+      }).trim()
+    : requireEnv("GITHUB_SHA");
 
-if (!isUsableGitRevision(beforeSha) || !isUsableGitRevision(currentSha)) {
+if (
+  !isUsableGitRevision(beforeSha) ||
+  !isUsableGitRevision(currentSha) ||
+  beforeSha === currentSha
+) {
   writeGitHubOutput("handles_json", JSON.stringify([]));
   process.exit(0);
 }
@@ -32,7 +41,9 @@ const handles = [
     diffOutput
       .split(/\r?\n/)
       .map((path) => {
-        const match = path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml)$/);
+        const match =
+          path.match(/^users\/([^/]+)\/(?:\.env|brain\.yaml|content\/.*)$/) ??
+          path.match(/^users\/([^/]+)\.secrets\.yaml\.age$/);
         return match?.[1] ?? null;
       })
       .filter((handle): handle is string => handle !== null)

package/templates/rover-pilot/deploy/scripts/resolve-missing-images.ts

@@ -0,0 +1,13 @@
+import {
+  requireEnv,
+  runResolveMissingImages,
+  writeGitHubOutput,
+} from "./helpers";
+
+// The Build workflow's resolve step — the logic lives in @rizom/ops; this
+// pilot repo only supplies its own paths and repository.
+await runResolveMissingImages({
+  rootDir: process.cwd(),
+  imageRepository: `ghcr.io/${requireEnv("GITHUB_REPOSITORY")}`,
+  writeOutput: writeGitHubOutput,
+});