@rizom/ops 0.2.0-alpha.28 → 0.2.0-alpha.280

package/templates/rover-pilot/.github/workflows/deploy.yml

@@ -7,6 +7,11 @@ on:
         description: User handle
         required: true
         type: string
+      release_stale_lock:
+        description: Release an operator-confirmed stale Kamal deploy lock before retrying
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ["main"]
     paths:
@@ -16,6 +21,10 @@ on:
       - users/*.secrets.yaml.age
       - deploy/**
       - .github/workflows/deploy.yml
+  workflow_run:
+    workflows: [Reconcile]
+    types: [completed]
+    branches: [main]
 
 permissions:
   contents: write
@@ -23,13 +32,16 @@ permissions:
 
 jobs:
   resolve_handles:
+    if: >
+      github.event_name != 'workflow_run' ||
+      github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     outputs:
       handles_json: ${{ steps.resolve.outputs.handles_json }}
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
           fetch-depth: 0
 
       - uses: oven-sh/setup-bun@v2
@@ -41,7 +53,7 @@ jobs:
         id: resolve
         env:
           HANDLE_INPUT: ${{ inputs.handle || '' }}
-          BEFORE_SHA: ${{ github.event.before || '' }}
+          BEFORE_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.event.before || '' }}
         run: bun deploy/scripts/resolve-deploy-handles.ts
 
   no_changes:
@@ -68,58 +80,36 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
       - name: Install operator tooling
         run: bun install
 
+      - name: Load shared env via Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
       - name: Decrypt user secrets
         id: user_secrets
-        env:
-          AGE_SECRET_KEY: ${{ secrets.AGE_SECRET_KEY }}
         run: bun deploy/scripts/decrypt-user-secrets.ts "$HANDLE"
 
       - name: Reconcile selected user config
-        env:
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
+        run: bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
 
       - name: Resolve generated user config
         id: user_config
         run: bun deploy/scripts/resolve-user-config.ts
 
       - name: Validate selected secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/validate-secrets.ts
+        run: bun deploy/scripts/validate-secrets.ts
 
       - name: Seed content repo
         env:
           CONTENT_REPO: ${{ steps.user_config.outputs.content_repo }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bun deploy/scripts/sync-content-repo.ts
+        run: bun deploy/scripts/sync-content-repo.ts
 
       - name: Log in to GHCR
         uses: docker/login-action@v3
@@ -128,15 +118,18 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Wait for shared image tag
+      - name: Wait for image tag
         run: |
-          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:brain-${{ steps.user_config.outputs.brain_version }}"
-          for attempt in $(seq 1 24); do
+          # The Build workflow fires off the same config push (it derives the
+          # declared image set from the registry), so this wait must cover a
+          # full image build, not just registry propagation.
+          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:${{ steps.user_config.outputs.image_tag }}"
+          for attempt in $(seq 1 60); do
             if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
               exit 0
             fi
-            echo "Shared image tag not ready yet ($attempt/24): $IMAGE_TAG"
-            sleep 10
+            echo "Image tag not ready yet ($attempt/60): $IMAGE_TAG"
+            sleep 20
           done
           echo "Timed out waiting for $IMAGE_TAG" >&2
           exit 1
@@ -147,8 +140,6 @@ jobs:
           ruby -r rubygems -e 'puts Gem.user_dir + "/bin"' >> "$GITHUB_PATH"
 
       - name: Write Kamal SSH key
-        env:
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
         run: bun deploy/scripts/write-ssh-key.ts
 
       - name: Configure SSH client
@@ -165,38 +156,26 @@ jobs:
           chmod 600 ~/.ssh/config
 
       - name: Write .kamal/secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/write-kamal-secrets.ts
+        run: bun deploy/scripts/write-kamal-secrets.ts
 
       - name: Provision server
         id: provision
         env:
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
           INSTANCE_NAME: ${{ steps.user_config.outputs.instance_name }}
         run: bun deploy/scripts/provision-server.ts
 
       - name: Update Cloudflare DNS
         env:
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+          CF_ZONE_ID: ${{ steps.user_config.outputs.cloudflare_zone_id }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
         run: |
           bun deploy/scripts/update-dns.ts
+          if [ -n "$WWW_DOMAIN" ]; then
+            BRAIN_DOMAIN="$WWW_DOMAIN" bun deploy/scripts/update-dns.ts
+          fi
           BRAIN_DOMAIN="$PREVIEW_DOMAIN" bun deploy/scripts/update-dns.ts
 
       - name: Validate SSH key
@@ -217,24 +196,49 @@ jobs:
           echo "SSH never became ready for $SSH_USER@$SERVER_IP" >&2
           exit 1
 
+      - name: Release operator-confirmed stale deploy lock
+        if: ${{ inputs.release_stale_lock }}
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
+          IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
+          REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
+          BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
+          BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
+        run: kamal lock release -c deploy/kamal/deploy.yml
+
       - name: Deploy
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
-          VERSION: brain-${{ steps.user_config.outputs.brain_version }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
           IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
           REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
         run: kamal setup --skip-push -c deploy/kamal/deploy.yml
 
+      - name: Install container health watchdog
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+        run: |
+          SSH_USER="$(ruby -e 'require "yaml"; config = YAML.load_file("deploy/kamal/deploy.yml") || {}; puts(config.dig("ssh", "user") || "root")')" \
+            bun deploy/scripts/install-health-watchdog.ts
+
       - name: Verify origin TLS
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
         run: |
-          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health"
+          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health/live"
+          if [ -n "$WWW_DOMAIN" ]; then
+            curl -I -k --max-time 20 --resolve "$WWW_DOMAIN:443:$SERVER_IP" "https://$WWW_DOMAIN/"
+          fi
           curl -I -k --max-time 20 --resolve "$PREVIEW_DOMAIN:443:$SERVER_IP" "https://$PREVIEW_DOMAIN/"
 
       - name: Upload generated config
@@ -271,7 +275,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
@@ -290,6 +294,12 @@ jobs:
 
       - name: Commit generated config
         run: |
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users views
+
           if git diff --quiet -- users views; then
             exit 0
           fi

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,21 +35,27 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
 
           git fetch origin "${{ github.ref_name }}"
-          if git diff --quiet "origin/${{ github.ref_name }}" -- views users; then
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
             exit 0
           fi
 
           patch_file="$(mktemp)"
-          git diff --binary -- views users > "$patch_file"
+          git diff --binary -- users > "$patch_file"
           git reset --hard "origin/${{ github.ref_name }}"
           git apply --3way --index "$patch_file"
 
-          if git diff --cached --quiet -- views users; then
+          if git diff --cached --quiet -- users; then
             exit 0
           fi
 

package/templates/rover-pilot/.github/workflows/upgrade.yml

@@ -0,0 +1,58 @@
+name: Upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Explicit @rizom/ops version (empty means latest)
+        required: false
+        type: string
+  schedule:
+    # Weekday morning check for new operator tooling releases.
+    - cron: "17 6 * * 1-5"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: upgrade
+  cancel-in-progress: false
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Upgrade operator tooling
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: bunx brains-ops upgrade . --to "${VERSION_INPUT:-latest}"
+
+      - name: Open upgrade PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "Already up to date."
+            exit 0
+          fi
+          VERSION=$(bun -e 'console.log(JSON.parse(await Bun.file("package.json").text()).devDependencies["@rizom/ops"])')
+          BRANCH="ops/upgrade-rizom-ops-$VERSION"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A
+          git commit -m "chore(ops): upgrade @rizom/ops to $VERSION"
+          git push --force -u origin "$BRANCH"
+          gh pr create \
+            --title "chore(ops): upgrade @rizom/ops to $VERSION" \
+            --body "Automated operator tooling upgrade. Review the scaffold diff, then merge; deploys stay operator-dispatched." \
+            --head "$BRANCH" \
+            || echo "PR already open for $BRANCH"

package/templates/rover-pilot/README.md

@@ -1,6 +1,6 @@
-# rover-pilot
+# private brain pilot
 
-Private desired-state repo for the rover pilot.
+Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
 
 This is a single operator-owned repo. Pilot users do not get their own brain repos.
 Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
@@ -29,18 +29,26 @@ The repo also checks in its deploy contract:
 - `.github/workflows/*`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
-The shared pilot image tag is `brain-${brainVersion}` end to end.
-When `pilot.yaml.brainVersion` changes and you push, CI rebuilds the shared tag, refreshes generated user env files, and redeploys affected users.
+Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
+The default pilot image tag is `brain-${brainVersion}` end to end. A user with `siteOverride` gets an isolated `brain-${brainVersion}-sites-${packageHash}` image instead.
+When the effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI rebuilds the required default/site tags, refreshes generated user env files, and redeploys affected users. Every external site and theme package uses its own required exact version pin; ops never infers either version from the brain or from another package.
 When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
 
 ## Commands
 
 - `brains-ops init <repo>`
-- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health`, and unauthenticated `/mcp` status checks
-- `brains-ops onboard <repo> <handle>`
+- `brains-ops upgrade <repo>` — bumps `@rizom/ops` (or `--to <version>`) and refreshes the generated scaffold; the Upgrade workflow runs this on a schedule and opens a PR
+- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
+- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
+- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
 - `brains-ops age-key:bootstrap <repo>`
 - `brains-ops ssh-key:bootstrap <repo>`
 - `brains-ops cert:bootstrap <repo>`
 - `brains-ops secrets:encrypt <repo> <handle>`
 - `brains-ops reconcile-cohort <repo> <cohort>`
 - `brains-ops reconcile-all <repo>`
+- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
+
+`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
+
+Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.

package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts

@@ -15,13 +15,52 @@ const decrypter = new Decrypter();
 decrypter.addIdentity(ageSecretKey);
 
 const plaintext = await decrypter.decrypt(decoded, "text");
-const secrets = parseFlatYaml(plaintext);
-const pilot = parseFlatYaml(readFileSync("pilot.yaml", "utf8"));
-
-writeGitHubEnv("AI_API_KEY", secrets["aiApiKey"] ?? "");
-writeGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"] ?? "");
-writeGitHubEnv("MCP_AUTH_TOKEN", secrets["mcpAuthToken"] ?? "");
-writeGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"] ?? "");
+const secrets = parseStringYamlMapping(plaintext);
+const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
+
+writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
+writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
+writeSecretGitHubEnv(
+  "CMS_CONTENT_REPO_PAT",
+  secrets["cmsContentRepoPat"] ?? secrets["gitSyncToken"],
+);
+writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
+writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
+writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
+// Per-user AT Protocol publishing credential (optional; from the user's
+// encrypted secrets file). The scaffold wires this so a pilot can publish its
+// brain's agent card to its PDS; a deployment that doesn't publish simply
+// leaves it unset.
+writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
+// Per-user custom-domain TLS overrides the shared fleet certificate when a
+// complete certificate/key pair is present in the encrypted user secrets.
+const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
+const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
+if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
+  throw new Error(
+    "Custom-domain TLS secrets require both certificatePem and privateKeyPem",
+  );
+}
+// A corrupted stored value would otherwise only surface as a kamal-proxy
+// "unable to load certificate" failure after the container has already booted.
+assertPemShape("certificatePem", certificatePem);
+assertPemShape("privateKeyPem", privateKeyPem);
+writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
+writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
+
+function assertPemShape(name: string, value: string | undefined): void {
+  if (value === undefined || value.trim().length === 0) {
+    return;
+  }
+  if (
+    !/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
+    !/-----END [A-Z0-9 ]+-----/.test(value)
+  ) {
+    throw new Error(
+      `Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
+    );
+  }
+}
 
 writeGitHubOutput(
   "shared_ai_api_key_secret_name",
@@ -32,10 +71,29 @@ writeGitHubOutput(
   requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
 );
 writeGitHubOutput(
-  "shared_mcp_auth_token_secret_name",
-  requireFlatValue(pilot, "mcpAuthToken", "pilot.yaml"),
+  "shared_content_repo_admin_token_secret_name",
+  requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
 );
 
+function writeSecretGitHubEnv(name: string, value: string | undefined): void {
+  if (!value || value.trim().length === 0) {
+    return;
+  }
+
+  maskGitHubSecret(value);
+  writeGitHubEnv(name, value);
+}
+
+function maskGitHubSecret(value: string): void {
+  const escaped = value
+    .replace(/%/g, "%25")
+    .replace(/\r/g, "%0D")
+    .replace(/\n/g, "%0A");
+  if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
+    console.log(`::add-mask::${escaped}`);
+  }
+}
+
 function extractAgeIdentity(contents: string): string {
   const line = contents
     .split(/\r?\n/)
@@ -49,27 +107,25 @@ function extractAgeIdentity(contents: string): string {
   return line;
 }
 
-function parseFlatYaml(contents: string): Record<string, string> {
-  const result: Record<string, string> = {};
-
-  for (const rawLine of contents.split(/\r?\n/)) {
-    const line = rawLine.trim();
-    if (!line || line.startsWith("#")) {
-      continue;
-    }
+function parseStringYamlMapping(contents: string): Record<string, string> {
+  const parsed: unknown = Bun.YAML.parse(contents);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Expected a YAML mapping");
+  }
 
-    const match = line.match(/^([A-Za-z0-9_]+):\s*(.*)$/);
-    if (!match) {
-      continue;
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string") {
+      result[key] = value;
     }
-
-    const [, key, rawValue] = match;
-    result[key] = rawValue.replace(/^['"]|['"]$/g, "");
   }
-
   return result;
 }
 
+function decodeEscapedSecret(value: string | undefined): string | undefined {
+  return value?.replace(/\\n/g, "\n");
+}
+
 function requireFlatValue(
   values: Record<string, string>,
   key: string,

package/templates/rover-pilot/deploy/scripts/helpers.ts

@@ -6,5 +6,8 @@ export {
   requireEnv,
   writeGitHubOutput,
   writeGitHubEnv,
+  siteImageTag,
+  sitePackagesFor,
+  runResolveMissingImages,
 } from "@rizom/ops/deploy";
 export type { EnvSchemaEntry } from "@rizom/ops/deploy";