@rizom/ops 0.2.0-alpha.37 → 0.2.0-alpha.370

package/templates/rover-pilot/.github/workflows/deploy.yml

@@ -7,6 +7,11 @@ on:
         description: User handle
         required: true
         type: string
+      release_stale_lock:
+        description: Release an operator-confirmed stale Kamal deploy lock before retrying
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ["main"]
     paths:
@@ -82,51 +87,29 @@ jobs:
       - name: Install operator tooling
         run: bun install
 
+      - name: Load shared env via Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
       - name: Decrypt user secrets
         id: user_secrets
-        env:
-          AGE_SECRET_KEY: ${{ secrets.AGE_SECRET_KEY }}
         run: bun deploy/scripts/decrypt-user-secrets.ts "$HANDLE"
 
       - name: Reconcile selected user config
-        env:
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
+        run: bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
 
       - name: Resolve generated user config
         id: user_config
         run: bun deploy/scripts/resolve-user-config.ts
 
       - name: Validate selected secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/validate-secrets.ts
+        run: bun deploy/scripts/validate-secrets.ts
 
       - name: Seed content repo
         env:
           CONTENT_REPO: ${{ steps.user_config.outputs.content_repo }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bun deploy/scripts/sync-content-repo.ts
+        run: bun deploy/scripts/sync-content-repo.ts
 
       - name: Log in to GHCR
         uses: docker/login-action@v3
@@ -135,15 +118,18 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Wait for shared image tag
+      - name: Wait for image tag
         run: |
-          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:brain-${{ steps.user_config.outputs.brain_version }}"
-          for attempt in $(seq 1 24); do
+          # The Build workflow fires off the same config push (it derives the
+          # declared image set from the registry), so this wait must cover a
+          # full image build, not just registry propagation.
+          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:${{ steps.user_config.outputs.image_tag }}"
+          for attempt in $(seq 1 60); do
             if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
               exit 0
             fi
-            echo "Shared image tag not ready yet ($attempt/24): $IMAGE_TAG"
-            sleep 10
+            echo "Image tag not ready yet ($attempt/60): $IMAGE_TAG"
+            sleep 20
           done
           echo "Timed out waiting for $IMAGE_TAG" >&2
           exit 1
@@ -154,8 +140,6 @@ jobs:
           ruby -r rubygems -e 'puts Gem.user_dir + "/bin"' >> "$GITHUB_PATH"
 
       - name: Write Kamal SSH key
-        env:
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
         run: bun deploy/scripts/write-ssh-key.ts
 
       - name: Configure SSH client
@@ -172,38 +156,26 @@ jobs:
           chmod 600 ~/.ssh/config
 
       - name: Write .kamal/secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/write-kamal-secrets.ts
+        run: bun deploy/scripts/write-kamal-secrets.ts
 
       - name: Provision server
         id: provision
         env:
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
           INSTANCE_NAME: ${{ steps.user_config.outputs.instance_name }}
         run: bun deploy/scripts/provision-server.ts
 
       - name: Update Cloudflare DNS
         env:
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+          CF_ZONE_ID: ${{ steps.user_config.outputs.cloudflare_zone_id }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
         run: |
           bun deploy/scripts/update-dns.ts
+          if [ -n "$WWW_DOMAIN" ]; then
+            BRAIN_DOMAIN="$WWW_DOMAIN" bun deploy/scripts/update-dns.ts
+          fi
           BRAIN_DOMAIN="$PREVIEW_DOMAIN" bun deploy/scripts/update-dns.ts
 
       - name: Validate SSH key
@@ -224,26 +196,62 @@ jobs:
           echo "SSH never became ready for $SSH_USER@$SERVER_IP" >&2
           exit 1
 
+      - name: Create verified predeploy backup
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          TARGET_HANDLE: ${{ matrix.handle }}
+          TARGET_VERSION: ${{ steps.user_config.outputs.image_tag }}
+          SERVICE_NAME: rover
+        run: bun deploy/scripts/create-predeploy-backup.ts
+
+      - name: Release operator-confirmed stale deploy lock
+        if: ${{ inputs.release_stale_lock }}
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
+          IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
+          REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
+          BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
+          BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
+        run: kamal lock release -c deploy/kamal/deploy.yml
+
       - name: Deploy
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
-          VERSION: brain-${{ steps.user_config.outputs.brain_version }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
           IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
           REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
         run: kamal setup --skip-push -c deploy/kamal/deploy.yml
 
+      - name: Install container health watchdog
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+        run: |
+          SSH_USER="$(ruby -e 'require "yaml"; config = YAML.load_file("deploy/kamal/deploy.yml") || {}; puts(config.dig("ssh", "user") || "root")')" \
+            bun deploy/scripts/install-health-watchdog.ts
+
       - name: Verify origin TLS
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
         run: |
-          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health"
+          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health/live"
+          if [ -n "$WWW_DOMAIN" ]; then
+            curl -I -k --max-time 20 --resolve "$WWW_DOMAIN:443:$SERVER_IP" "https://$WWW_DOMAIN/"
+          fi
           curl -I -k --max-time 20 --resolve "$PREVIEW_DOMAIN:443:$SERVER_IP" "https://$PREVIEW_DOMAIN/"
 
+      - name: Verify operational health
+        run: bunx brains-ops verify-user "$GITHUB_WORKSPACE" "$HANDLE"
+
       - name: Upload generated config
         uses: actions/upload-artifact@v4
         with:
@@ -297,6 +305,12 @@ jobs:
 
       - name: Commit generated config
         run: |
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users views
+
           if git diff --quiet -- users views; then
             exit 0
           fi

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/health-watchdog-smoke.yml

@@ -0,0 +1,95 @@
+name: Health Watchdog Smoke
+
+on:
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke fleet handle only
+        required: true
+        type: string
+        default: smoke
+      confirm:
+        description: Type watchdog-smoke:<handle> to confirm temporary container restarts
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-${{ inputs.handle }}
+  cancel-in-progress: false
+
+env:
+  WATCHDOG_SMOKE_RUN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Run fleet watchdog isolation smoke
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke"
+
+      - name: Upload watchdog smoke evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke
+          if-no-files-found: warn
+
+  cleanup:
+    needs: smoke
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual watchdog smoke fixtures
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke-cleanup"
+
+      - name: Upload watchdog smoke cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-cleanup-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,21 +35,27 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
 
           git fetch origin "${{ github.ref_name }}"
-          if git diff --quiet "origin/${{ github.ref_name }}" -- views users; then
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
             exit 0
           fi
 
           patch_file="$(mktemp)"
-          git diff --binary -- views users > "$patch_file"
+          git diff --binary -- users > "$patch_file"
           git reset --hard "origin/${{ github.ref_name }}"
           git apply --3way --index "$patch_file"
 
-          if git diff --cached --quiet -- views users; then
+          if git diff --cached --quiet -- users; then
             exit 0
           fi
 

package/templates/rover-pilot/.github/workflows/upgrade.yml

@@ -0,0 +1,104 @@
+name: Upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Explicit @rizom/ops version (empty means latest)
+        required: false
+        type: string
+  schedule:
+    # Weekday morning check for new operator tooling releases.
+    - cron: "17 6 * * 1-5"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: upgrade
+  cancel-in-progress: false
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      # Check only presence here; the private key itself is never exposed to
+      # this step. A missing private-key secret fails before checkout.
+      - name: Require upgrade App configuration
+        env:
+          APP_ID: ${{ vars.OPS_UPGRADE_APP_ID }}
+          HAS_PRIVATE_KEY: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY != '' }}
+        run: |
+          missing=""
+          if [ -z "$APP_ID" ]; then
+            missing="OPS_UPGRADE_APP_ID"
+          fi
+          if [ "$HAS_PRIVATE_KEY" != "true" ]; then
+            missing="${missing:+$missing and }OPS_UPGRADE_APP_PRIVATE_KEY"
+          fi
+          if [ -n "$missing" ]; then
+            echo "$missing is not set for this repository." >&2
+            echo "Configure the upgrade GitHub App as described in docs/operator-playbook.md." >&2
+            exit 1
+          fi
+
+      # No credential is persisted in .git/config while the workflow installs
+      # and executes a freshly published @rizom/ops package.
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Upgrade operator tooling
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: bunx brains-ops upgrade . --to "${VERSION_INPUT:-latest}"
+
+      - name: Detect upgrade changes
+        id: upgrade-changes
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "Already up to date."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create repository-scoped upgrade token
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        id: upgrade-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.OPS_UPGRADE_APP_ID }}
+          private-key: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write
+
+      - name: Open upgrade PR
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ steps.upgrade-token.outputs.token }}
+        run: |
+          VERSION=$(bun -e 'console.log(JSON.parse(await Bun.file("package.json").text()).devDependencies["@rizom/ops"])')
+          BRANCH="ops/upgrade-rizom-ops-$VERSION"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A
+          git commit -m "chore(ops): upgrade @rizom/ops to $VERSION"
+          git push --force \
+            "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \
+            "$BRANCH"
+          if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            echo "PR already open for $BRANCH"
+            exit 0
+          fi
+          gh pr create \
+            --title "chore(ops): upgrade @rizom/ops to $VERSION" \
+            --body "Automated operator tooling upgrade. Review the scaffold diff, then merge; deploys stay operator-dispatched." \
+            --head "$BRANCH"

package/templates/rover-pilot/README.md

@@ -1,6 +1,6 @@
-# rover-pilot
+# private brain pilot
 
-Private desired-state repo for the rover pilot.
+Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
 
 This is a single operator-owned repo. Pilot users do not get their own brain repos.
 Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
@@ -29,18 +29,28 @@ The repo also checks in its deploy contract:
 - `.github/workflows/*`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
-The shared pilot image tag is `brain-${brainVersion}` end to end.
-When `pilot.yaml.brainVersion` changes and you push, CI rebuilds the shared tag, refreshes generated user env files, and redeploys affected users.
+Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
+The fleet publishes one `brain-${brainVersion}` image per effective Brain version and installs the union of exact site/theme package pins required by instances on that version.
+When an effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI builds every missing version image, refreshes generated user env files, and redeploys affected users. Every external site and theme package keeps its own exact version pin; change that package set only together with a fresh Brain version so published tags remain immutable.
 When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
 
 ## Commands
 
 - `brains-ops init <repo>`
-- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health`, and unauthenticated `/mcp` status checks
-- `brains-ops onboard <repo> <handle>`
+- `brains-ops upgrade <repo>` — bumps `@rizom/ops` (or `--to <version>`) and refreshes the generated scaffold; the Upgrade workflow runs this on a schedule and opens a PR
+- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
+- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
+- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
 - `brains-ops age-key:bootstrap <repo>`
 - `brains-ops ssh-key:bootstrap <repo>`
 - `brains-ops cert:bootstrap <repo>`
 - `brains-ops secrets:encrypt <repo> <handle>`
 - `brains-ops reconcile-cohort <repo> <cohort>`
 - `brains-ops reconcile-all <repo>`
+- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
+
+`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
+
+The Upgrade workflow uses a repository-scoped GitHub App token because an Actions `GITHUB_TOKEN` cannot update generated files under `.github/workflows/`. Checkout persists no credential, and the App token is minted only after the freshly published tooling finishes and only when there is a change to push. Configure the App and its `OPS_UPGRADE_APP_ID` Actions variable and `OPS_UPGRADE_APP_PRIVATE_KEY` Actions secret as described in `docs/operator-playbook.md`. If token creation or the upgrade push fails, stop; do not substitute an operator's personal credentials.
+
+Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.