@rizom/ops 0.2.0-alpha.29 → 0.2.0-alpha.290

package/templates/rover-pilot/.github/workflows/deploy.yml

@@ -7,6 +7,11 @@ on:
         description: User handle
         required: true
         type: string
+      release_stale_lock:
+        description: Release an operator-confirmed stale Kamal deploy lock before retrying
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ["main"]
     paths:
@@ -16,6 +21,10 @@ on:
       - users/*.secrets.yaml.age
       - deploy/**
       - .github/workflows/deploy.yml
+  workflow_run:
+    workflows: [Reconcile]
+    types: [completed]
+    branches: [main]
 
 permissions:
   contents: write
@@ -23,13 +32,16 @@ permissions:
 
 jobs:
   resolve_handles:
+    if: >
+      github.event_name != 'workflow_run' ||
+      github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     outputs:
       handles_json: ${{ steps.resolve.outputs.handles_json }}
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
           fetch-depth: 0
 
       - uses: oven-sh/setup-bun@v2
@@ -41,7 +53,7 @@ jobs:
         id: resolve
         env:
           HANDLE_INPUT: ${{ inputs.handle || '' }}
-          BEFORE_SHA: ${{ github.event.before || '' }}
+          BEFORE_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.event.before || '' }}
         run: bun deploy/scripts/resolve-deploy-handles.ts
 
   no_changes:
@@ -68,58 +80,36 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
       - name: Install operator tooling
         run: bun install
 
+      - name: Load shared env via Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
       - name: Decrypt user secrets
         id: user_secrets
-        env:
-          AGE_SECRET_KEY: ${{ secrets.AGE_SECRET_KEY }}
         run: bun deploy/scripts/decrypt-user-secrets.ts "$HANDLE"
 
       - name: Reconcile selected user config
-        env:
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
+        run: bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
 
       - name: Resolve generated user config
         id: user_config
         run: bun deploy/scripts/resolve-user-config.ts
 
       - name: Validate selected secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/validate-secrets.ts
+        run: bun deploy/scripts/validate-secrets.ts
 
       - name: Seed content repo
         env:
           CONTENT_REPO: ${{ steps.user_config.outputs.content_repo }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-        run: |
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          bun deploy/scripts/sync-content-repo.ts
+        run: bun deploy/scripts/sync-content-repo.ts
 
       - name: Log in to GHCR
         uses: docker/login-action@v3
@@ -128,15 +118,18 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Wait for shared image tag
+      - name: Wait for image tag
         run: |
-          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:brain-${{ steps.user_config.outputs.brain_version }}"
-          for attempt in $(seq 1 24); do
+          # The Build workflow fires off the same config push (it derives the
+          # declared image set from the registry), so this wait must cover a
+          # full image build, not just registry propagation.
+          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:${{ steps.user_config.outputs.image_tag }}"
+          for attempt in $(seq 1 60); do
             if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
               exit 0
             fi
-            echo "Shared image tag not ready yet ($attempt/24): $IMAGE_TAG"
-            sleep 10
+            echo "Image tag not ready yet ($attempt/60): $IMAGE_TAG"
+            sleep 20
           done
           echo "Timed out waiting for $IMAGE_TAG" >&2
           exit 1
@@ -147,8 +140,6 @@ jobs:
           ruby -r rubygems -e 'puts Gem.user_dir + "/bin"' >> "$GITHUB_PATH"
 
       - name: Write Kamal SSH key
-        env:
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
         run: bun deploy/scripts/write-ssh-key.ts
 
       - name: Configure SSH client
@@ -165,38 +156,26 @@ jobs:
           chmod 600 ~/.ssh/config
 
       - name: Write .kamal/secrets
-        env:
-          SHARED_AI_API_KEY: ${{ secrets[steps.user_secrets.outputs.shared_ai_api_key_secret_name] }}
-          SHARED_GIT_SYNC_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_git_sync_token_secret_name] }}
-          SHARED_MCP_AUTH_TOKEN: ${{ secrets[steps.user_secrets.outputs.shared_mcp_auth_token_secret_name] }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
-        run: |
-          export AI_API_KEY="${AI_API_KEY:-$SHARED_AI_API_KEY}"
-          export GIT_SYNC_TOKEN="${GIT_SYNC_TOKEN:-$SHARED_GIT_SYNC_TOKEN}"
-          export MCP_AUTH_TOKEN="${MCP_AUTH_TOKEN:-$SHARED_MCP_AUTH_TOKEN}"
-          bun deploy/scripts/write-kamal-secrets.ts
+        run: bun deploy/scripts/write-kamal-secrets.ts
 
       - name: Provision server
         id: provision
         env:
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
           INSTANCE_NAME: ${{ steps.user_config.outputs.instance_name }}
         run: bun deploy/scripts/provision-server.ts
 
       - name: Update Cloudflare DNS
         env:
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+          CF_ZONE_ID: ${{ steps.user_config.outputs.cloudflare_zone_id }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
         run: |
           bun deploy/scripts/update-dns.ts
+          if [ -n "$WWW_DOMAIN" ]; then
+            BRAIN_DOMAIN="$WWW_DOMAIN" bun deploy/scripts/update-dns.ts
+          fi
           BRAIN_DOMAIN="$PREVIEW_DOMAIN" bun deploy/scripts/update-dns.ts
 
       - name: Validate SSH key
@@ -217,26 +196,54 @@ jobs:
           echo "SSH never became ready for $SSH_USER@$SERVER_IP" >&2
           exit 1
 
+      - name: Release operator-confirmed stale deploy lock
+        if: ${{ inputs.release_stale_lock }}
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
+          IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
+          REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
+          BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
+          BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
+        run: kamal lock release -c deploy/kamal/deploy.yml
+
       - name: Deploy
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
-          VERSION: brain-${{ steps.user_config.outputs.brain_version }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
           IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
           REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
         run: kamal setup --skip-push -c deploy/kamal/deploy.yml
 
+      - name: Install container health watchdog
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+        run: |
+          SSH_USER="$(ruby -e 'require "yaml"; config = YAML.load_file("deploy/kamal/deploy.yml") || {}; puts(config.dig("ssh", "user") || "root")')" \
+            bun deploy/scripts/install-health-watchdog.ts
+
       - name: Verify origin TLS
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
           PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
         run: |
-          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health"
+          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health/live"
+          if [ -n "$WWW_DOMAIN" ]; then
+            curl -I -k --max-time 20 --resolve "$WWW_DOMAIN:443:$SERVER_IP" "https://$WWW_DOMAIN/"
+          fi
           curl -I -k --max-time 20 --resolve "$PREVIEW_DOMAIN:443:$SERVER_IP" "https://$PREVIEW_DOMAIN/"
 
+      - name: Verify operational health
+        run: bunx brains-ops verify-user "$GITHUB_WORKSPACE" "$HANDLE"
+
       - name: Upload generated config
         uses: actions/upload-artifact@v4
         with:
@@ -271,7 +278,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
@@ -290,6 +297,12 @@ jobs:
 
       - name: Commit generated config
         run: |
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users views
+
           if git diff --quiet -- users views; then
             exit 0
           fi

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,21 +35,27 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
 
           git fetch origin "${{ github.ref_name }}"
-          if git diff --quiet "origin/${{ github.ref_name }}" -- views users; then
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
             exit 0
           fi
 
           patch_file="$(mktemp)"
-          git diff --binary -- views users > "$patch_file"
+          git diff --binary -- users > "$patch_file"
           git reset --hard "origin/${{ github.ref_name }}"
           git apply --3way --index "$patch_file"
 
-          if git diff --cached --quiet -- views users; then
+          if git diff --cached --quiet -- users; then
             exit 0
           fi
 

package/templates/rover-pilot/.github/workflows/upgrade.yml

@@ -0,0 +1,104 @@
+name: Upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Explicit @rizom/ops version (empty means latest)
+        required: false
+        type: string
+  schedule:
+    # Weekday morning check for new operator tooling releases.
+    - cron: "17 6 * * 1-5"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: upgrade
+  cancel-in-progress: false
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      # Check only presence here; the private key itself is never exposed to
+      # this step. A missing private-key secret fails before checkout.
+      - name: Require upgrade App configuration
+        env:
+          APP_ID: ${{ vars.OPS_UPGRADE_APP_ID }}
+          HAS_PRIVATE_KEY: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY != '' }}
+        run: |
+          missing=""
+          if [ -z "$APP_ID" ]; then
+            missing="OPS_UPGRADE_APP_ID"
+          fi
+          if [ "$HAS_PRIVATE_KEY" != "true" ]; then
+            missing="${missing:+$missing and }OPS_UPGRADE_APP_PRIVATE_KEY"
+          fi
+          if [ -n "$missing" ]; then
+            echo "$missing is not set for this repository." >&2
+            echo "Configure the upgrade GitHub App as described in docs/operator-playbook.md." >&2
+            exit 1
+          fi
+
+      # No credential is persisted in .git/config while the workflow installs
+      # and executes a freshly published @rizom/ops package.
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Upgrade operator tooling
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: bunx brains-ops upgrade . --to "${VERSION_INPUT:-latest}"
+
+      - name: Detect upgrade changes
+        id: upgrade-changes
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "Already up to date."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create repository-scoped upgrade token
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        id: upgrade-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.OPS_UPGRADE_APP_ID }}
+          private-key: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write
+
+      - name: Open upgrade PR
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ steps.upgrade-token.outputs.token }}
+        run: |
+          VERSION=$(bun -e 'console.log(JSON.parse(await Bun.file("package.json").text()).devDependencies["@rizom/ops"])')
+          BRANCH="ops/upgrade-rizom-ops-$VERSION"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A
+          git commit -m "chore(ops): upgrade @rizom/ops to $VERSION"
+          git push --force \
+            "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \
+            "$BRANCH"
+          if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            echo "PR already open for $BRANCH"
+            exit 0
+          fi
+          gh pr create \
+            --title "chore(ops): upgrade @rizom/ops to $VERSION" \
+            --body "Automated operator tooling upgrade. Review the scaffold diff, then merge; deploys stay operator-dispatched." \
+            --head "$BRANCH"

package/templates/rover-pilot/README.md

@@ -1,6 +1,6 @@
-# rover-pilot
+# private brain pilot
 
-Private desired-state repo for the rover pilot.
+Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
 
 This is a single operator-owned repo. Pilot users do not get their own brain repos.
 Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
@@ -29,18 +29,28 @@ The repo also checks in its deploy contract:
 - `.github/workflows/*`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
-The shared pilot image tag is `brain-${brainVersion}` end to end.
-When `pilot.yaml.brainVersion` changes and you push, CI rebuilds the shared tag, refreshes generated user env files, and redeploys affected users.
+Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
+The default pilot image tag is `brain-${brainVersion}` end to end. A user with `siteOverride` gets an isolated `brain-${brainVersion}-sites-${packageHash}` image instead.
+When the effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI rebuilds the required default/site tags, refreshes generated user env files, and redeploys affected users. Every external site and theme package uses its own required exact version pin; ops never infers either version from the brain or from another package.
 When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
 
 ## Commands
 
 - `brains-ops init <repo>`
-- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health`, and unauthenticated `/mcp` status checks
-- `brains-ops onboard <repo> <handle>`
+- `brains-ops upgrade <repo>` — bumps `@rizom/ops` (or `--to <version>`) and refreshes the generated scaffold; the Upgrade workflow runs this on a schedule and opens a PR
+- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
+- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
+- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
 - `brains-ops age-key:bootstrap <repo>`
 - `brains-ops ssh-key:bootstrap <repo>`
 - `brains-ops cert:bootstrap <repo>`
 - `brains-ops secrets:encrypt <repo> <handle>`
 - `brains-ops reconcile-cohort <repo> <cohort>`
 - `brains-ops reconcile-all <repo>`
+- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
+
+`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
+
+The Upgrade workflow uses a repository-scoped GitHub App token because an Actions `GITHUB_TOKEN` cannot update generated files under `.github/workflows/`. Checkout persists no credential, and the App token is minted only after the freshly published tooling finishes and only when there is a change to push. Configure the App and its `OPS_UPGRADE_APP_ID` Actions variable and `OPS_UPGRADE_APP_PRIVATE_KEY` Actions secret as described in `docs/operator-playbook.md`. If token creation or the upgrade push fails, stop; do not substitute an operator's personal credentials.
+
+Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.