@rizom/ops 0.2.0-alpha.6 → 0.2.0-alpha.600

package/templates/rover-pilot/.github/workflows/deploy.yml

@@ -7,13 +7,24 @@ on:
         description: User handle
         required: true
         type: string
+      release_stale_lock:
+        description: Release an operator-confirmed stale Kamal deploy lock before retrying
+        required: false
+        type: boolean
+        default: false
   push:
     branches: ["main"]
     paths:
       - users/*/.env
       - users/*/brain.yaml
+      - users/*/content/**
+      - users/*.secrets.yaml.age
       - deploy/**
       - .github/workflows/deploy.yml
+  workflow_run:
+    workflows: [Reconcile]
+    types: [completed]
+    branches: [main]
 
 permissions:
   contents: write
@@ -21,13 +32,16 @@ permissions:
 
 jobs:
   resolve_handles:
+    if: >
+      github.event_name != 'workflow_run' ||
+      github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     outputs:
       handles_json: ${{ steps.resolve.outputs.handles_json }}
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
           fetch-depth: 0
 
       - uses: oven-sh/setup-bun@v2
@@ -39,7 +53,7 @@ jobs:
         id: resolve
         env:
           HANDLE_INPUT: ${{ inputs.handle || '' }}
-          BEFORE_SHA: ${{ github.event.before || '' }}
+          BEFORE_SHA: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.event.before || '' }}
         run: bun deploy/scripts/resolve-deploy-handles.ts
 
   no_changes:
@@ -66,13 +80,22 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
       - name: Install operator tooling
         run: bun install
 
+      - name: Load shared env via Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Decrypt user secrets
+        id: user_secrets
+        run: bun deploy/scripts/decrypt-user-secrets.ts "$HANDLE"
+
       - name: Reconcile selected user config
         run: bunx brains-ops onboard "$GITHUB_WORKSPACE" "$HANDLE"
 
@@ -81,23 +104,13 @@ jobs:
         run: bun deploy/scripts/resolve-user-config.ts
 
       - name: Validate selected secrets
-        env:
-          AI_API_KEY: ${{ secrets[steps.user_config.outputs.ai_api_key_secret_name] }}
-          GIT_SYNC_TOKEN: ${{ secrets[steps.user_config.outputs.git_sync_token_secret_name] }}
-          MCP_AUTH_TOKEN: ${{ secrets[steps.user_config.outputs.mcp_auth_token_secret_name] }}
-          DISCORD_BOT_TOKEN: ${{ steps.user_config.outputs.discord_bot_token_secret_name != '' && secrets[steps.user_config.outputs.discord_bot_token_secret_name] || '' }}
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
         run: bun deploy/scripts/validate-secrets.ts
 
+      - name: Seed content repo
+        env:
+          CONTENT_REPO: ${{ steps.user_config.outputs.content_repo }}
+        run: bun deploy/scripts/sync-content-repo.ts
+
       - name: Log in to GHCR
         uses: docker/login-action@v3
         with:
@@ -105,27 +118,31 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Wait for shared image tag
+      - name: Wait for image tag
         run: |
-          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:brain-${{ steps.user_config.outputs.brain_version }}"
-          for attempt in $(seq 1 24); do
+          # The Build workflow fires off the same config push (it derives the
+          # declared image set from the registry), so this wait must cover a
+          # full image build, not just registry propagation.
+          IMAGE_TAG="${{ steps.user_config.outputs.image_repository }}:${{ steps.user_config.outputs.image_tag }}"
+          for attempt in $(seq 1 60); do
             if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
               exit 0
             fi
-            echo "Shared image tag not ready yet ($attempt/24): $IMAGE_TAG"
-            sleep 10
+            echo "Image tag not ready yet ($attempt/60): $IMAGE_TAG"
+            sleep 20
           done
           echo "Timed out waiting for $IMAGE_TAG" >&2
           exit 1
 
+      - name: Verify runtime image packages
+        run: bun deploy/scripts/verify-runtime-image.ts
+
       - name: Install Kamal
         run: |
           gem install --user-install kamal
           ruby -r rubygems -e 'puts Gem.user_dir + "/bin"' >> "$GITHUB_PATH"
 
       - name: Write Kamal SSH key
-        env:
-          KAMAL_SSH_PRIVATE_KEY: ${{ secrets.KAMAL_SSH_PRIVATE_KEY }}
         run: bun deploy/scripts/write-ssh-key.ts
 
       - name: Configure SSH client
@@ -142,33 +159,27 @@ jobs:
           chmod 600 ~/.ssh/config
 
       - name: Write .kamal/secrets
-        env:
-          AI_API_KEY: ${{ secrets[steps.user_config.outputs.ai_api_key_secret_name] }}
-          GIT_SYNC_TOKEN: ${{ secrets[steps.user_config.outputs.git_sync_token_secret_name] }}
-          MCP_AUTH_TOKEN: ${{ secrets[steps.user_config.outputs.mcp_auth_token_secret_name] }}
-          DISCORD_BOT_TOKEN: ${{ steps.user_config.outputs.discord_bot_token_secret_name != '' && secrets[steps.user_config.outputs.discord_bot_token_secret_name] || '' }}
-          KAMAL_REGISTRY_PASSWORD: ${{ secrets.KAMAL_REGISTRY_PASSWORD }}
-          CERTIFICATE_PEM: ${{ secrets.CERTIFICATE_PEM }}
-          PRIVATE_KEY_PEM: ${{ secrets.PRIVATE_KEY_PEM }}
         run: bun deploy/scripts/write-kamal-secrets.ts
 
       - name: Provision server
         id: provision
         env:
-          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
-          HCLOUD_SSH_KEY_NAME: ${{ secrets.HCLOUD_SSH_KEY_NAME }}
-          HCLOUD_SERVER_TYPE: ${{ secrets.HCLOUD_SERVER_TYPE }}
-          HCLOUD_LOCATION: ${{ secrets.HCLOUD_LOCATION }}
           INSTANCE_NAME: ${{ steps.user_config.outputs.instance_name }}
         run: bun deploy/scripts/provision-server.ts
 
       - name: Update Cloudflare DNS
         env:
-          CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
-          CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+          CF_ZONE_ID: ${{ steps.user_config.outputs.cloudflare_zone_id }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
-        run: bun deploy/scripts/update-dns.ts
+        run: |
+          bun deploy/scripts/update-dns.ts
+          if [ -n "$WWW_DOMAIN" ]; then
+            BRAIN_DOMAIN="$WWW_DOMAIN" bun deploy/scripts/update-dns.ts
+          fi
+          BRAIN_DOMAIN="$PREVIEW_DOMAIN" bun deploy/scripts/update-dns.ts
 
       - name: Validate SSH key
         run: ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null
@@ -188,21 +199,61 @@ jobs:
           echo "SSH never became ready for $SSH_USER@$SERVER_IP" >&2
           exit 1
 
+      - name: Create verified predeploy backup
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          TARGET_HANDLE: ${{ matrix.handle }}
+          TARGET_VERSION: ${{ steps.user_config.outputs.image_tag }}
+          SERVICE_NAME: rover
+        run: bun deploy/scripts/create-predeploy-backup.ts
+
+      - name: Release operator-confirmed stale deploy lock
+        if: ${{ inputs.release_stale_lock }}
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
+          IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
+          REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
+          BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
+          BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
+        run: kamal lock release -c deploy/kamal/deploy.yml
+
       - name: Deploy
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
-          VERSION: brain-${{ steps.user_config.outputs.brain_version }}
+          VERSION: ${{ steps.user_config.outputs.image_tag }}
           IMAGE_REPOSITORY: ${{ steps.user_config.outputs.image_repository }}
           REGISTRY_USERNAME: ${{ steps.user_config.outputs.registry_username }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
           BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
         run: kamal setup --skip-push -c deploy/kamal/deploy.yml
 
+      - name: Install container health watchdog
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+        run: |
+          SSH_USER="$(ruby -e 'require "yaml"; config = YAML.load_file("deploy/kamal/deploy.yml") || {}; puts(config.dig("ssh", "user") || "root")')" \
+            bun deploy/scripts/install-health-watchdog.ts
+
       - name: Verify origin TLS
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}
           BRAIN_DOMAIN: ${{ steps.user_config.outputs.brain_domain }}
-        run: curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health"
+          WWW_DOMAIN: ${{ steps.user_config.outputs.www_domain }}
+          PREVIEW_DOMAIN: ${{ steps.user_config.outputs.preview_domain }}
+        run: |
+          curl -I -k --max-time 20 --resolve "$BRAIN_DOMAIN:443:$SERVER_IP" "https://$BRAIN_DOMAIN/health/live"
+          if [ -n "$WWW_DOMAIN" ]; then
+            curl -I -k --max-time 20 --resolve "$WWW_DOMAIN:443:$SERVER_IP" "https://$WWW_DOMAIN/"
+          fi
+          curl -I -k --max-time 20 --resolve "$PREVIEW_DOMAIN:443:$SERVER_IP" "https://$PREVIEW_DOMAIN/"
+
+      - name: Verify operational health
+        run: bunx brains-ops verify-user "$GITHUB_WORKSPACE" "$HANDLE"
 
       - name: Upload generated config
         uses: actions/upload-artifact@v4
@@ -211,6 +262,7 @@ jobs:
           path: |
             users/${{ matrix.handle }}/brain.yaml
             users/${{ matrix.handle }}/.env
+            users/${{ matrix.handle }}/content
 
       - name: Dump remote proxy diagnostics
         if: failure()
@@ -237,7 +289,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         with:
-          ref: ${{ github.sha }}
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.sha }}
 
       - uses: oven-sh/setup-bun@v2
 
@@ -251,16 +303,33 @@ jobs:
           merge-multiple: true
           path: .
 
-      - name: Regenerate users table
-        run: bunx brains-ops render "$GITHUB_WORKSPACE"
-
       - name: Commit generated config
         run: |
-          if git diff --quiet -- users views; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          patch_file="$(mktemp)"
+          git diff --binary -- users > "$patch_file"
+          git fetch origin "${{ github.ref_name }}"
+          git reset --hard "origin/${{ github.ref_name }}"
+          if [ -s "$patch_file" ]; then
+            git apply --3way --index "$patch_file"
+          fi
+
+          # views/users.md is generated, and a concurrent deploy may have
+          # rewritten it since this run's checkout; render it on fresh main
+          # instead of merging a stale copy.
+          bunx brains-ops render "$GITHUB_WORKSPACE"
+          git add -- views
+
+          if git diff --cached --quiet -- users views; then
             exit 0
           fi
+
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add users views
           git commit -m "chore(ops): reconcile generated config"
-          git push
+          git push origin HEAD:${{ github.ref_name }}

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/health-watchdog-smoke.yml

@@ -0,0 +1,95 @@
+name: Health Watchdog Smoke
+
+on:
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke fleet handle only
+        required: true
+        type: string
+        default: smoke
+      confirm:
+        description: Type watchdog-smoke:<handle> to confirm temporary container restarts
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-${{ inputs.handle }}
+  cancel-in-progress: false
+
+env:
+  WATCHDOG_SMOKE_RUN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Run fleet watchdog isolation smoke
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke"
+
+      - name: Upload watchdog smoke evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke
+          if-no-files-found: warn
+
+  cleanup:
+    needs: smoke
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual watchdog smoke fixtures
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke-cleanup"
+
+      - name: Upload watchdog smoke cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-cleanup-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/offboard.yml

@@ -0,0 +1,107 @@
+name: Offboard
+
+on:
+  workflow_dispatch:
+    inputs:
+      handles:
+        description: Comma-separated pilot handles
+        required: true
+        type: string
+      confirm:
+        description: Exact confirmation (sunset:<sorted-comma-separated-handles>)
+        required: true
+        type: string
+      apply:
+        description: Archive content repos and destroy DNS, servers, and desired state
+        required: true
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+
+concurrency:
+  group: offboard-pilot-users
+  cancel-in-progress: false
+
+jobs:
+  offboard:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: main
+          fetch-depth: 0
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load provider credentials
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Resolve handles
+        id: handles
+        env:
+          OFFBOARD_HANDLES: ${{ inputs.handles }}
+        run: |
+          node <<'NODE'
+          const handles = [...new Set(process.env.OFFBOARD_HANDLES
+            .split(',')
+            .map((handle) => handle.trim())
+            .filter(Boolean))]
+            .sort((left, right) => left.localeCompare(right));
+          if (handles.length === 0 || handles.some((handle) => !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(handle))) {
+            throw new Error('Invalid offboarding handle list');
+          }
+          const output = process.env.GITHUB_OUTPUT;
+          require('node:fs').appendFileSync(output, `json=${JSON.stringify(handles)}\n`);
+          NODE
+
+      - name: Preview offboarding plan
+        env:
+          OFFBOARD_HANDLES_JSON: ${{ steps.handles.outputs.json }}
+        run: |
+          mapfile -t handles < <(node -e 'for (const handle of JSON.parse(process.env.OFFBOARD_HANDLES_JSON)) console.log(handle)')
+          bunx brains-ops user:offboard "$GITHUB_WORKSPACE" "${handles[@]}"
+
+      - name: Apply offboarding plan
+        if: inputs.apply
+        env:
+          OFFBOARD_CONFIRM: ${{ inputs.confirm }}
+          OFFBOARD_HANDLES_JSON: ${{ steps.handles.outputs.json }}
+        run: |
+          mapfile -t handles < <(node -e 'for (const handle of JSON.parse(process.env.OFFBOARD_HANDLES_JSON)) console.log(handle)')
+          bunx brains-ops user:offboard "$GITHUB_WORKSPACE" "${handles[@]}" \
+            --apply --confirm "$OFFBOARD_CONFIRM"
+
+      - name: Validate changed paths
+        if: inputs.apply
+        run: |
+          unexpected="$(git status --short | awk '{print $2}' | grep -Ev '^(cohorts/|users/|views/users\.md$)' || true)"
+          if [ -n "$unexpected" ]; then
+            echo "Offboarding changed unexpected paths:" >&2
+            echo "$unexpected" >&2
+            exit 1
+          fi
+
+      - name: Commit desired-state removal
+        if: inputs.apply
+        env:
+          OFFBOARD_HANDLES: ${{ inputs.handles }}
+        run: |
+          git add --all -- cohorts users views/users.md
+          if git diff --cached --quiet; then
+            echo "No desired-state changes remain to commit."
+            exit 0
+          fi
+
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "chore(ops): sunset pilot brains" \
+            -m "Retired handles: $OFFBOARD_HANDLES"
+          git pull --rebase origin main
+          git push origin HEAD:main

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,11 +35,31 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
+
+          git fetch origin "${{ github.ref_name }}"
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
+            exit 0
+          fi
+
+          patch_file="$(mktemp)"
+          git diff --binary -- users > "$patch_file"
+          git reset --hard "origin/${{ github.ref_name }}"
+          git apply --3way --index "$patch_file"
+
+          if git diff --cached --quiet -- users; then
+            exit 0
+          fi
+
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add views users
           git commit -m "chore(ops): reconcile pilot outputs"
-          git push
+          git push origin HEAD:${{ github.ref_name }}