@rizom/ops 0.2.0-alpha.34 → 0.2.0-alpha.341

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,119 @@
+name: Directory Sync Stress
+
+on:
+  schedule:
+    - cron: "23 4 * * 1"
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+      verify_only:
+        description: Verify the content credential path without remote writes or a workload
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Verify smoke directory-sync credential path
+        if: ${{ github.event_name != 'schedule' && inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:verify-access "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Run smoke directory-sync stress profile
+        if: ${{ github.event_name == 'schedule' || !inputs.verify_only }}
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          PROFILE_INPUT: ${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "$PROFILE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.event_name == 'schedule' && 'regression' || inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: ${{ always() && (github.event_name == 'schedule' || !inputs.verify_only) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}
+          CONFIRM_INPUT: ${{ github.event_name == 'schedule' && 'stress:smoke' || inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ github.event_name == 'schedule' && 'smoke' || inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/health-watchdog-smoke.yml

@@ -0,0 +1,95 @@
+name: Health Watchdog Smoke
+
+on:
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke fleet handle only
+        required: true
+        type: string
+        default: smoke
+      confirm:
+        description: Type watchdog-smoke:<handle> to confirm temporary container restarts
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-${{ inputs.handle }}
+  cancel-in-progress: false
+
+env:
+  WATCHDOG_SMOKE_RUN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Run fleet watchdog isolation smoke
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke"
+
+      - name: Upload watchdog smoke evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke
+          if-no-files-found: warn
+
+  cleanup:
+    needs: smoke
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual watchdog smoke fixtures
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke-cleanup"
+
+      - name: Upload watchdog smoke cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-cleanup-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/.github/workflows/reconcile.yml

@@ -35,21 +35,27 @@ jobs:
 
       - name: Commit generated outputs
         run: |
-          if git diff --quiet -- views users; then
+          # A newly added user's generated directory is untracked, and
+          # `git diff` cannot see untracked files — without intent-to-add
+          # the new config is silently dropped here and every later deploy
+          # skips that user (their files never appear in a commit range).
+          git add --intent-to-add -- users
+
+          if git diff --quiet -- users; then
             exit 0
           fi
 
           git fetch origin "${{ github.ref_name }}"
-          if git diff --quiet "origin/${{ github.ref_name }}" -- views users; then
+          if git diff --quiet "origin/${{ github.ref_name }}" -- users; then
             exit 0
           fi
 
           patch_file="$(mktemp)"
-          git diff --binary -- views users > "$patch_file"
+          git diff --binary -- users > "$patch_file"
           git reset --hard "origin/${{ github.ref_name }}"
           git apply --3way --index "$patch_file"
 
-          if git diff --cached --quiet -- views users; then
+          if git diff --cached --quiet -- users; then
             exit 0
           fi
 

package/templates/rover-pilot/.github/workflows/upgrade.yml

@@ -0,0 +1,104 @@
+name: Upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Explicit @rizom/ops version (empty means latest)
+        required: false
+        type: string
+  schedule:
+    # Weekday morning check for new operator tooling releases.
+    - cron: "17 6 * * 1-5"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: upgrade
+  cancel-in-progress: false
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      # Check only presence here; the private key itself is never exposed to
+      # this step. A missing private-key secret fails before checkout.
+      - name: Require upgrade App configuration
+        env:
+          APP_ID: ${{ vars.OPS_UPGRADE_APP_ID }}
+          HAS_PRIVATE_KEY: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY != '' }}
+        run: |
+          missing=""
+          if [ -z "$APP_ID" ]; then
+            missing="OPS_UPGRADE_APP_ID"
+          fi
+          if [ "$HAS_PRIVATE_KEY" != "true" ]; then
+            missing="${missing:+$missing and }OPS_UPGRADE_APP_PRIVATE_KEY"
+          fi
+          if [ -n "$missing" ]; then
+            echo "$missing is not set for this repository." >&2
+            echo "Configure the upgrade GitHub App as described in docs/operator-playbook.md." >&2
+            exit 1
+          fi
+
+      # No credential is persisted in .git/config while the workflow installs
+      # and executes a freshly published @rizom/ops package.
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Upgrade operator tooling
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: bunx brains-ops upgrade . --to "${VERSION_INPUT:-latest}"
+
+      - name: Detect upgrade changes
+        id: upgrade-changes
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "Already up to date."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create repository-scoped upgrade token
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        id: upgrade-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.OPS_UPGRADE_APP_ID }}
+          private-key: ${{ secrets.OPS_UPGRADE_APP_PRIVATE_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+          permission-workflows: write
+
+      - name: Open upgrade PR
+        if: steps.upgrade-changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ steps.upgrade-token.outputs.token }}
+        run: |
+          VERSION=$(bun -e 'console.log(JSON.parse(await Bun.file("package.json").text()).devDependencies["@rizom/ops"])')
+          BRANCH="ops/upgrade-rizom-ops-$VERSION"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$BRANCH"
+          git add -A
+          git commit -m "chore(ops): upgrade @rizom/ops to $VERSION"
+          git push --force \
+            "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \
+            "$BRANCH"
+          if [ -n "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            echo "PR already open for $BRANCH"
+            exit 0
+          fi
+          gh pr create \
+            --title "chore(ops): upgrade @rizom/ops to $VERSION" \
+            --body "Automated operator tooling upgrade. Review the scaffold diff, then merge; deploys stay operator-dispatched." \
+            --head "$BRANCH"

package/templates/rover-pilot/README.md

@@ -1,6 +1,6 @@
-# rover-pilot
+# private brain pilot
 
-Private desired-state repo for the rover pilot.
+Private desired-state repository for the hosted brain pilot. The generated repository/service name may remain `rover-pilot` for deployment identity compatibility.
 
 This is a single operator-owned repo. Pilot users do not get their own brain repos.
 Per-user deploy config lives under `users/<handle>/`, while content stays in per-user content repos.
@@ -29,18 +29,28 @@ The repo also checks in its deploy contract:
 - `.github/workflows/*`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
-The shared pilot image tag is `brain-${brainVersion}` end to end.
-When `pilot.yaml.brainVersion` changes and you push, CI rebuilds the shared tag, refreshes generated user env files, and redeploys affected users.
+Use separate GitHub tokens: `CONTENT_REPO_ADMIN_TOKEN` for operator-side content repo creation/checks, and `GIT_SYNC_TOKEN` for runtime directory-sync git access.
+The default pilot image tag is `brain-${brainVersion}` end to end. A user with `siteOverride` gets an isolated `brain-${brainVersion}-sites-${packageHash}` image instead.
+When the effective brain version (`pilot.yaml.brainVersion`, or a cohort override) changes and you push, CI rebuilds the required default/site tags, refreshes generated user env files, and redeploys affected users. Every external site and theme package uses its own required exact version pin; ops never infers either version from the brain or from another package.
 When a push changes only deploy contract files, CI prints `No affected user configs; skipping deploy.` and stops before Kamal.
 
 ## Commands
 
 - `brains-ops init <repo>`
-- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health`, and unauthenticated `/mcp` status checks
-- `brains-ops onboard <repo> <handle>`
+- `brains-ops upgrade <repo>` — bumps `@rizom/ops` (or `--to <version>`) and refreshes the generated scaffold; the Upgrade workflow runs this on a schedule and opens a PR
+- `brains-ops render <repo>` — regenerates `views/users.md` with live DNS, `/health/ready`, and unauthenticated `/mcp` status checks
+- `brains-ops user:add <repo> <handle> --cohort <cohort>` — scaffolds a user file, per-user secrets template, and cohort membership
+- `brains-ops onboard <repo> <handle>` — creates/seeds the user's content repo with separate admin and sync tokens
 - `brains-ops age-key:bootstrap <repo>`
 - `brains-ops ssh-key:bootstrap <repo>`
 - `brains-ops cert:bootstrap <repo>`
 - `brains-ops secrets:encrypt <repo> <handle>`
 - `brains-ops reconcile-cohort <repo> <cohort>`
 - `brains-ops reconcile-all <repo>`
+- `brains-ops reconcile-all <repo> --dry-run` (isolated, no external content-repository access; lists both passes' changed files)
+
+`render` owns the observational `views/users.md` projection. Reconcile owns generated per-user config and never rewrites observed status rows.
+
+The Upgrade workflow uses a repository-scoped GitHub App token because an Actions `GITHUB_TOKEN` cannot update generated files under `.github/workflows/`. Checkout persists no credential, and the App token is minted only after the freshly published tooling finishes and only when there is a change to push. Configure the App and its `OPS_UPGRADE_APP_ID` Actions variable and `OPS_UPGRADE_APP_PRIVATE_KEY` Actions secret as described in `docs/operator-playbook.md`. If token creation or the upgrade push fails, stop; do not substitute an operator's personal credentials.
+
+Use `docs/canonical-crossover-record.md` to record exact forward and rollback artifact pins before an approved crossover window.

package/templates/rover-pilot/deploy/scripts/decrypt-user-secrets.ts

@@ -15,13 +15,52 @@ const decrypter = new Decrypter();
 decrypter.addIdentity(ageSecretKey);
 
 const plaintext = await decrypter.decrypt(decoded, "text");
-const secrets = parseFlatYaml(plaintext);
-const pilot = parseFlatYaml(readFileSync("pilot.yaml", "utf8"));
-
-writeGitHubEnv("AI_API_KEY", secrets["aiApiKey"] ?? "");
-writeGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"] ?? "");
-writeGitHubEnv("MCP_AUTH_TOKEN", secrets["mcpAuthToken"] ?? "");
-writeGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"] ?? "");
+const secrets = parseStringYamlMapping(plaintext);
+const pilot = parseStringYamlMapping(readFileSync("pilot.yaml", "utf8"));
+
+writeSecretGitHubEnv("AI_API_KEY", secrets["aiApiKey"]);
+writeSecretGitHubEnv("GIT_SYNC_TOKEN", secrets["gitSyncToken"]);
+writeSecretGitHubEnv(
+  "STUDIO_CONTENT_REPO_PAT",
+  secrets["studioContentRepoPat"] ?? secrets["gitSyncToken"],
+);
+writeSecretGitHubEnv("DISCORD_BOT_TOKEN", secrets["discordBotToken"]);
+writeSecretGitHubEnv("DISCORD_PUBLIC_KEY", secrets["discordPublicKey"]);
+writeSecretGitHubEnv("DISCORD_APPLICATION_ID", secrets["discordApplicationId"]);
+// Per-user AT Protocol publishing credential (optional; from the user's
+// encrypted secrets file). The scaffold wires this so a pilot can publish its
+// brain's agent card to its PDS; a deployment that doesn't publish simply
+// leaves it unset.
+writeSecretGitHubEnv("ATPROTO_APP_PASSWORD", secrets["atprotoAppPassword"]);
+// Per-user custom-domain TLS overrides the shared fleet certificate when a
+// complete certificate/key pair is present in the encrypted user secrets.
+const certificatePem = decodeEscapedSecret(secrets["certificatePem"]);
+const privateKeyPem = decodeEscapedSecret(secrets["privateKeyPem"]);
+if (Boolean(certificatePem) !== Boolean(privateKeyPem)) {
+  throw new Error(
+    "Custom-domain TLS secrets require both certificatePem and privateKeyPem",
+  );
+}
+// A corrupted stored value would otherwise only surface as a kamal-proxy
+// "unable to load certificate" failure after the container has already booted.
+assertPemShape("certificatePem", certificatePem);
+assertPemShape("privateKeyPem", privateKeyPem);
+writeSecretGitHubEnv("CERTIFICATE_PEM", certificatePem);
+writeSecretGitHubEnv("PRIVATE_KEY_PEM", privateKeyPem);
+
+function assertPemShape(name: string, value: string | undefined): void {
+  if (value === undefined || value.trim().length === 0) {
+    return;
+  }
+  if (
+    !/-----BEGIN [A-Z0-9 ]+-----/.test(value) ||
+    !/-----END [A-Z0-9 ]+-----/.test(value)
+  ) {
+    throw new Error(
+      `Secret ${name} is not PEM-shaped after unescaping; the stored value is corrupt`,
+    );
+  }
+}
 
 writeGitHubOutput(
   "shared_ai_api_key_secret_name",
@@ -32,10 +71,29 @@ writeGitHubOutput(
   requireFlatValue(pilot, "gitSyncToken", "pilot.yaml"),
 );
 writeGitHubOutput(
-  "shared_mcp_auth_token_secret_name",
-  requireFlatValue(pilot, "mcpAuthToken", "pilot.yaml"),
+  "shared_content_repo_admin_token_secret_name",
+  requireFlatValue(pilot, "contentRepoAdminToken", "pilot.yaml"),
 );
 
+function writeSecretGitHubEnv(name: string, value: string | undefined): void {
+  if (!value || value.trim().length === 0) {
+    return;
+  }
+
+  maskGitHubSecret(value);
+  writeGitHubEnv(name, value);
+}
+
+function maskGitHubSecret(value: string): void {
+  const escaped = value
+    .replace(/%/g, "%25")
+    .replace(/\r/g, "%0D")
+    .replace(/\n/g, "%0A");
+  if (escaped.length > 0 && process.env["GITHUB_ACTIONS"] === "true") {
+    console.log(`::add-mask::${escaped}`);
+  }
+}
+
 function extractAgeIdentity(contents: string): string {
   const line = contents
     .split(/\r?\n/)
@@ -49,27 +107,25 @@ function extractAgeIdentity(contents: string): string {
   return line;
 }
 
-function parseFlatYaml(contents: string): Record<string, string> {
-  const result: Record<string, string> = {};
-
-  for (const rawLine of contents.split(/\r?\n/)) {
-    const line = rawLine.trim();
-    if (!line || line.startsWith("#")) {
-      continue;
-    }
+function parseStringYamlMapping(contents: string): Record<string, string> {
+  const parsed: unknown = Bun.YAML.parse(contents);
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Expected a YAML mapping");
+  }
 
-    const match = line.match(/^([A-Za-z0-9_]+):\s*(.*)$/);
-    if (!match) {
-      continue;
+  const result: Record<string, string> = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    if (typeof value === "string") {
+      result[key] = value;
     }
-
-    const [, key, rawValue] = match;
-    result[key] = rawValue.replace(/^['"]|['"]$/g, "");
   }
-
   return result;
 }
 
+function decodeEscapedSecret(value: string | undefined): string | undefined {
+  return value?.replace(/\\n/g, "\n");
+}
+
 function requireFlatValue(
   values: Record<string, string>,
   key: string,

package/templates/rover-pilot/deploy/scripts/helpers.ts

@@ -6,5 +6,8 @@ export {
   requireEnv,
   writeGitHubOutput,
   writeGitHubEnv,
+  siteImageTag,
+  sitePackagesFor,
+  runResolveMissingImages,
 } from "@rizom/ops/deploy";
 export type { EnvSchemaEntry } from "@rizom/ops/deploy";