@rizom/ops 0.2.0-alpha.254 → 0.2.0-alpha.256

package/dist/run-command.d.ts

@@ -5,6 +5,7 @@ import { type LookupHost } from "./observed-status";
 import { type RunCommand as OpsRunCommand } from "./run-subprocess";
 import { type SshKeygen } from "./ssh-key-bootstrap";
 import type { UserRunner } from "./user-runner";
+import type { cleanupDirectorySyncStress, runDeployedDirectorySyncStress } from "./directory-sync-stress-system";
 export interface CommandResult {
     success: boolean;
     message?: string;
@@ -17,6 +18,8 @@ export interface CommandDependencies extends LoadPilotRegistryOptions {
     lookupHost?: LookupHost | undefined;
     bootstrapRunCommand?: OpsRunCommand | undefined;
     sshKeygen?: SshKeygen | undefined;
+    directorySyncStressRunner?: typeof runDeployedDirectorySyncStress | undefined;
+    directorySyncStressCleanupRunner?: typeof cleanupDirectorySyncStress | undefined;
 }
 export declare const globalFlags: FlagDefinitions;
 export declare const commands: readonly CommandDefinition<CommandDependencies, CommandResult>[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rizom/ops",
-  "version": "0.2.0-alpha.254",
+  "version": "0.2.0-alpha.256",
   "description": "Operator CLI for managing private brain fleet registry repos",
   "keywords": [
     "brains",

package/templates/rover-pilot/.github/actions/varlock-env/action.yml

@@ -0,0 +1,47 @@
+name: Load Bitwarden environment through Varlock
+description: Load, mask, and export the pilot environment from Varlock
+
+runs:
+  using: composite
+  steps:
+    - name: Load shared environment
+      shell: bash
+      run: |
+        for attempt in 1 2 3; do
+          if bunx varlock@1.1.0 load --path .env.schema --format json --compact > /tmp/varlock-env.json; then
+            break
+          fi
+          if [ "$attempt" = "3" ]; then
+            exit 1
+          fi
+          echo "Varlock load failed (attempt $attempt/3); retrying in 5s..."
+          sleep 5
+        done
+        node <<'NODE'
+        import { appendFileSync, readFileSync, rmSync } from "node:fs";
+        const env = JSON.parse(readFileSync("/tmp/varlock-env.json", "utf8"));
+        const output = process.env.GITHUB_ENV;
+        if (!output) throw new Error("Missing GITHUB_ENV");
+        const newline = "\n";
+        const chunks = [];
+        for (const [key, value] of Object.entries(env)) {
+          if (key === "BWS_ACCESS_TOKEN" || value === null || value === undefined) continue;
+          const text = String(value).replace(/\r\n/g, newline);
+          const escapedMask = text
+            .replace(/%/g, "%25")
+            .replace(/\r/g, "%0D")
+            .replace(/\n/g, "%0A");
+          if (escapedMask.length > 0) {
+            process.stdout.write(`::add-mask::${escapedMask}${newline}`);
+          }
+          if (text.includes(newline)) {
+            let delimiter = `VARLOCK_${key}_EOF`;
+            while (text.includes(delimiter)) delimiter += "_X";
+            chunks.push(`${key}<<${delimiter}${newline}${text}${newline}${delimiter}`);
+          } else {
+            chunks.push(`${key}=${text}`);
+          }
+        }
+        appendFileSync(output, `${chunks.join(newline)}${newline}`);
+        rmSync("/tmp/varlock-env.json", { force: true });
+        NODE

package/templates/rover-pilot/.github/workflows/deploy.yml

@@ -87,60 +87,10 @@ jobs:
       - name: Install operator tooling
         run: bun install
 
-      - name: Load shared env via varlock
+      - name: Load shared env via Varlock
+        uses: ./.github/actions/varlock-env
         env:
           BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
-        run: |
-          for attempt in 1 2 3; do
-            if bunx varlock@1.1.0 load --path .env.schema --format json --compact > /tmp/varlock-env.json; then
-              break
-            fi
-            if [ "$attempt" = "3" ]; then
-              exit 1
-            fi
-            echo "Varlock load failed (attempt $attempt/3); retrying in 5s..."
-            sleep 5
-          done
-          node <<'NODE'
-          import { appendFileSync, readFileSync } from "node:fs";
-          const env = JSON.parse(readFileSync('/tmp/varlock-env.json', 'utf8'));
-          const githubEnvPath = process.env.GITHUB_ENV;
-          if (!githubEnvPath) {
-            throw new Error('Missing GITHUB_ENV');
-          }
-
-          const newline = String.fromCharCode(10);
-          const carriageReturn = String.fromCharCode(13);
-          const chunks = [];
-          for (const [key, value] of Object.entries(env)) {
-            if (key === 'BWS_ACCESS_TOKEN' || value === null || value === undefined) {
-              continue;
-            }
-
-            const text = String(value)
-              .split(carriageReturn + newline)
-              .join(newline);
-            const escapedMask = text
-              .replace(/%/g, '%25')
-              .replace(/\r/g, '%0D')
-              .replace(/\n/g, '%0A');
-            if (escapedMask.length > 0) {
-              process.stdout.write('::add-mask::' + escapedMask + newline);
-            }
-
-            if (text.includes(newline)) {
-              let delimiter = `VARLOCK_${key}_EOF`;
-              while (text.includes(delimiter)) {
-                delimiter += '_X';
-              }
-              chunks.push(`${key}<<${delimiter}${newline}${text}${newline}${delimiter}`);
-            } else {
-              chunks.push(`${key}=${text}`);
-            }
-          }
-
-          appendFileSync(githubEnvPath, chunks.join(newline) + newline);
-          NODE
 
       - name: Decrypt user secrets
         id: user_secrets
@@ -271,6 +221,13 @@ jobs:
           BRAIN_YAML_PATH: ${{ steps.user_config.outputs.brain_yaml_path }}
         run: kamal setup --skip-push -c deploy/kamal/deploy.yml
 
+      - name: Install container health watchdog
+        env:
+          SERVER_IP: ${{ steps.provision.outputs.server_ip }}
+        run: |
+          SSH_USER="$(ruby -e 'require "yaml"; config = YAML.load_file("deploy/kamal/deploy.yml") || {}; puts(config.dig("ssh", "user") || "root")')" \
+            bun deploy/scripts/install-health-watchdog.ts
+
       - name: Verify origin TLS
         env:
           SERVER_IP: ${{ steps.provision.outputs.server_ip }}

package/templates/rover-pilot/.github/workflows/directory-sync-stress.yml

@@ -0,0 +1,100 @@
+name: Directory Sync Stress
+
+on:
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke handle only
+        required: true
+        type: string
+        default: smoke
+      profile:
+        description: Workload profile
+        required: true
+        type: choice
+        default: regression
+        options:
+          - regression
+          - load
+          - stress
+      confirm:
+        description: Type stress:<handle> to confirm reversible content writes
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: directory-sync-stress-${{ inputs.handle }}
+  cancel-in-progress: false
+
+jobs:
+  stress:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Run smoke directory-sync stress profile
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --profile "${{ inputs.profile }}" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress"
+
+      - name: Upload stress evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-${{ inputs.handle }}-${{ inputs.profile }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress
+          if-no-files-found: warn
+
+  cleanup:
+    needs: stress
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual stress probes
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops stress:directory-sync:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --artifacts-dir "$RUNNER_TEMP/directory-sync-stress-cleanup"
+
+      - name: Upload cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: directory-sync-stress-cleanup-${{ inputs.handle }}-${{ github.run_id }}
+          path: ${{ runner.temp }}/directory-sync-stress-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/deploy/scripts/install-health-watchdog.ts

@@ -0,0 +1,133 @@
+import { requireEnv } from "./helpers";
+
+const WATCHDOG_PATH = "/usr/local/sbin/brains-health-watchdog";
+const SERVICE_PATH = "/etc/systemd/system/brains-health-watchdog.service";
+const TIMER_PATH = "/etc/systemd/system/brains-health-watchdog.timer";
+
+export const healthWatchdogScript: string = `#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+lock_path="\${BRAIN_WATCHDOG_LOCK_PATH:-/run/brains-health-watchdog.lock}"
+exec 9>"$lock_path"
+flock -n 9 || exit 0
+
+incident_dir="\${BRAIN_WATCHDOG_INCIDENT_DIR:-/var/log/brains-health-watchdog}"
+state_dir="\${BRAIN_WATCHDOG_STATE_DIR:-/var/lib/brains-health-watchdog}"
+MAX_RESTARTS=3
+WINDOW_SECONDS=3600
+mkdir -p "$incident_dir" "$state_dir"
+
+write_safe_inspect() {
+  docker inspect --format '{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State}},"restartPolicy":{{json .HostConfig.RestartPolicy}}}' "$1"
+}
+
+while read -r container_id; do
+  [ -n "$container_id" ] || continue
+  service_name=$(docker inspect --format '{{ index .Config.Labels "service" }}' "$container_id" 2>/dev/null || true)
+  container_name=$(docker inspect --format '{{ .Name }}' "$container_id" 2>/dev/null | tr -d '/' || true)
+  state_key=$(printf '%s-%s' "$service_name" "$container_name" | tr -c 'A-Za-z0-9_.-' '_')
+  state_path="$state_dir/$state_key.restarts"
+  recent_path="$state_path.tmp"
+  now_epoch=$(date -u +%s)
+  cutoff_epoch=$((now_epoch - WINDOW_SECONDS))
+  touch "$state_path"
+  awk -v cutoff="$cutoff_epoch" '$1 >= cutoff' "$state_path" >"$recent_path"
+  restart_count=$(wc -l <"$recent_path")
+
+  timestamp=$(date -u +%Y%m%dT%H%M%SZ)
+  incident_path="$incident_dir/$timestamp-$state_key.log"
+  temporary_path="$incident_path.tmp"
+
+  if [ "$restart_count" -ge "$MAX_RESTARTS" ]; then
+    {
+      printf '{"timestamp":"%s","event":"restart-suppressed","reason":"restart-budget-exhausted","service":"%s","container":"%s","restartsInWindow":%s}\\n' "$timestamp" "$service_name" "$container_name" "$restart_count"
+      write_safe_inspect "$container_id"
+      docker logs --timestamps --tail 1000 "$container_id"
+    } >"$temporary_path" 2>&1
+    mv "$temporary_path" "$incident_path"
+    mv "$recent_path" "$state_path"
+    logger -t brains-health-watchdog "restart budget exhausted container=$container_name service=$service_name incident=$incident_path"
+    continue
+  fi
+
+  {
+    printf '{"timestamp":"%s","event":"container-restart","reason":"docker-health-unhealthy","service":"%s","container":"%s"}\\n' "$timestamp" "$service_name" "$container_name"
+    write_safe_inspect "$container_id"
+    docker logs --timestamps --tail 1000 "$container_id"
+  } >"$temporary_path" 2>&1
+  mv "$temporary_path" "$incident_path"
+  printf '%s\\n' "$now_epoch" >>"$recent_path"
+  mv "$recent_path" "$state_path"
+
+  logger -t brains-health-watchdog "restarting unhealthy container=$container_name service=$service_name incident=$incident_path"
+  docker restart "$container_id"
+done < <(docker ps --filter health=unhealthy --filter label=service --format '{{.ID}}')
+`;
+
+export const healthWatchdogServiceUnit: string = `[Unit]
+Description=Restart persistently unhealthy Brain containers
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=${WATCHDOG_PATH}
+`;
+
+export const healthWatchdogTimerUnit: string = `[Unit]
+Description=Monitor Brain container liveness
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=30s
+RandomizedDelaySec=5s
+Persistent=true
+Unit=brains-health-watchdog.service
+
+[Install]
+WantedBy=timers.target
+`;
+
+function installCommand(path: string, content: string, mode: string): string {
+  const encoded = Buffer.from(content, "utf8").toString("base64");
+  return `printf '%s' '${encoded}' | $SUDO base64 --decode | $SUDO tee '${path}' >/dev/null\n$SUDO chmod ${mode} '${path}'`;
+}
+
+export function buildHealthWatchdogInstallScript(): string {
+  return `#!/usr/bin/env bash
+set -euo pipefail
+SUDO=""
+if [ "$(id -u)" -ne 0 ]; then
+  SUDO=sudo
+fi
+
+${installCommand(WATCHDOG_PATH, healthWatchdogScript, "0755")}
+${installCommand(SERVICE_PATH, healthWatchdogServiceUnit, "0644")}
+${installCommand(TIMER_PATH, healthWatchdogTimerUnit, "0644")}
+
+$SUDO systemctl daemon-reload
+$SUDO systemctl enable --now brains-health-watchdog.timer
+`;
+}
+
+export async function installHealthWatchdog(): Promise<void> {
+  const serverIp = requireEnv("SERVER_IP");
+  const sshUser = requireEnv("SSH_USER").trim();
+  if (!sshUser) throw new Error("Missing SSH_USER");
+  const child = Bun.spawn(["ssh", `${sshUser}@${serverIp}`, "bash -s"], {
+    stdin: new Blob([buildHealthWatchdogInstallScript()]),
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+  const exitCode = await child.exited;
+  if (exitCode !== 0) {
+    throw new Error(
+      `Health watchdog installation failed with exit ${exitCode}`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  await installHealthWatchdog();
+}

package/templates/rover-pilot/docs/operator-playbook.md

@@ -9,6 +9,7 @@ Treat these as checked-in deploy artifacts in the pilot repo:
 - `deploy/scripts/`
 - `.github/workflows/build.yml`
 - `.github/workflows/deploy.yml`
+- `.github/workflows/directory-sync-stress.yml`
 - `.github/workflows/reconcile.yml`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
@@ -66,6 +67,20 @@ During the approved window:
 
 If any gate fails, disable all three workflows again. Restore the prior pilot desired-state and dependency revision, reconcile with the prior ops version, and redeploy the prior image tag/digest as one rollback pair. Verify the prior `/health` version and identity/content/site checks before re-enabling automation. Never restore only config or only an image.
 
+## Directory-sync stress gate
+
+Use the manual `Directory Sync Stress` workflow only against a disposable smoke user. It refuses a target unless the handle, domain, and content repository all identify smoke and the confirmation input exactly matches `stress:<handle>`.
+
+Profiles are deterministic and reversible:
+
+- `regression`: 20 probes;
+- `load`: ramps to 350 probes, updates all, renames 100, updates again, then deletes all;
+- `stress`: ramps to 700 probes and renames 200 before cleanup.
+
+The workflow loads operator credentials through Bitwarden/Varlock, but it is separate from Deploy and cannot deploy an image. It creates a rollback branch before the first content write, gates on health timeouts during the monitored workload window, preserves warmup and cleanup samples as evidence, uploads JSON/Markdown/runtime artifacts, and runs an independent idempotent cleanup job with `if: always()`. Once cleanup confirms that no probes remain, it also prunes retained `ops/directory-sync-stress-backup-*` branches; if probes remain, the branches stay available for recovery.
+
+Treat any gated health failure, restart, OOM, residual probe, or entity-baseline drift as a failed gate. Do not restart the target during measurement. Recovery is a separate operator action after evidence collection.
+
 ## Stale deploy lock recovery
 
 Kamal intentionally leaves its remote deploy lock in place when a deployment is cancelled or interrupted. Confirm that no deployment for the user is still active before releasing the lock, then use the deploy workflow's explicit recovery input: