@rizom/ops 0.2.0-alpha.305 → 0.2.0-alpha.306

package/dist/run-command.d.ts

@@ -6,6 +6,7 @@ import { type RunCommand as OpsRunCommand } from "./run-subprocess";
 import { type SshKeygen } from "./ssh-key-bootstrap";
 import type { UserRunner } from "./user-runner";
 import type { cleanupDirectorySyncStress, runDeployedDirectorySyncStress, verifyDirectorySyncStressAccess } from "./directory-sync-stress-system";
+import type { cleanupHealthWatchdogSmoke, runHealthWatchdogSmoke } from "./health-watchdog-smoke";
 export interface CommandResult {
     success: boolean;
     message?: string;
@@ -24,6 +25,8 @@ export interface CommandDependencies extends LoadPilotRegistryOptions {
     directorySyncStressRunner?: typeof runDeployedDirectorySyncStress | undefined;
     directorySyncStressAccessRunner?: typeof verifyDirectorySyncStressAccess | undefined;
     directorySyncStressCleanupRunner?: typeof cleanupDirectorySyncStress | undefined;
+    healthWatchdogSmokeRunner?: typeof runHealthWatchdogSmoke | undefined;
+    healthWatchdogSmokeCleanupRunner?: typeof cleanupHealthWatchdogSmoke | undefined;
 }
 export declare const globalFlags: FlagDefinitions;
 export declare const commands: readonly CommandDefinition<CommandDependencies, CommandResult>[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rizom/ops",
-  "version": "0.2.0-alpha.305",
+  "version": "0.2.0-alpha.306",
   "description": "Operator CLI for managing private brain fleet registry repos",
   "keywords": [
     "brains",

package/templates/rover-pilot/.github/workflows/health-watchdog-smoke.yml

@@ -0,0 +1,95 @@
+name: Health Watchdog Smoke
+
+on:
+  workflow_dispatch:
+    inputs:
+      handle:
+        description: Smoke fleet handle only
+        required: true
+        type: string
+        default: smoke
+      confirm:
+        description: Type watchdog-smoke:<handle> to confirm temporary container restarts
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-${{ inputs.handle }}
+  cancel-in-progress: false
+
+env:
+  WATCHDOG_SMOKE_RUN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Run fleet watchdog isolation smoke
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke"
+
+      - name: Upload watchdog smoke evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke
+          if-no-files-found: warn
+
+  cleanup:
+    needs: smoke
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install operator tooling
+        run: bun install
+
+      - name: Load Bitwarden environment through Varlock
+        uses: ./.github/actions/varlock-env
+        env:
+          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
+
+      - name: Remove residual watchdog smoke fixtures
+        env:
+          HANDLE_INPUT: ${{ inputs.handle }}
+          CONFIRM_INPUT: ${{ inputs.confirm }}
+        run: |
+          bunx brains-ops smoke:health-watchdog:cleanup "$GITHUB_WORKSPACE" "$HANDLE_INPUT" \
+            --confirm "$CONFIRM_INPUT" \
+            --run-id "$WATCHDOG_SMOKE_RUN_ID" \
+            --artifacts-dir "$RUNNER_TEMP/health-watchdog-smoke-cleanup"
+
+      - name: Upload watchdog smoke cleanup evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: health-watchdog-smoke-cleanup-${{ inputs.handle }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/health-watchdog-smoke-cleanup
+          if-no-files-found: warn

package/templates/rover-pilot/docs/operator-playbook.md

@@ -10,6 +10,7 @@ Treat these as checked-in deploy artifacts in the pilot repo:
 - `.github/workflows/build.yml`
 - `.github/workflows/deploy.yml`
 - `.github/workflows/directory-sync-stress.yml`
+- `.github/workflows/health-watchdog-smoke.yml`
 - `.github/workflows/reconcile.yml`
 
 `.env.schema` is the single source of truth for required and sensitive deploy vars.
@@ -88,6 +89,14 @@ The workflow loads operator credentials through Bitwarden/Varlock, but it is sep
 
 Treat any gated health failure, restart, OOM, residual probe, or entity-baseline drift as a failed gate. Do not restart the target during measurement. Recovery is a separate operator action after evidence collection.
 
+## Health watchdog smoke gate
+
+Use the manual `Health Watchdog Smoke` workflow only after the `smoke` fleet user has completed a normal Deploy run. The workflow shares the `deploy-<handle>` concurrency group, resolves the server from pilot desired state through Hetzner, loads the existing deploy SSH key through Bitwarden/Varlock, and refuses targets whose handle and domain do not identify smoke. Confirm with `watchdog-smoke:<handle>`.
+
+The smoke does not deploy or replace the installed systemd units. It verifies that the installed watchdog exactly matches the packaged canonical payload, then exercises that installed script. It also verifies the deployed rover image label, exact-label selector, active timer, and `/health/live` Docker healthcheck. While holding the timer's global lock, it creates temporary containers for eligible unhealthy, unrelated service-labelled unhealthy, false-labelled unhealthy, and eligible healthy cases. It then requires exactly three eligible restarts followed by budget suppression, diagnostics and state only for the eligible fixture, and no restart of the deployed rover or ineligible fixtures.
+
+The primary job uploads remote incident, state, and command evidence. An independent `if: always()` cleanup job removes deterministic fixture names and remote temporary files using the same workflow run ID. Treat missing evidence, unexpected eligibility, deployed-container movement, cleanup failure, or any non-smoke target rejection as a failed gate.
+
 ## Stale deploy lock recovery
 
 Kamal intentionally leaves its remote deploy lock in place when a deployment is cancelled or interrupted. Confirm that no deployment for the user is still active before releasing the lock, then use the deploy workflow's explicit recovery input: