@aiassesstech/mighty-mark 0.3.29 → 0.3.32

package/src/watchdog/fleet-backup/test/test-restore.sh

@@ -0,0 +1,302 @@
+#!/usr/bin/env bash
+# ═══════════════════════════════════════════════════════════════════
+# nole-fleet-backup — Restore Script Test Suite
+#
+# Tests fleet-restore.sh in a sandboxed environment.
+# Creates real archives, then tests list, verify, inspect, extract.
+# Interactive restore is tested non-interactively where possible.
+#
+# Usage:
+#   ./test/test-restore.sh
+# ═══════════════════════════════════════════════════════════════════
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+PASS_COUNT=0
+FAIL_COUNT=0
+SKIP_COUNT=0
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+pass() { PASS_COUNT=$((PASS_COUNT + 1)); echo -e "  ${GREEN}PASS${NC}: $1"; }
+fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); echo -e "  ${RED}FAIL${NC}: $1"; echo -e "        $2"; }
+skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); echo -e "  ${YELLOW}SKIP${NC}: $1 — $2"; }
+
+# ─── Sandbox ─────────────────────────────────────────────────────
+
+SANDBOX=""
+
+create_test_archive() {
+  local name="$1"
+  local content_dir="$SANDBOX/content"
+  mkdir -p "$content_dir/fleet-backup/extensions/grillo/agent"
+  mkdir -p "$content_dir/fleet-backup/agents/grillo/memory"
+  mkdir -p "$content_dir/fleet-backup/system"
+  echo "# Grillo SOUL" > "$content_dir/fleet-backup/extensions/grillo/agent/SOUL.md"
+  echo "# Memory" > "$content_dir/fleet-backup/agents/grillo/memory/context.md"
+  echo "config" > "$content_dir/fleet-backup/system/openclaw-gateway.service"
+
+  tar -czf "$SANDBOX/backups/${name}.tar.gz" -C "$content_dir" fleet-backup
+  rm -rf "$content_dir"
+
+  local checksum
+  checksum=$(sha256sum "$SANDBOX/backups/${name}.tar.gz" 2>/dev/null | awk '{print $1}' || \
+             shasum -a 256 "$SANDBOX/backups/${name}.tar.gz" | awk '{print $1}')
+
+  local date_part
+  date_part=$(echo "$name" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -1)
+  [[ -z "$date_part" ]] && date_part=$(date +%Y-%m-%d)
+
+  local tier="full"
+  echo "$name" | grep -q "light" && tier="light"
+
+  cat > "$SANDBOX/latest-manifest.json" << MEOF
+{
+  "date": "$date_part",
+  "timestamp": "${date_part}T09:00:00Z",
+  "tier": "$tier",
+  "archive": "${name}.tar.gz",
+  "size_bytes": $(stat -c%s "$SANDBOX/backups/${name}.tar.gz" 2>/dev/null || stat -f%z "$SANDBOX/backups/${name}.tar.gz"),
+  "size_mb": 0.1,
+  "checksum": "sha256:$checksum",
+  "extensions": 6,
+  "agents": 6,
+  "memory_files": 50,
+  "retention_days": 35,
+  "agent": "mighty-mark",
+  "workflow": "nole-fleet-backup"
+}
+MEOF
+}
+
+setup_sandbox() {
+  SANDBOX=$(mktemp -d "/tmp/fleet-restore-test-XXXXXX")
+  mkdir -p "$SANDBOX/backups"
+
+  cat > "$SANDBOX/config.sh" << CFGEOF
+BACKUP_DIR="$SANDBOX/backups"
+RETENTION_DAYS=35
+LOG_FILE="$SANDBOX/logs/backup.log"
+MANIFEST_FILE="$SANDBOX/latest-manifest.json"
+OPENCLAW_HOME="$SANDBOX/openclaw"
+CLAWDBOT_HOME="$SANDBOX/clawdbot"
+CLAWD_WORKSPACE="$SANDBOX/clawd"
+MIGHTY_MARK_OPT="$SANDBOX/mighty-mark"
+NOLE_CREDS="$SANDBOX/nole"
+FULL_BACKUP_DAY="0"
+SYSTEMD_UNITS=()
+CRON_FILES=()
+LOGROTATE_FILES=()
+EXCLUDE_PATTERNS=()
+SANITIZE_OPENCLAW_JSON=true
+PUSH_TO_GITHUB=false
+GITHUB_REPO_DIR=""
+GITHUB_SSH_HOST=""
+GITHUB_BACKUP_BRANCH="backups"
+GITHUB_MAX_FILE_SIZE=\$((95 * 1024 * 1024))
+CFGEOF
+
+  cp "$PROJECT_DIR/fleet-restore.sh" "$SANDBOX/fleet-restore.sh"
+  chmod +x "$SANDBOX/fleet-restore.sh"
+
+  cat > "$SANDBOX/config.local.sh" << LOCALEOF
+source "$SANDBOX/config.sh"
+LOCALEOF
+}
+
+teardown_sandbox() {
+  [[ -n "$SANDBOX" ]] && rm -rf "$SANDBOX"
+  rm -rf /tmp/fleet-restore-fleet-* 2>/dev/null || true
+}
+
+run_restore() {
+  cd "$SANDBOX"
+  SCRIPT_DIR="$SANDBOX" bash "$SANDBOX/fleet-restore.sh" "$@" 2>&1
+}
+
+# ─── Tests ────────────────────────────────────────────────────────
+
+echo "═══════════════════════════════════════════════════════"
+echo " nole-fleet-backup — Restore Test Suite"
+echo "═══════════════════════════════════════════════════════"
+echo ""
+
+# --- Test 1: --list with no backups ---
+echo "▸ List (empty)"
+setup_sandbox
+output=$(run_restore --list)
+if echo "$output" | grep -q "0 backup(s) found"; then
+  pass "--list shows 0 backups when empty"
+else
+  fail "--list shows 0 backups when empty" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 2: --list with backups ---
+echo "▸ List (with archives)"
+setup_sandbox
+create_test_archive "fleet-full-2026-03-01-090000"
+create_test_archive "fleet-light-2026-03-02-090000"
+output=$(run_restore --list)
+if echo "$output" | grep -q "2 backup(s) found"; then
+  pass "--list shows 2 backups"
+else
+  fail "--list shows 2 backups" "Output: $output"
+fi
+
+if echo "$output" | grep -q "FULL"; then
+  pass "--list identifies full tier"
+else
+  fail "--list identifies full tier" "Output: $output"
+fi
+
+if echo "$output" | grep -q "light"; then
+  pass "--list identifies light tier"
+else
+  fail "--list identifies light tier" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 3: --inspect shows contents ---
+echo "▸ Inspect"
+setup_sandbox
+create_test_archive "fleet-full-2026-03-01-090000"
+output=$(run_restore --inspect "fleet-full-2026-03-01-090000")
+if echo "$output" | grep -q "SOUL.md"; then
+  pass "--inspect shows archive contents"
+else
+  fail "--inspect shows archive contents" "Output: $output"
+fi
+
+if echo "$output" | grep -q "Total entries"; then
+  pass "--inspect shows entry count"
+else
+  fail "--inspect shows entry count" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 4: --verify passes with matching checksum ---
+echo "▸ Verify (valid)"
+setup_sandbox
+create_test_archive "fleet-full-$(date +%Y-%m-%d)-090000"
+output=$(run_restore --verify "fleet-full-$(date +%Y-%m-%d)-090000")
+if echo "$output" | grep -q "VERIFIED"; then
+  pass "--verify passes with correct checksum"
+else
+  fail "--verify passes with correct checksum" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 5: --verify detects mismatch ---
+echo "▸ Verify (tampered)"
+setup_sandbox
+create_test_archive "fleet-full-$(date +%Y-%m-%d)-090000"
+echo "tampered" >> "$SANDBOX/backups/fleet-full-$(date +%Y-%m-%d)-090000.tar.gz"
+output=$(run_restore --verify "fleet-full-$(date +%Y-%m-%d)-090000" 2>&1 || true)
+if echo "$output" | grep -q "MISMATCH"; then
+  pass "--verify detects tampered archive"
+else
+  fail "--verify detects tampered archive" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 6: --extract creates temp dir ---
+echo "▸ Extract"
+setup_sandbox
+create_test_archive "fleet-full-2026-03-01-090000"
+output=$(run_restore --extract "fleet-full-2026-03-01-090000")
+extract_dir="/tmp/fleet-restore-fleet-full-2026-03-01-090000"
+if [[ -d "$extract_dir" ]]; then
+  pass "--extract creates temp directory"
+  if [[ -f "$extract_dir/fleet-backup/extensions/grillo/agent/SOUL.md" ]]; then
+    pass "--extract preserves file structure"
+  else
+    fail "--extract preserves file structure" "SOUL.md not found in extracted archive"
+  fi
+  rm -rf "$extract_dir"
+else
+  fail "--extract creates temp directory" "Not found: $extract_dir"
+fi
+teardown_sandbox
+
+# --- Test 7: --help exits 0 ---
+echo "▸ Help"
+setup_sandbox
+if run_restore --help > /dev/null 2>&1; then
+  pass "--help exits 0"
+else
+  fail "--help exits 0" "Non-zero exit"
+fi
+teardown_sandbox
+
+# --- Test 8: Unknown command exits 1 ---
+echo "▸ Unknown command"
+setup_sandbox
+if run_restore --bogus > /dev/null 2>&1; then
+  fail "Unknown command exits 1" "Got exit 0"
+else
+  pass "Unknown command exits 1"
+fi
+teardown_sandbox
+
+# --- Test 9: --inspect without archive name exits 1 ---
+echo "▸ Missing archive name"
+setup_sandbox
+if run_restore --inspect 2>/dev/null; then
+  fail "--inspect without name exits 1" "Got exit 0"
+else
+  pass "--inspect without name exits 1"
+fi
+teardown_sandbox
+
+# --- Test 10: Non-existent archive exits 1 ---
+echo "▸ Missing archive"
+setup_sandbox
+if run_restore --inspect "nonexistent-2026-01-01-090000" > /dev/null 2>&1; then
+  fail "Non-existent archive exits 1" "Got exit 0"
+else
+  pass "Non-existent archive exits 1"
+fi
+teardown_sandbox
+
+# --- Test 11: Partial name matching ---
+echo "▸ Partial name match"
+setup_sandbox
+create_test_archive "fleet-full-2026-03-01-090000"
+output=$(run_restore --inspect "2026-03-01" 2>&1)
+if echo "$output" | grep -q "SOUL.md\|Total entries"; then
+  pass "Partial name match finds archive"
+else
+  fail "Partial name match finds archive" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 12: Default (no args) lists backups ---
+echo "▸ Default action"
+setup_sandbox
+create_test_archive "fleet-full-2026-03-01-090000"
+output=$(run_restore)
+if echo "$output" | grep -q "1 backup(s) found"; then
+  pass "Default action lists backups"
+else
+  fail "Default action lists backups" "Output: $output"
+fi
+teardown_sandbox
+
+# ─── Summary ──────────────────────────────────────────────────────
+
+echo ""
+echo "═══════════════════════════════════════════════════════"
+TOTAL=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))
+echo -e " Results: ${GREEN}${PASS_COUNT} passed${NC}, ${RED}${FAIL_COUNT} failed${NC}, ${YELLOW}${SKIP_COUNT} skipped${NC} (${TOTAL} total)"
+echo "═══════════════════════════════════════════════════════"
+
+if [[ "$FAIL_COUNT" -gt 0 ]]; then
+  exit 1
+fi

package/src/watchdog/install.sh

@@ -147,8 +147,14 @@ CRON_TZ=America/Chicago
 */5 * * * * root ${MARK_HOME}/watchdog.sh >> ${MARK_HOME}/logs/watchdog-cron.log 2>&1
 
 # Morning Check: daily at 6:00 AM Chicago time (auto-adjusts for CDT/CST)
+# Runs fleet backup first, then health check (including backup freshness validation)
 0 6 * * * root ${MARK_HOME}/morning-check.sh >> ${MARK_HOME}/logs/morning-cron.log 2>&1
 CRONEOF
+  # Remove standalone fleet-backup cron if it exists (now integrated into morning check)
+  if [ -f /etc/cron.d/fleet-backup ]; then
+    rm -f /etc/cron.d/fleet-backup
+    echo "  Removed: /etc/cron.d/fleet-backup (now integrated into morning check)"
+  fi
   chmod 644 /etc/cron.d/mighty-mark
   echo "  Installed: /etc/cron.d/mighty-mark"
   echo "    Watchdog:      */5 * * * * (every 5 min)"
@@ -182,8 +188,107 @@ LREOF
   echo "  Installed: /etc/logrotate.d/mighty-mark (14-day rotation)"
 fi
 
-# ── Step 6: Initialize state ──
-echo "[6/7] Initializing state files..."
+# ── Step 6: Install fleet backup scripts ──
+echo "[6/8] Installing fleet backup..."
+
+FLEET_BACKUP_DIR="/opt/fleet-backup"
+FLEET_REPO_DIR="/opt/nole-fleet-backup"
+
+# Source priority:
+#   1. Bundled in npm package (src/watchdog/fleet-backup/) — preferred
+#   2. Standalone git clone (/opt/nole-fleet-backup) — fallback
+FLEET_BUNDLED_DIR="${SOURCE_DIR}/fleet-backup"
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would set up fleet backup at ${FLEET_BACKUP_DIR}"
+elif [ -d "$FLEET_BUNDLED_DIR" ] && [ -f "$FLEET_BUNDLED_DIR/fleet-backup.sh" ]; then
+  echo "  Source: bundled in mighty-mark npm package"
+  mkdir -p "${FLEET_BACKUP_DIR}/backups"
+  mkdir -p "${FLEET_BACKUP_DIR}/logs"
+  mkdir -p "${FLEET_BACKUP_DIR}/test"
+  cp "$FLEET_BUNDLED_DIR/fleet-backup.sh"  "${FLEET_BACKUP_DIR}/"
+  cp "$FLEET_BUNDLED_DIR/fleet-restore.sh" "${FLEET_BACKUP_DIR}/"
+  cp "$FLEET_BUNDLED_DIR/config.sh"        "${FLEET_BACKUP_DIR}/"
+  # Preserve config.local.sh if it already exists on disk (user customizations)
+  if [ ! -f "${FLEET_BACKUP_DIR}/config.local.sh" ] && [ -f "$FLEET_BUNDLED_DIR/config.local.sh" ]; then
+    cp "$FLEET_BUNDLED_DIR/config.local.sh" "${FLEET_BACKUP_DIR}/"
+  fi
+  # Install test scripts for daily verification
+  if [ -d "$FLEET_BUNDLED_DIR/test" ]; then
+    cp "$FLEET_BUNDLED_DIR/test/test-backup.sh"  "${FLEET_BACKUP_DIR}/test/" 2>/dev/null || true
+    cp "$FLEET_BUNDLED_DIR/test/test-restore.sh" "${FLEET_BACKUP_DIR}/test/" 2>/dev/null || true
+    chmod +x "${FLEET_BACKUP_DIR}/test/"*.sh 2>/dev/null || true
+    echo "  Tests:     ${FLEET_BACKUP_DIR}/test/"
+  fi
+  chmod +x "${FLEET_BACKUP_DIR}/fleet-backup.sh"
+  chmod +x "${FLEET_BACKUP_DIR}/fleet-restore.sh"
+  echo "  Installed: fleet-backup.sh, fleet-restore.sh, config.sh"
+  echo "  Backups:   ${FLEET_BACKUP_DIR}/backups/"
+  echo "  Logs:      ${FLEET_BACKUP_DIR}/logs/"
+
+  # Install fleet-backup logrotate
+  cat > /etc/logrotate.d/fleet-backup <<FBLREOF
+${FLEET_BACKUP_DIR}/logs/*.log {
+    weekly
+    rotate 8
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 644 root root
+}
+FBLREOF
+  chmod 644 /etc/logrotate.d/fleet-backup
+  echo "  Logrotate: /etc/logrotate.d/fleet-backup"
+elif [ -d "$FLEET_REPO_DIR" ] && [ -f "$FLEET_REPO_DIR/fleet-backup.sh" ]; then
+  echo "  Source: standalone git clone at ${FLEET_REPO_DIR}"
+  mkdir -p "${FLEET_BACKUP_DIR}/backups"
+  mkdir -p "${FLEET_BACKUP_DIR}/logs"
+  mkdir -p "${FLEET_BACKUP_DIR}/test"
+  cp "$FLEET_REPO_DIR/fleet-backup.sh"  "${FLEET_BACKUP_DIR}/"
+  cp "$FLEET_REPO_DIR/fleet-restore.sh" "${FLEET_BACKUP_DIR}/"
+  cp "$FLEET_REPO_DIR/config.sh"        "${FLEET_BACKUP_DIR}/"
+  if [ -f "$FLEET_REPO_DIR/config.local.sh" ]; then
+    cp "$FLEET_REPO_DIR/config.local.sh" "${FLEET_BACKUP_DIR}/"
+  fi
+  if [ -d "$FLEET_REPO_DIR/test" ]; then
+    cp "$FLEET_REPO_DIR/test/"*.sh "${FLEET_BACKUP_DIR}/test/" 2>/dev/null || true
+    chmod +x "${FLEET_BACKUP_DIR}/test/"*.sh 2>/dev/null || true
+    echo "  Tests:     ${FLEET_BACKUP_DIR}/test/"
+  fi
+  chmod +x "${FLEET_BACKUP_DIR}/fleet-backup.sh"
+  chmod +x "${FLEET_BACKUP_DIR}/fleet-restore.sh"
+  echo "  Installed: fleet-backup.sh, fleet-restore.sh, config.sh"
+  echo "  Backups:   ${FLEET_BACKUP_DIR}/backups/"
+  echo "  Logs:      ${FLEET_BACKUP_DIR}/logs/"
+
+  cat > /etc/logrotate.d/fleet-backup <<FBLREOF
+${FLEET_BACKUP_DIR}/logs/*.log {
+    weekly
+    rotate 8
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 644 root root
+}
+FBLREOF
+  chmod 644 /etc/logrotate.d/fleet-backup
+  echo "  Logrotate: /etc/logrotate.d/fleet-backup"
+else
+  echo "  Skipped — fleet backup scripts not found"
+  echo "  Expected in npm package at: ${FLEET_BUNDLED_DIR}/"
+  echo "  Or standalone repo at: ${FLEET_REPO_DIR}/"
+fi
+
+# Remove standalone fleet-backup cron — now orchestrated by morning-check.sh via TypeScript engine
+if [ -f /etc/cron.d/fleet-backup ]; then
+  rm -f /etc/cron.d/fleet-backup
+  echo "  Removed: /etc/cron.d/fleet-backup (now orchestrated by Mighty Mark)"
+fi
+
+# ── Step 7: Initialize state ──
+echo "[7/8] Initializing state files..."
 
 if [ "$DRY_RUN" = true ]; then
   echo "  [DRY RUN] Would initialize state files"
@@ -196,8 +301,8 @@ else
   fi
 fi
 
-# ── Step 7: Test Telegram connectivity ──
-echo "[7/7] Testing Telegram connectivity..."
+# ── Step 8: Test Telegram connectivity ──
+echo "[8/8] Testing Telegram connectivity..."
 
 if [ "$DRY_RUN" = true ]; then
   echo "  [DRY RUN] Would test Telegram API"
@@ -240,5 +345,8 @@ if [ -z "${TELEGRAM_TOKEN}" ]; then
   echo ""
 fi
 
+echo "  Fleet backup: ${FLEET_BACKUP_DIR}/fleet-backup.sh"
+echo "  Restore:      ${FLEET_BACKUP_DIR}/fleet-restore.sh --help"
+echo ""
 echo "  Manual test: bash ${MARK_HOME}/watchdog.sh"
 echo "  View logs:   tail -f ${MARK_HOME}/logs/watchdog.log"

package/src/watchdog/morning-check.sh

@@ -21,6 +21,12 @@ mkdir -p "${MARK_LOG_DIR}" "${MARK_STATE_DIR}"
 
 log_morning "Morning check starting"
 
+# Fleet backup is now orchestrated by the TypeScript health engine
+# (backup-orchestrator.ts). It runs backup + tests + manifest validation
+# as part of the data integrity checks. No need to trigger it separately.
+# If the TypeScript engine fails and we fall back to bash, the backup
+# won't run — but it will catch up on the next successful morning check.
+
 # ── Try Node.js health check first ──
 if command -v npx &> /dev/null; then
   log_morning "Running TypeScript health engine via npx"