@aiassesstech/mighty-mark 0.3.31 → 0.3.33

package/src/watchdog/fleet-backup/test/test-backup.sh

@@ -0,0 +1,395 @@
+#!/usr/bin/env bash
+# ═══════════════════════════════════════════════════════════════════
+# nole-fleet-backup — Backup Script Test Suite
+#
+# Tests fleet-backup.sh in a sandboxed environment using temp dirs.
+# No real VPS data is touched. No real GitHub push occurs.
+#
+# Usage:
+#   ./test/test-backup.sh          # run all tests
+#   ./test/test-backup.sh -v       # verbose mode
+#
+# Exit code: 0 if all tests pass, 1 if any test fails.
+# ═══════════════════════════════════════════════════════════════════
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+VERBOSE="${1:-}"
+
+PASS_COUNT=0
+FAIL_COUNT=0
+SKIP_COUNT=0
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+pass() { PASS_COUNT=$((PASS_COUNT + 1)); echo -e "  ${GREEN}PASS${NC}: $1"; }
+fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); echo -e "  ${RED}FAIL${NC}: $1"; echo -e "        $2"; }
+skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); echo -e "  ${YELLOW}SKIP${NC}: $1 — $2"; }
+
+# ─── Test Sandbox Setup ──────────────────────────────────────────
+
+SANDBOX=""
+setup_sandbox() {
+  SANDBOX=$(mktemp -d "/tmp/fleet-backup-test-XXXXXX")
+  mkdir -p "$SANDBOX"/{openclaw/extensions/grillo/agent,openclaw/extensions/jessie/agent}
+  mkdir -p "$SANDBOX"/{openclaw/extensions/noah/agent,openclaw/extensions/nole/agent}
+  mkdir -p "$SANDBOX"/{openclaw/extensions/mark/agent,openclaw/extensions/sam/agent}
+  mkdir -p "$SANDBOX"/{openclaw/agents/grillo/memory,openclaw/agents/jessie/memory}
+  mkdir -p "$SANDBOX"/{openclaw/agents/noah/memory,openclaw/agents/nole/memory}
+  mkdir -p "$SANDBOX"/{openclaw/agents/mark/memory,openclaw/agents/sam/memory}
+  mkdir -p "$SANDBOX"/{openclaw/agents/grillo/sessions,openclaw/agents/jessie/sessions}
+  mkdir -p "$SANDBOX"/{openclaw/memory,openclaw/grillo-data,openclaw/workspace}
+  mkdir -p "$SANDBOX"/{openclaw/.noah-data,openclaw/nole-data}
+  mkdir -p "$SANDBOX"/{openclaw/.fleet-data,openclaw/.sam-artifacts}
+  mkdir -p "$SANDBOX"/{clawdbot,clawd,mighty-mark-opt}
+  mkdir -p "$SANDBOX"/{jessie-data,mark-data}
+  mkdir -p "$SANDBOX"/{backup-dir/backups,backup-dir/logs}
+
+  echo "# Grillo SOUL" > "$SANDBOX/openclaw/extensions/grillo/agent/SOUL.md"
+  echo "# Grillo Memory" > "$SANDBOX/openclaw/agents/grillo/memory/context.md"
+  echo '{"model":"claude"}' > "$SANDBOX/openclaw/agents/grillo/memory/state.json"
+  echo "session data" > "$SANDBOX/openclaw/agents/grillo/sessions/001.jsonl"
+  echo '{"agents":[]}' > "$SANDBOX/openclaw/grillo-data/chain.json"
+  echo '{"apiKey":"sk-secret-key-123","model":"claude"}' > "$SANDBOX/openclaw/openclaw.json"
+  echo '{"apiKey":"sk-another-key","provider":"anthropic"}' > "$SANDBOX/clawdbot/openclaw.json"
+  echo '{"name":"grillo"}' > "$SANDBOX/openclaw/extensions/grillo/package.json"
+  echo "watchdog script" > "$SANDBOX/mighty-mark-opt/watchdog.sh"
+  echo "SECRET=value" > "$SANDBOX/mighty-mark-opt/.env"
+
+  # Create test config
+  cat > "$SANDBOX/config.sh" << 'CFGEOF'
+BACKUP_DIR="__SANDBOX__/backup-dir/backups"
+RETENTION_DAYS=35
+LOG_FILE="__SANDBOX__/backup-dir/logs/backup.log"
+MANIFEST_FILE="__SANDBOX__/backup-dir/latest-manifest.json"
+OPENCLAW_HOME="__SANDBOX__/openclaw"
+CLAWDBOT_HOME="__SANDBOX__/clawdbot"
+CLAWD_WORKSPACE="__SANDBOX__/clawd"
+MIGHTY_MARK_OPT="__SANDBOX__/mighty-mark-opt"
+NOLE_CREDS="__SANDBOX__/nole"
+FULL_BACKUP_DAY="0"
+SYSTEMD_UNITS=()
+CRON_FILES=()
+LOGROTATE_FILES=()
+EXCLUDE_PATTERNS=()
+SANITIZE_OPENCLAW_JSON=true
+PUSH_TO_GITHUB=false
+GITHUB_REPO_DIR=""
+GITHUB_SSH_HOST=""
+GITHUB_BACKUP_BRANCH="backups"
+GITHUB_MAX_FILE_SIZE=$((95 * 1024 * 1024))
+CFGEOF
+  sed -i.bak "s|__SANDBOX__|${SANDBOX}|g" "$SANDBOX/config.sh" 2>/dev/null || \
+    sed -i '' "s|__SANDBOX__|${SANDBOX}|g" "$SANDBOX/config.sh"
+  rm -f "$SANDBOX/config.sh.bak"
+
+  cp "$PROJECT_DIR/fleet-backup.sh" "$SANDBOX/fleet-backup.sh"
+  chmod +x "$SANDBOX/fleet-backup.sh"
+
+  # Override config loading: point to sandbox config
+  # fleet-backup.sh looks for config.local.sh in SCRIPT_DIR, which we set to $SANDBOX
+  cat > "$SANDBOX/config.local.sh" << LOCALEOF
+source "$SANDBOX/config.sh"
+LOCALEOF
+}
+
+teardown_sandbox() {
+  [[ -n "$SANDBOX" ]] && rm -rf "$SANDBOX"
+}
+
+run_backup() {
+  local args="${1:-}"
+  cd "$SANDBOX"
+  # fleet-backup.sh resolves SCRIPT_DIR from BASH_SOURCE[0], so we
+  # can't override it via env. Instead we put config.local.sh next to the script.
+  bash "$SANDBOX/fleet-backup.sh" $args 2>&1
+}
+
+# ─── Tests ────────────────────────────────────────────────────────
+
+echo "═══════════════════════════════════════════════════════"
+echo " nole-fleet-backup — Backup Test Suite"
+echo "═══════════════════════════════════════════════════════"
+echo ""
+
+# --- Test 1: Light backup creates archive ---
+echo "▸ Light backup"
+setup_sandbox
+output=$(FULL_BACKUP_DAY=9 run_backup "--light")
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' 2>/dev/null | head -1)
+if [[ -n "$archive" && -f "$archive" ]]; then
+  pass "Light backup creates archive"
+else
+  fail "Light backup creates archive" "No archive found in $SANDBOX/backup-dir/backups"
+fi
+
+if echo "$output" | grep -q "Starting light backup"; then
+  pass "Light backup reports correct tier"
+else
+  fail "Light backup reports correct tier" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 2: Full backup creates archive ---
+echo "▸ Full backup"
+setup_sandbox
+output=$(run_backup "--full")
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-full-*.tar.gz' 2>/dev/null | head -1)
+if [[ -n "$archive" && -f "$archive" ]]; then
+  pass "Full backup creates archive"
+else
+  fail "Full backup creates archive" "No archive found"
+fi
+
+if echo "$output" | grep -q "Starting full backup"; then
+  pass "Full backup reports correct tier"
+else
+  fail "Full backup reports correct tier" "Output: $output"
+fi
+teardown_sandbox
+
+# --- Test 3: Full backup is larger than light ---
+echo "▸ Size comparison"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+light_archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | head -1)
+light_size=$(stat -c%s "$light_archive" 2>/dev/null || stat -f%z "$light_archive" 2>/dev/null)
+
+run_backup "--full" > /dev/null 2>&1
+full_archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-full-*.tar.gz' | head -1)
+full_size=$(stat -c%s "$full_archive" 2>/dev/null || stat -f%z "$full_archive" 2>/dev/null)
+
+if [[ "$full_size" -gt "$light_size" ]]; then
+  pass "Full backup is larger than light ($full_size > $light_size bytes)"
+else
+  fail "Full backup should be larger than light" "full=$full_size light=$light_size"
+fi
+teardown_sandbox
+
+# --- Test 4: Manifest is written with correct fields ---
+echo "▸ Manifest"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+manifest="$SANDBOX/backup-dir/latest-manifest.json"
+if [[ -f "$manifest" ]]; then
+  pass "Manifest file created"
+else
+  fail "Manifest file created" "Not found at $manifest"
+  teardown_sandbox
+  # skip remaining manifest tests
+  echo ""
+  echo "═══════════════════════════════════════════════════════"
+  echo " Results: ${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped"
+  echo "═══════════════════════════════════════════════════════"
+  exit 1
+fi
+
+for field in date timestamp tier archive size_bytes checksum extensions agents memory_files retention_days; do
+  if grep -q "\"$field\"" "$manifest"; then
+    pass "Manifest contains field: $field"
+  else
+    fail "Manifest contains field: $field" "Missing from $manifest"
+  fi
+done
+
+tier_value=$(grep -o '"tier": *"[^"]*"' "$manifest" | sed 's/.*"tier": *"//;s/"//')
+if [[ "$tier_value" == "light" ]]; then
+  pass "Manifest tier is 'light'"
+else
+  fail "Manifest tier is 'light'" "Got: $tier_value"
+fi
+teardown_sandbox
+
+# --- Test 5: Full manifest has tier=full ---
+echo "▸ Full manifest tier"
+setup_sandbox
+run_backup "--full" > /dev/null 2>&1
+manifest="$SANDBOX/backup-dir/latest-manifest.json"
+tier_value=$(grep -o '"tier": *"[^"]*"' "$manifest" | sed 's/.*"tier": *"//;s/"//')
+if [[ "$tier_value" == "full" ]]; then
+  pass "Full backup manifest has tier=full"
+else
+  fail "Full backup manifest has tier=full" "Got: $tier_value"
+fi
+teardown_sandbox
+
+# --- Test 6: API keys are redacted ---
+echo "▸ Secret redaction"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | head -1)
+extract_dir=$(mktemp -d)
+tar -xzf "$archive" -C "$extract_dir" 2>/dev/null
+
+if grep -r "sk-secret-key-123" "$extract_dir" 2>/dev/null; then
+  fail "API keys are redacted from backup" "Found sk-secret-key-123 in archive"
+else
+  pass "API keys are redacted from backup"
+fi
+
+if grep -r "REDACTED_FOR_BACKUP" "$extract_dir" 2>/dev/null >/dev/null; then
+  pass "Redaction placeholder present"
+else
+  fail "Redaction placeholder present" "REDACTED_FOR_BACKUP not found"
+fi
+
+rm -rf "$extract_dir"
+teardown_sandbox
+
+# --- Test 7: .env files are excluded ---
+echo "▸ Secret exclusion"
+setup_sandbox
+run_backup "--full" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-full-*.tar.gz' | head -1)
+
+if tar -tzf "$archive" | grep -q "\.env"; then
+  fail ".env files excluded from backup" "Found .env in archive listing"
+else
+  pass ".env files excluded from backup"
+fi
+teardown_sandbox
+
+# --- Test 8: Light backup excludes heavy data ---
+echo "▸ Light backup exclusions"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | head -1)
+listing=$(tar -tzf "$archive")
+
+if echo "$listing" | grep -q "sessions/"; then
+  fail "Light backup excludes sessions" "Found sessions/ in archive"
+else
+  pass "Light backup excludes sessions"
+fi
+
+if echo "$listing" | grep -q "grillo-data/"; then
+  fail "Light backup excludes data stores" "Found grillo-data/ in archive"
+else
+  pass "Light backup excludes data stores"
+fi
+teardown_sandbox
+
+# --- Test 9: Full backup includes heavy data ---
+echo "▸ Full backup inclusions"
+setup_sandbox
+run_backup "--full" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-full-*.tar.gz' | head -1)
+listing=$(tar -tzf "$archive")
+
+if echo "$listing" | grep -q "sessions/"; then
+  pass "Full backup includes sessions"
+else
+  fail "Full backup includes sessions" "sessions/ not in archive"
+fi
+
+if echo "$listing" | grep -q "grillo-data/"; then
+  pass "Full backup includes data stores"
+else
+  fail "Full backup includes data stores" "grillo-data/ not in archive"
+fi
+teardown_sandbox
+
+# --- Test 10: Light backup includes .md memory files ---
+echo "▸ Light backup .md inclusion"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | head -1)
+listing=$(tar -tzf "$archive")
+
+if echo "$listing" | grep -q "context.md"; then
+  pass "Light backup includes memory .md files"
+else
+  fail "Light backup includes memory .md files" "context.md not in archive"
+fi
+teardown_sandbox
+
+# --- Test 11: Checksum is valid ---
+echo "▸ Checksum validation"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+archive=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | head -1)
+manifest="$SANDBOX/backup-dir/latest-manifest.json"
+
+manifest_checksum=$(grep -o '"checksum": *"[^"]*"' "$manifest" | sed 's/.*sha256://;s/"//')
+actual_checksum=$(sha256sum "$archive" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$archive" | awk '{print $1}')
+
+if [[ "$manifest_checksum" == "$actual_checksum" ]]; then
+  pass "Checksum matches archive"
+else
+  fail "Checksum matches archive" "manifest=$manifest_checksum actual=$actual_checksum"
+fi
+teardown_sandbox
+
+# --- Test 12: --help exits 0 ---
+echo "▸ Help flag"
+setup_sandbox
+if run_backup "--help" > /dev/null 2>&1; then
+  pass "--help exits 0"
+else
+  fail "--help exits 0" "Non-zero exit code"
+fi
+teardown_sandbox
+
+# --- Test 13: Multiple backups on same day get unique names ---
+echo "▸ Unique archive names"
+setup_sandbox
+run_backup "--light" > /dev/null 2>&1
+sleep 1
+run_backup "--light" > /dev/null 2>&1
+count=$(find "$SANDBOX/backup-dir/backups" -name 'fleet-light-*.tar.gz' | wc -l | tr -d ' ')
+if [[ "$count" -ge 2 ]]; then
+  pass "Multiple same-day backups get unique names ($count archives)"
+else
+  fail "Multiple same-day backups get unique names" "Only $count archive(s) found"
+fi
+teardown_sandbox
+
+# --- Test 14: Extension and agent counts are correct ---
+echo "▸ Content counts"
+setup_sandbox
+run_backup "--full" > /dev/null 2>&1
+manifest="$SANDBOX/backup-dir/latest-manifest.json"
+ext_count=$(grep -o '"extensions": *[0-9]*' "$manifest" | grep -o '[0-9]*$')
+agent_count=$(grep -o '"agents": *[0-9]*' "$manifest" | grep -o '[0-9]*$')
+
+if [[ "$ext_count" -eq 6 ]]; then
+  pass "Extension count is 6"
+else
+  fail "Extension count is 6" "Got: $ext_count"
+fi
+
+if [[ "$agent_count" -eq 6 ]]; then
+  pass "Agent count is 6"
+else
+  fail "Agent count is 6" "Got: $agent_count"
+fi
+teardown_sandbox
+
+# --- Test 15: GitHub push disabled doesn't error ---
+echo "▸ GitHub push disabled"
+setup_sandbox
+output=$(run_backup "--light" 2>&1)
+if echo "$output" | grep -q "Pushing backup to GitHub"; then
+  fail "GitHub push skipped when PUSH_TO_GITHUB=false" "Push was attempted"
+else
+  pass "GitHub push skipped when PUSH_TO_GITHUB=false"
+fi
+teardown_sandbox
+
+# ─── Summary ──────────────────────────────────────────────────────
+
+echo ""
+echo "═══════════════════════════════════════════════════════"
+TOTAL=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))
+echo -e " Results: ${GREEN}${PASS_COUNT} passed${NC}, ${RED}${FAIL_COUNT} failed${NC}, ${YELLOW}${SKIP_COUNT} skipped${NC} (${TOTAL} total)"
+echo "═══════════════════════════════════════════════════════"
+
+if [[ "$FAIL_COUNT" -gt 0 ]]; then
+  exit 1
+fi