@aiassesstech/mighty-mark 0.3.29 → 0.3.32

package/src/watchdog/fleet-backup/config.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# nole-fleet-backup configuration
+# Copy to config.local.sh and customize for your VPS. config.local.sh is gitignored.
+
+# ─── Backup tiers ────────────────────────────────────────────────
+# Full:  Everything (extensions, memory, data stores, vector indexes, sessions)
+# Light: Agent .md files, sanitized configs, systemd/cron — no heavy binary data
+#
+# Schedule: full on Sundays, light Mon-Sat
+# Override: fleet-backup.sh --full  or  fleet-backup.sh --light
+FULL_BACKUP_DAY="0"  # 0=Sunday, 1=Monday, ... 6=Saturday
+
+# Where to store backup archives
+BACKUP_DIR="/opt/fleet-backup/backups"
+
+# How many days to keep backups (covers 5+ weekly fulls)
+RETENTION_DAYS=35
+
+# Log file
+LOG_FILE="/opt/fleet-backup/logs/backup.log"
+
+# Manifest location (read by Mighty Mark health check)
+MANIFEST_FILE="/opt/fleet-backup/latest-manifest.json"
+
+# ─── OpenClaw paths ───────────────────────────────────────────────
+OPENCLAW_HOME="/root/.openclaw"
+CLAWDBOT_HOME="/root/.clawdbot"
+CLAWD_WORKSPACE="/root/clawd"
+MIGHTY_MARK_OPT="/opt/mighty-mark"
+NOLE_CREDS="/root/.nole"
+
+# ─── Systemd / cron configs to back up ────────────────────────────
+SYSTEMD_UNITS=(
+  "/etc/systemd/system/openclaw-gateway.service"
+)
+CRON_FILES=(
+  "/etc/cron.d/mighty-mark"
+  "/etc/cron.d/fleet-backup"
+)
+LOGROTATE_FILES=(
+  "/etc/logrotate.d/mighty-mark"
+  "/etc/logrotate.d/fleet-backup"
+)
+
+# ─── Exclude patterns (secrets, large caches, node_modules) ──────
+EXCLUDE_PATTERNS=(
+  "*/node_modules/*"
+  "*/.cache/*"
+  "*/.node-llama-cpp/*"
+  "*.env"
+  "*/credentials"
+  "*/openclaw.json"
+)
+
+# ─── Separate secret-safe config backup ───────────────────────────
+# openclaw.json is backed up with API keys stripped
+SANITIZE_OPENCLAW_JSON=true
+
+# ─── GitHub push (off-site copy) ──────────────────────────────────
+# Pushes backups to GitHub after each run
+PUSH_TO_GITHUB=true
+
+# The repo clone used for pushing (cloned with deploy key)
+GITHUB_REPO_DIR="/opt/nole-fleet-backup"
+
+# SSH host alias from ~/.ssh/config (for the deploy key)
+GITHUB_SSH_HOST="github-nole-backup"
+
+# Branch to store backup archives (keeps main branch clean)
+GITHUB_BACKUP_BRANCH="backups"
+
+# Max file size for a single GitHub push (bytes). Files larger than
+# this are split into chunks. GitHub's limit is 100 MB.
+GITHUB_MAX_FILE_SIZE=$((95 * 1024 * 1024))  # 95 MB (with margin)

package/src/watchdog/fleet-backup/fleet-backup.sh

@@ -0,0 +1,363 @@
+#!/usr/bin/env bash
+# ═══════════════════════════════════════════════════════════════════
+# nole-fleet-backup — Tiered backup of entire OpenClaw fleet
+# Agent: Mighty Mark (sentinel duty — fleet safety checkpoint)
+#
+# Tiers:
+#   full  — everything (extensions, memory, data stores, vectors,
+#           sessions, fleet-bus, watchdog, configs). Weekly (Sunday).
+#   light — agent .md files, sanitized configs, systemd/cron only.
+#           Small enough to push to GitHub daily. Mon–Sat.
+#
+# Usage:
+#   fleet-backup.sh           # auto-detect tier from day of week
+#   fleet-backup.sh --full    # force full backup
+#   fleet-backup.sh --light   # force light backup
+#
+# Excludes: node_modules, secrets (.env, credentials, API keys)
+# Retention: 35 days (configurable)
+# ═══════════════════════════════════════════════════════════════════
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ -f "$SCRIPT_DIR/config.local.sh" ]]; then
+  source "$SCRIPT_DIR/config.local.sh"
+else
+  source "$SCRIPT_DIR/config.sh"
+fi
+
+DATE=$(date +%Y-%m-%d)
+TIME=$(date +%H%M%S)
+TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+STAGING_DIR=$(mktemp -d "/tmp/fleet-backup-staging-XXXXXX")
+
+# ─── Determine backup tier ────────────────────────────────────────
+
+DOW=$(date +%w)  # 0=Sunday
+TIER="light"
+if [[ "$DOW" == "${FULL_BACKUP_DAY:-0}" ]]; then
+  TIER="full"
+fi
+
+case "${1:-}" in
+  --full)  TIER="full" ;;
+  --light) TIER="light" ;;
+  --help|-h)
+    echo "Usage: $0 [--full|--light]"
+    echo "  --full   Force a full backup (all data)"
+    echo "  --light  Force a light backup (agent .md + configs only)"
+    echo "  (none)   Auto-detect: full on day $FULL_BACKUP_DAY, light otherwise"
+    exit 0
+    ;;
+esac
+
+ARCHIVE_NAME="fleet-${TIER}-${DATE}-${TIME}.tar.gz"
+BACKUP_FAILED=false
+
+log() {
+  local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
+  echo "$msg"
+  if [[ -n "${LOG_FILE:-}" ]]; then
+    mkdir -p "$(dirname "$LOG_FILE")"
+    echo "$msg" >> "$LOG_FILE"
+  fi
+}
+
+# Load Mighty Mark's Telegram credentials if available
+MARK_ENV="/opt/mighty-mark/.env"
+if [[ -f "$MARK_ENV" ]]; then
+  set -a
+  source "$MARK_ENV"
+  set +a
+fi
+
+notify_telegram() {
+  local token="${MIGHTY_MARK_TELEGRAM_TOKEN:-}"
+  local chat="${MIGHTY_MARK_TELEGRAM_CHAT_ID:-}"
+  if [[ -n "$token" && -n "$chat" ]]; then
+    curl -s -X POST \
+      "https://api.telegram.org/bot${token}/sendMessage" \
+      -d "chat_id=${chat}" \
+      -d "text=$1" \
+      -d "parse_mode=Markdown" > /dev/null 2>&1 || true
+  fi
+}
+
+cleanup() {
+  rm -rf "$STAGING_DIR"
+}
+
+on_error() {
+  local exit_code=$?
+  log "ERROR: Backup failed with exit code $exit_code"
+  BACKUP_FAILED=true
+  notify_telegram "🔴 *MIGHTY MARK — Backup FAILED*%0A%0ATier: ${TIER}%0AExit code: ${exit_code}%0A%0ACheck logs: /opt/fleet-backup/logs/backup.log%0AServer: $(hostname)%0ATime: $(date)"
+  cleanup
+  exit $exit_code
+}
+
+trap on_error ERR
+trap cleanup EXIT
+
+# ─── Collect backup contents ──────────────────────────────────────
+
+log "Starting $TIER backup for $DATE"
+mkdir -p "$STAGING_DIR/fleet-backup"
+DEST="$STAGING_DIR/fleet-backup"
+
+# --- Items included in BOTH tiers ---
+
+# Agent identity/prompt .md files from extensions
+if [[ -d "$OPENCLAW_HOME/extensions" ]]; then
+  log "Backing up agent extension docs..."
+  mkdir -p "$DEST/extensions"
+  for agent_dir in "$OPENCLAW_HOME/extensions"/*/; do
+    agent_name=$(basename "$agent_dir")
+    mkdir -p "$DEST/extensions/$agent_name"
+    if [[ -d "$agent_dir/agent" ]]; then
+      cp -r "$agent_dir/agent" "$DEST/extensions/$agent_name/"
+    fi
+    [[ -f "$agent_dir/package.json" ]] && cp "$agent_dir/package.json" "$DEST/extensions/$agent_name/"
+    if [[ "$TIER" == "full" ]]; then
+      [[ -d "$agent_dir/dist" ]] && cp -r "$agent_dir/dist" "$DEST/extensions/$agent_name/"
+    fi
+  done
+fi
+
+# Agent workspace .md files (memory markdown, prompts)
+if [[ -d "$OPENCLAW_HOME/agents" ]]; then
+  log "Backing up agent workspace docs..."
+  mkdir -p "$DEST/agents"
+  for agent_dir in "$OPENCLAW_HOME/agents"/*/; do
+    agent_name=$(basename "$agent_dir")
+    mkdir -p "$DEST/agents/$agent_name"
+
+    if [[ -d "$agent_dir/memory" ]]; then
+      mkdir -p "$DEST/agents/$agent_name/memory"
+      find "$agent_dir/memory" -name '*.md' -exec cp {} "$DEST/agents/$agent_name/memory/" \;
+    fi
+    if [[ -d "$agent_dir/agent" ]]; then
+      cp -r "$agent_dir/agent" "$DEST/agents/$agent_name/"
+    fi
+
+    if [[ "$TIER" == "full" ]]; then
+      [[ -d "$agent_dir/sessions" ]] && cp -r "$agent_dir/sessions" "$DEST/agents/$agent_name/"
+      # Include non-.md memory files (JSON, JSONL) only in full
+      if [[ -d "$agent_dir/memory" ]]; then
+        find "$agent_dir/memory" \( -name '*.json' -o -name '*.jsonl' \) -exec cp {} "$DEST/agents/$agent_name/memory/" \;
+      fi
+    fi
+  done
+fi
+
+# Sanitized openclaw.json (strip API keys)
+if [[ "$SANITIZE_OPENCLAW_JSON" == "true" ]]; then
+  for config_file in "$OPENCLAW_HOME/openclaw.json" "$CLAWDBOT_HOME/openclaw.json"; do
+    if [[ -f "$config_file" ]]; then
+      config_name=$(echo "$config_file" | sed 's|/|_|g')
+      log "Backing up sanitized config: $config_file"
+      sed -E 's/("apiKey"\s*:\s*)"[^"]+"/\1"REDACTED_FOR_BACKUP"/g' \
+        "$config_file" > "$DEST/$config_name"
+    fi
+  done
+fi
+
+# Systemd, cron, logrotate configs
+mkdir -p "$DEST/system"
+for unit in "${SYSTEMD_UNITS[@]}"; do
+  [[ -f "$unit" ]] && cp "$unit" "$DEST/system/"
+done
+for cron in "${CRON_FILES[@]}"; do
+  [[ -f "$cron" ]] && cp "$cron" "$DEST/system/"
+done
+for lr in "${LOGROTATE_FILES[@]}"; do
+  [[ -f "$lr" ]] && cp "$lr" "$DEST/system/"
+done
+
+# --- Items included in FULL tier only ---
+
+if [[ "$TIER" == "full" ]]; then
+
+  # Vector indexes (sqlite-vec)
+  if [[ -d "$OPENCLAW_HOME/memory" ]]; then
+    log "Backing up vector indexes..."
+    mkdir -p "$DEST/memory"
+    find "$OPENCLAW_HOME/memory" -name '*.sqlite' -exec cp {} "$DEST/memory/" \;
+  fi
+
+  # Agent data stores (JSON assessment chains, temporal stores)
+  for data_dir in \
+    "$OPENCLAW_HOME/grillo-data" \
+    "$OPENCLAW_HOME/.noah-data" \
+    "$OPENCLAW_HOME/nole-data" \
+    "$OPENCLAW_HOME/.mark-data" \
+    "$OPENCLAW_HOME/.fleet-data" \
+    "$OPENCLAW_HOME/.sam-artifacts" \
+    "/root/.mark-data" \
+    "/root/.jessie-data"; do
+    if [[ -d "$data_dir" ]]; then
+      dir_name=$(basename "$data_dir")
+      log "Backing up data store: $dir_name"
+      cp -r "$data_dir" "$DEST/$dir_name"
+    fi
+  done
+
+  # Fleet-bus workspace (audit logs, agent cards, events)
+  if [[ -d "$OPENCLAW_HOME/workspace" ]]; then
+    log "Backing up workspace (fleet-bus, skills)..."
+    cp -r "$OPENCLAW_HOME/workspace" "$DEST/workspace"
+  fi
+
+  # Clawd workspace (Jessie)
+  if [[ -d "$CLAWD_WORKSPACE" ]]; then
+    log "Backing up clawd workspace..."
+    cp -r "$CLAWD_WORKSPACE" "$DEST/clawd"
+  fi
+
+  # Mighty Mark watchdog (scripts, state — excluding .env)
+  if [[ -d "$MIGHTY_MARK_OPT" ]]; then
+    log "Backing up Mighty Mark watchdog..."
+    mkdir -p "$DEST/mighty-mark-watchdog"
+    for item in "$MIGHTY_MARK_OPT"/*; do
+      item_name=$(basename "$item")
+      [[ "$item_name" == ".env" ]] && continue
+      [[ "$item_name" == "backups" ]] && continue
+      [[ "$item_name" == "logs" ]] && continue
+      cp -r "$item" "$DEST/mighty-mark-watchdog/"
+    done
+  fi
+
+fi
+
+# ─── Create archive ───────────────────────────────────────────────
+
+log "Creating archive..."
+mkdir -p "$BACKUP_DIR"
+tar -czf "$BACKUP_DIR/$ARCHIVE_NAME" -C "$STAGING_DIR" fleet-backup
+
+ARCHIVE_SIZE=$(stat -c%s "$BACKUP_DIR/$ARCHIVE_NAME" 2>/dev/null || stat -f%z "$BACKUP_DIR/$ARCHIVE_NAME" 2>/dev/null || echo 0)
+ARCHIVE_SIZE_MB=$(awk "BEGIN {printf \"%.1f\", $ARCHIVE_SIZE / 1048576}")
+
+CHECKSUM=$(sha256sum "$BACKUP_DIR/$ARCHIVE_NAME" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$BACKUP_DIR/$ARCHIVE_NAME" | awk '{print $1}')
+
+EXTENSION_COUNT=$(find "$DEST/extensions" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l | tr -d ' ')
+AGENT_COUNT=$(find "$DEST/agents" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l | tr -d ' ')
+MEMORY_FILE_COUNT=$(find "$DEST" -name '*.md' -o -name '*.json' -o -name '*.jsonl' -o -name '*.sqlite' 2>/dev/null | wc -l | tr -d ' ')
+
+# ─── Write manifest ──────────────────────────────────────────────
+
+cat > "$MANIFEST_FILE" << MANIFEST_EOF
+{
+  "date": "$DATE",
+  "timestamp": "$TIMESTAMP",
+  "tier": "$TIER",
+  "archive": "$ARCHIVE_NAME",
+  "size_bytes": $ARCHIVE_SIZE,
+  "size_mb": $ARCHIVE_SIZE_MB,
+  "checksum": "sha256:$CHECKSUM",
+  "extensions": $EXTENSION_COUNT,
+  "agents": $AGENT_COUNT,
+  "memory_files": $MEMORY_FILE_COUNT,
+  "retention_days": $RETENTION_DAYS,
+  "agent": "mighty-mark",
+  "workflow": "nole-fleet-backup"
+}
+MANIFEST_EOF
+
+log "Archive: $ARCHIVE_NAME ($ARCHIVE_SIZE_MB MB) [$TIER]"
+log "  Extensions: $EXTENSION_COUNT"
+log "  Agents: $AGENT_COUNT"
+log "  Memory files: $MEMORY_FILE_COUNT"
+log "  Checksum: sha256:$CHECKSUM"
+
+# ─── Prune old backups ────────────────────────────────────────────
+
+log "Pruning backups older than $RETENTION_DAYS days..."
+find "$BACKUP_DIR" -name 'fleet-*.tar.gz' -mtime +"$RETENTION_DAYS" -print -delete 2>/dev/null | while read -r old; do
+  log "  Pruned: $(basename "$old")"
+done
+
+REMAINING=$(find "$BACKUP_DIR" -name 'fleet-*.tar.gz' 2>/dev/null | wc -l | tr -d ' ')
+log "Backup complete. $REMAINING archives retained."
+log "Manifest: $MANIFEST_FILE"
+
+# ─── Push to GitHub (off-site copy) ──────────────────────────────
+
+if [[ "${PUSH_TO_GITHUB:-false}" == "true" ]] && [[ -d "${GITHUB_REPO_DIR:-}" ]]; then
+  log "Pushing backup to GitHub (branch: $GITHUB_BACKUP_BRANCH)..."
+
+  PUSH_WORK=$(mktemp -d "/tmp/fleet-backup-push-XXXXXX")
+
+  push_cleanup() {
+    rm -rf "$PUSH_WORK"
+    git -C "$GITHUB_REPO_DIR" worktree prune 2>/dev/null || true
+  }
+  trap push_cleanup EXIT
+
+  (
+    cd "$GITHUB_REPO_DIR"
+
+    REMOTE_URL=$(git remote get-url origin 2>/dev/null || echo "")
+    if [[ -z "$REMOTE_URL" ]]; then
+      log "  ERROR: No git remote found in $GITHUB_REPO_DIR"
+      exit 1
+    fi
+
+    git fetch origin "$GITHUB_BACKUP_BRANCH" 2>/dev/null || true
+
+    if git rev-parse --verify "origin/$GITHUB_BACKUP_BRANCH" >/dev/null 2>&1; then
+      git worktree add "$PUSH_WORK" "origin/$GITHUB_BACKUP_BRANCH" 2>/dev/null || true
+      cd "$PUSH_WORK"
+      git checkout -B "$GITHUB_BACKUP_BRANCH" 2>/dev/null || true
+    else
+      cd "$PUSH_WORK"
+      git init
+      git remote add origin "$REMOTE_URL"
+      git checkout --orphan "$GITHUB_BACKUP_BRANCH"
+    fi
+
+    # Clean previous backup files from the working tree
+    rm -f fleet-*.tar.gz fleet-*.tar.gz.part-* latest-manifest.json
+
+    # Split large archives into chunks under GitHub's file size limit
+    if [[ "$ARCHIVE_SIZE" -gt "${GITHUB_MAX_FILE_SIZE:-99614720}" ]]; then
+      log "  Splitting archive into chunks (${ARCHIVE_SIZE_MB} MB > limit)..."
+      split -b "${GITHUB_MAX_FILE_SIZE:-99614720}" -d -a 2 \
+        "$BACKUP_DIR/$ARCHIVE_NAME" \
+        "${ARCHIVE_NAME}.part-"
+      CHUNK_COUNT=$(ls -1 "${ARCHIVE_NAME}".part-* 2>/dev/null | wc -l | tr -d ' ')
+      log "  Split into $CHUNK_COUNT chunks"
+    else
+      cp "$BACKUP_DIR/$ARCHIVE_NAME" .
+    fi
+
+    cp "$MANIFEST_FILE" ./latest-manifest.json
+
+    git add -A
+    git config user.name "Mighty Mark (Automated)"
+    git config user.email "mighty-mark@aiassesstech.com"
+
+    if git diff --cached --quiet 2>/dev/null; then
+      log "  No changes to push."
+    else
+      git commit -m "fleet $TIER backup: $DATE
+
+Automated $TIER backup — $ARCHIVE_SIZE_MB MB
+Extensions: $EXTENSION_COUNT | Agents: $AGENT_COUNT | Memory files: $MEMORY_FILE_COUNT
+Checksum: sha256:${CHECKSUM:0:16}...
+Agent: mighty-mark (sentinel duty — fleet safety checkpoint)"
+
+      git push origin "$GITHUB_BACKUP_BRANCH" --force
+      log "  Pushed to origin/$GITHUB_BACKUP_BRANCH"
+    fi
+  )
+  PUSH_EXIT=$?
+  if [[ $PUSH_EXIT -ne 0 ]]; then
+    log "  WARNING: GitHub push failed (backup is still saved locally)"
+    notify_telegram "🟡 *MIGHTY MARK — GitHub Push Failed*%0A%0ATier: ${TIER} | Size: ${ARCHIVE_SIZE_MB} MB%0ABackup saved locally but GitHub push failed.%0A%0ACheck logs: /opt/fleet-backup/logs/backup.log%0AServer: $(hostname)%0ATime: $(date)"
+  fi
+
+  push_cleanup
+fi