@aiassesstech/mighty-mark 0.3.29 → 0.3.32

package/src/watchdog/fleet-backup/fleet-restore.sh

@@ -0,0 +1,437 @@
+#!/usr/bin/env bash
+# ═══════════════════════════════════════════════════════════════════
+# nole-fleet-backup — Restore from a backup archive
+#
+# Supports both full and light tiered backups, and archives that
+# were split into chunks for GitHub storage.
+#
+# Usage:
+#   ./fleet-restore.sh                                  # List all
+#   ./fleet-restore.sh --list                           # List all
+#   ./fleet-restore.sh --inspect fleet-full-2026-03-02-090000   # Show contents
+#   ./fleet-restore.sh --verify  fleet-full-2026-03-02-090000   # Verify checksum
+#   ./fleet-restore.sh --extract fleet-full-2026-03-02-090000   # Extract to /tmp
+#   ./fleet-restore.sh --restore fleet-full-2026-03-02-090000   # Full restore
+#   ./fleet-restore.sh --from-github                    # Download + restore from GitHub
+# ═══════════════════════════════════════════════════════════════════
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ -f "$SCRIPT_DIR/config.local.sh" ]]; then
+  source "$SCRIPT_DIR/config.local.sh"
+else
+  source "$SCRIPT_DIR/config.sh"
+fi
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+log()  { echo -e "${GREEN}[restore]${NC} $1"; }
+warn() { echo -e "${YELLOW}[restore]${NC} $1"; }
+err()  { echo -e "${RED}[restore]${NC} $1" >&2; }
+
+list_backups() {
+  echo "Available backups in $BACKUP_DIR:"
+  echo ""
+  if [[ ! -d "$BACKUP_DIR" ]]; then
+    err "Backup directory not found: $BACKUP_DIR"
+    exit 1
+  fi
+
+  local count=0
+  for archive in $(find "$BACKUP_DIR" -name 'fleet-*.tar.gz' | sort -r); do
+    local name=$(basename "$archive" .tar.gz)
+    local size=$(stat -c%s "$archive" 2>/dev/null || stat -f%z "$archive" 2>/dev/null || echo 0)
+    local size_mb=$(awk "BEGIN {printf \"%.1f\", $size / 1048576}")
+
+    local tier="unknown"
+    if [[ "$name" == *"-full-"* ]]; then
+      tier="FULL"
+    elif [[ "$name" == *"-light-"* ]]; then
+      tier="light"
+    elif [[ "$name" == *"fleet-backup-"* ]]; then
+      tier="legacy"
+    fi
+
+    printf "  %-8s  %-42s  %8s MB\n" "[$tier]" "$name" "$size_mb"
+    count=$((count + 1))
+  done
+
+  echo ""
+  echo "$count backup(s) found. Retention: $RETENTION_DAYS days."
+}
+
+get_archive_path() {
+  local id="$1"
+  local path="$BACKUP_DIR/${id}.tar.gz"
+
+  if [[ -f "$path" ]]; then
+    echo "$path"
+    return
+  fi
+
+  # Try legacy naming (fleet-backup-YYYY-MM-DD.tar.gz)
+  local legacy="$BACKUP_DIR/fleet-backup-${id}.tar.gz"
+  if [[ -f "$legacy" ]]; then
+    echo "$legacy"
+    return
+  fi
+
+  # Search for partial match
+  local matches
+  matches=$(find "$BACKUP_DIR" -name "*${id}*.tar.gz" 2>/dev/null | head -1)
+  if [[ -n "$matches" ]]; then
+    echo "$matches"
+    return
+  fi
+
+  err "No backup found matching: $id"
+  err "Expected: $path"
+  echo ""
+  list_backups
+  exit 1
+}
+
+verify_backup() {
+  local id="$1"
+  local archive
+  archive=$(get_archive_path "$id")
+
+  log "Verifying checksum for $(basename "$archive")..."
+
+  local actual
+  actual=$(sha256sum "$archive" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$archive" | awk '{print $1}')
+
+  if [[ -f "$MANIFEST_FILE" ]]; then
+    local manifest_checksum
+    manifest_checksum=$(grep -o '"checksum": *"[^"]*"' "$MANIFEST_FILE" | head -1 | sed 's/.*: *"sha256://;s/"//')
+    local manifest_archive
+    manifest_archive=$(grep -o '"archive": *"[^"]*"' "$MANIFEST_FILE" | head -1 | sed 's/.*: *"//;s/"//')
+
+    if [[ "$(basename "$archive")" == "$manifest_archive" && -n "$manifest_checksum" ]]; then
+      if [[ "$actual" == "$manifest_checksum" ]]; then
+        log "Checksum VERIFIED: sha256:$actual"
+        return 0
+      else
+        err "Checksum MISMATCH!"
+        err "  Expected: $manifest_checksum"
+        err "  Actual:   $actual"
+        return 1
+      fi
+    fi
+  fi
+
+  log "Checksum: sha256:$actual (no matching manifest to compare against)"
+}
+
+inspect_backup() {
+  local id="$1"
+  local archive
+  archive=$(get_archive_path "$id")
+
+  log "Contents of $(basename "$archive"):"
+  echo ""
+  tar -tzf "$archive" | head -100
+  echo ""
+
+  local total
+  total=$(tar -tzf "$archive" | wc -l | tr -d ' ')
+  log "Total entries: $total"
+}
+
+extract_backup() {
+  local id="$1"
+  local archive
+  archive=$(get_archive_path "$id")
+  local name=$(basename "$archive" .tar.gz)
+  local extract_dir="/tmp/fleet-restore-${name}"
+
+  if [[ -d "$extract_dir" ]]; then
+    warn "Extract directory already exists: $extract_dir"
+    read -p "Remove and re-extract? (y/N): " confirm
+    [[ "$confirm" =~ ^[Yy]$ ]] || exit 0
+    rm -rf "$extract_dir"
+  fi
+
+  log "Extracting to $extract_dir..."
+  mkdir -p "$extract_dir"
+  tar -xzf "$archive" -C "$extract_dir"
+
+  log "Extracted. Review contents:"
+  echo ""
+  ls -la "$extract_dir/fleet-backup/"
+  echo ""
+  log "To inspect: ls $extract_dir/fleet-backup/"
+  log "To clean up: rm -rf $extract_dir"
+}
+
+from_github() {
+  if [[ ! -d "${GITHUB_REPO_DIR:-}" ]]; then
+    err "GITHUB_REPO_DIR not set or does not exist: ${GITHUB_REPO_DIR:-<unset>}"
+    exit 1
+  fi
+
+  log "Fetching latest backup from GitHub..."
+  local work_dir
+  work_dir=$(mktemp -d "/tmp/fleet-github-restore-XXXXXX")
+
+  (
+    cd "$GITHUB_REPO_DIR"
+    git fetch origin "${GITHUB_BACKUP_BRANCH}" 2>/dev/null || {
+      err "Could not fetch branch: $GITHUB_BACKUP_BRANCH"
+      exit 1
+    }
+
+    git worktree add "$work_dir" "origin/$GITHUB_BACKUP_BRANCH" 2>/dev/null || {
+      # Fallback: archive from remote
+      git archive "origin/$GITHUB_BACKUP_BRANCH" | tar -x -C "$work_dir"
+    }
+  )
+
+  cd "$work_dir"
+
+  # Check for chunked archive and reassemble
+  local chunks
+  chunks=$(find . -name 'fleet-*.tar.gz.part-*' 2>/dev/null | sort)
+  if [[ -n "$chunks" ]]; then
+    local base_name
+    base_name=$(echo "$chunks" | head -1 | sed 's/\.part-[0-9]*//')
+    base_name=$(basename "$base_name")
+    log "Reassembling ${base_name} from $(echo "$chunks" | wc -l | tr -d ' ') chunks..."
+    cat $chunks > "$BACKUP_DIR/$base_name"
+    log "Reassembled: $BACKUP_DIR/$base_name"
+  else
+    local archive
+    archive=$(find . -name 'fleet-*.tar.gz' 2>/dev/null | head -1)
+    if [[ -n "$archive" ]]; then
+      cp "$archive" "$BACKUP_DIR/"
+      log "Copied: $(basename "$archive") to $BACKUP_DIR/"
+    else
+      err "No archive found on the backups branch"
+      rm -rf "$work_dir"
+      exit 1
+    fi
+  fi
+
+  # Copy manifest
+  [[ -f "latest-manifest.json" ]] && cp "latest-manifest.json" "$MANIFEST_FILE"
+
+  rm -rf "$work_dir"
+  git -C "$GITHUB_REPO_DIR" worktree prune 2>/dev/null || true
+
+  log "Archive downloaded from GitHub. Use --list to see it, --restore to apply."
+}
+
+restore_backup() {
+  local id="$1"
+  local archive
+  archive=$(get_archive_path "$id")
+  local name=$(basename "$archive" .tar.gz)
+
+  local tier="unknown"
+  if [[ "$name" == *"-full-"* ]]; then
+    tier="FULL"
+  elif [[ "$name" == *"-light-"* ]]; then
+    tier="LIGHT"
+  fi
+
+  echo ""
+  warn "═══════════════════════════════════════════════════════"
+  warn "  $tier RESTORE from backup: $name"
+  warn "═══════════════════════════════════════════════════════"
+  echo ""
+
+  if [[ "$tier" == "LIGHT" ]]; then
+    warn "This is a LIGHT backup. It contains only:"
+    echo "  - Agent .md files (identity, prompts, memory markdown)"
+    echo "  - Sanitized openclaw.json configs"
+    echo "  - Systemd/cron/logrotate configs"
+    echo ""
+    warn "It does NOT contain:"
+    echo "  - Extension dist/ code"
+    echo "  - Vector indexes (.sqlite)"
+    echo "  - Data stores (grillo-data, .noah-data, etc.)"
+    echo "  - Session transcripts"
+    echo "  - Fleet-bus workspace"
+    echo ""
+    warn "For a complete restore, use the latest FULL backup first,"
+    warn "then overlay a LIGHT backup for the most recent .md state."
+  else
+    warn "This will overwrite the following on this system:"
+    echo "  - $OPENCLAW_HOME/extensions/*/dist/"
+    echo "  - $OPENCLAW_HOME/extensions/*/agent/"
+    echo "  - $OPENCLAW_HOME/agents/"
+    echo "  - Agent data stores (grillo-data, .noah-data, etc.)"
+    echo "  - Fleet-bus workspace"
+    echo "  - Mighty Mark watchdog scripts"
+    echo "  - Systemd/cron configs"
+  fi
+  echo ""
+  warn "Secrets (.env, credentials, API keys) are NOT in the"
+  warn "backup and will NOT be affected."
+  echo ""
+
+  read -p "Stop openclaw-gateway before restore? (Y/n): " stop_gw
+  if [[ ! "$stop_gw" =~ ^[Nn]$ ]]; then
+    log "Stopping openclaw-gateway..."
+    systemctl stop openclaw-gateway 2>/dev/null || warn "Could not stop gateway (may not be running)"
+  fi
+
+  verify_backup "$id" || {
+    err "Checksum verification failed. Aborting restore."
+    exit 1
+  }
+
+  local extract_dir="/tmp/fleet-restore-${name}"
+  rm -rf "$extract_dir"
+  mkdir -p "$extract_dir"
+  tar -xzf "$archive" -C "$extract_dir"
+  local src="$extract_dir/fleet-backup"
+
+  read -p "Proceed with restore? Type YES to confirm: " confirm
+  if [[ "$confirm" != "YES" ]]; then
+    err "Restore cancelled."
+    rm -rf "$extract_dir"
+    exit 1
+  fi
+
+  # Restore extensions
+  if [[ -d "$src/extensions" ]]; then
+    log "Restoring extensions..."
+    for agent_dir in "$src/extensions"/*/; do
+      agent_name=$(basename "$agent_dir")
+      target="$OPENCLAW_HOME/extensions/$agent_name"
+      if [[ -d "$target" ]]; then
+        [[ -d "$agent_dir/dist" ]] && cp -r "$agent_dir/dist"/* "$target/dist/" 2>/dev/null || true
+        [[ -d "$agent_dir/agent" ]] && cp -r "$agent_dir/agent"/* "$target/agent/" 2>/dev/null || true
+        [[ -f "$agent_dir/package.json" ]] && cp "$agent_dir/package.json" "$target/"
+        log "  Restored: $agent_name"
+      else
+        warn "  Skipped $agent_name (target dir does not exist)"
+      fi
+    done
+  fi
+
+  # Restore agent workspaces
+  if [[ -d "$src/agents" ]]; then
+    log "Restoring agent workspaces..."
+    cp -r "$src/agents"/* "$OPENCLAW_HOME/agents/" 2>/dev/null || true
+  fi
+
+  # Full-only restores
+  if [[ "$tier" == "FULL" ]] || [[ "$tier" == "unknown" ]]; then
+
+    # Restore vector indexes
+    if [[ -d "$src/memory" ]]; then
+      log "Restoring vector indexes..."
+      mkdir -p "$OPENCLAW_HOME/memory"
+      cp "$src/memory"/*.sqlite "$OPENCLAW_HOME/memory/" 2>/dev/null || true
+    fi
+
+    # Restore data stores
+    for data_dir in grillo-data .noah-data nole-data .mark-data .fleet-data .sam-artifacts; do
+      if [[ -d "$src/$data_dir" ]]; then
+        log "Restoring data store: $data_dir"
+        mkdir -p "$OPENCLAW_HOME/$data_dir"
+        cp -r "$src/$data_dir"/* "$OPENCLAW_HOME/$data_dir/" 2>/dev/null || true
+      fi
+    done
+
+    # Restore workspace
+    if [[ -d "$src/workspace" ]]; then
+      log "Restoring workspace (fleet-bus, skills)..."
+      mkdir -p "$OPENCLAW_HOME/workspace"
+      cp -r "$src/workspace"/* "$OPENCLAW_HOME/workspace/" 2>/dev/null || true
+    fi
+
+    # Restore watchdog
+    if [[ -d "$src/mighty-mark-watchdog" ]]; then
+      log "Restoring Mighty Mark watchdog scripts..."
+      cp -r "$src/mighty-mark-watchdog"/* "$MIGHTY_MARK_OPT/" 2>/dev/null || true
+    fi
+
+  fi
+
+  # Restore system configs
+  if [[ -d "$src/system" ]]; then
+    log "Restoring system configs..."
+    for f in "$src/system"/*; do
+      fname=$(basename "$f")
+      case "$fname" in
+        *.service) cp "$f" /etc/systemd/system/ && log "  Restored: $fname" ;;
+        mighty-mark|fleet-backup) cp "$f" /etc/cron.d/ && log "  Restored cron: $fname" ;;
+        *) log "  Skipped system file: $fname" ;;
+      esac
+    done
+    systemctl daemon-reload 2>/dev/null || true
+  fi
+
+  rm -rf "$extract_dir"
+
+  echo ""
+  log "Restore complete from backup: $name [$tier]"
+  echo ""
+  warn "IMPORTANT — Manual steps remaining:"
+  echo "  1. Verify openclaw.json has correct API keys (not in backup)"
+  echo "  2. Verify /opt/mighty-mark/.env has Telegram credentials"
+  echo "  3. Verify ~/.nole/credentials if Nole wallet is configured"
+  echo "  4. Re-install fleet-bus for each extension (npm pack pattern)"
+  echo "  5. Restart gateway: systemctl start openclaw-gateway"
+  echo "  6. Verify logs: journalctl -u openclaw-gateway --no-pager -n 50"
+}
+
+# ─── CLI ──────────────────────────────────────────────────────────
+
+case "${1:---list}" in
+  --list|-l)
+    list_backups
+    ;;
+  --inspect|-i)
+    [[ -z "${2:-}" ]] && { err "Usage: $0 --inspect ARCHIVE-NAME"; exit 1; }
+    inspect_backup "$2"
+    ;;
+  --verify|-v)
+    [[ -z "${2:-}" ]] && { err "Usage: $0 --verify ARCHIVE-NAME"; exit 1; }
+    verify_backup "$2"
+    ;;
+  --extract|-e)
+    [[ -z "${2:-}" ]] && { err "Usage: $0 --extract ARCHIVE-NAME"; exit 1; }
+    extract_backup "$2"
+    ;;
+  --restore|-r)
+    [[ -z "${2:-}" ]] && { err "Usage: $0 --restore ARCHIVE-NAME"; exit 1; }
+    restore_backup "$2"
+    ;;
+  --from-github|-g)
+    from_github
+    ;;
+  --help|-h)
+    echo "Usage: $0 [command] [archive-name]"
+    echo ""
+    echo "Commands:"
+    echo "  --list, -l                     List available backups (default)"
+    echo "  --inspect, -i ARCHIVE-NAME     Show archive contents"
+    echo "  --verify, -v ARCHIVE-NAME      Verify checksum against manifest"
+    echo "  --extract, -e ARCHIVE-NAME     Extract to /tmp for review"
+    echo "  --restore, -r ARCHIVE-NAME     Restore (interactive, with confirmation)"
+    echo "  --from-github, -g              Download latest backup from GitHub"
+    echo "  --help, -h                     Show this help"
+    echo ""
+    echo "Archive name: e.g. fleet-full-2026-03-02-090000"
+    echo "              (omit .tar.gz extension)"
+    echo ""
+    echo "Recommended disaster recovery:"
+    echo "  1. $0 --from-github                          # download from GitHub"
+    echo "  2. $0 --restore fleet-full-YYYY-MM-DD-HHMMSS  # restore latest full"
+    echo "  3. $0 --restore fleet-light-YYYY-MM-DD-HHMMSS # overlay latest light"
+    ;;
+  *)
+    err "Unknown command: $1"
+    err "Run $0 --help for usage"
+    exit 1
+    ;;
+esac