@aiassesstech/mighty-mark 0.1.0

package/src/watchdog/install.sh

@@ -0,0 +1,244 @@
+#!/bin/bash
+# /opt/mighty-mark/install.sh
+# Mighty Mark — Watchdog Installer for Hetzner VPS
+#
+# What this does:
+#   1. Creates /opt/mighty-mark/ directory structure
+#   2. Copies watchdog.sh, morning-check.sh, and lib/ scripts
+#   3. Installs cron job in /etc/cron.d/mighty-mark (CRON_TZ=America/Chicago)
+#   4. Sets up logrotate in /etc/logrotate.d/mighty-mark
+#   5. Installs systemd service file (optional)
+#   6. Creates .env template for Telegram credentials
+#   7. Initializes state files
+#   8. Tests Telegram connectivity
+#
+# Usage:
+#   sudo bash install.sh [--telegram-token <token>] [--telegram-chat-id <id>]
+#
+# Must be run as root.
+
+set -euo pipefail
+
+# ── Parse Arguments ──
+TELEGRAM_TOKEN=""
+TELEGRAM_CHAT_ID=""
+SKIP_CRON=false
+SKIP_LOGROTATE=false
+DRY_RUN=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --telegram-token)     TELEGRAM_TOKEN="$2"; shift 2 ;;
+    --telegram-chat-id)   TELEGRAM_CHAT_ID="$2"; shift 2 ;;
+    --skip-cron)          SKIP_CRON=true; shift ;;
+    --skip-logrotate)     SKIP_LOGROTATE=true; shift ;;
+    --dry-run)            DRY_RUN=true; shift ;;
+    *)                    echo "Unknown option: $1"; exit 1 ;;
+  esac
+done
+
+# ── Check root ──
+if [ "$(id -u)" -ne 0 ] && [ "$DRY_RUN" = false ]; then
+  echo "ERROR: This script must be run as root (sudo bash install.sh)"
+  exit 1
+fi
+
+MARK_HOME="/opt/mighty-mark"
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "╔══════════════════════════════════════════════════╗"
+echo "║   Mighty Mark — Watchdog Installer               ║"
+echo "╚══════════════════════════════════════════════════╝"
+echo ""
+
+# ── Step 1: Create directory structure ──
+echo "[1/7] Creating directory structure..."
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would create: ${MARK_HOME}/{logs,state,lib}"
+else
+  mkdir -p "${MARK_HOME}"/{logs,state,lib}
+  echo "  Created: ${MARK_HOME}/"
+  echo "  Created: ${MARK_HOME}/logs/"
+  echo "  Created: ${MARK_HOME}/state/"
+  echo "  Created: ${MARK_HOME}/lib/"
+fi
+
+# ── Step 2: Copy scripts ──
+echo "[2/7] Installing scripts..."
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would copy watchdog.sh, morning-check.sh, lib/*.sh"
+else
+  cp "${SOURCE_DIR}/watchdog.sh"       "${MARK_HOME}/watchdog.sh"
+  cp "${SOURCE_DIR}/morning-check.sh"  "${MARK_HOME}/morning-check.sh"
+  cp "${SOURCE_DIR}/lib/config.sh"     "${MARK_HOME}/lib/config.sh"
+  cp "${SOURCE_DIR}/lib/logging.sh"    "${MARK_HOME}/lib/logging.sh"
+  cp "${SOURCE_DIR}/lib/notify.sh"     "${MARK_HOME}/lib/notify.sh"
+
+  chmod +x "${MARK_HOME}/watchdog.sh"
+  chmod +x "${MARK_HOME}/morning-check.sh"
+
+  echo "  Installed: watchdog.sh"
+  echo "  Installed: morning-check.sh"
+  echo "  Installed: lib/config.sh, logging.sh, notify.sh"
+fi
+
+# ── Step 3: Create .env file ──
+echo "[3/7] Setting up environment..."
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would create ${MARK_HOME}/.env"
+else
+  if [ ! -f "${MARK_HOME}/.env" ]; then
+    cat > "${MARK_HOME}/.env" <<ENVEOF
+# Mighty Mark — Environment Configuration
+# Created by install.sh on $(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+# Telegram Bot (@MightyMarkBot)
+MIGHTY_MARK_TELEGRAM_TOKEN="${TELEGRAM_TOKEN}"
+MIGHTY_MARK_TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID}"
+
+# Thresholds (uncomment to override defaults)
+# MAX_RESTART_ATTEMPTS=3
+# DISK_WARN_PERCENT=20
+# MEMORY_WARN_MB=256
+# CPU_WARN_LOAD=3.0
+ENVEOF
+    chmod 600 "${MARK_HOME}/.env"
+    echo "  Created: ${MARK_HOME}/.env (permissions: 600)"
+  else
+    echo "  ${MARK_HOME}/.env already exists, skipping"
+    if [ -n "${TELEGRAM_TOKEN}" ]; then
+      # Update token in existing .env
+      if grep -q "MIGHTY_MARK_TELEGRAM_TOKEN" "${MARK_HOME}/.env"; then
+        sed -i "s|MIGHTY_MARK_TELEGRAM_TOKEN=.*|MIGHTY_MARK_TELEGRAM_TOKEN=\"${TELEGRAM_TOKEN}\"|" "${MARK_HOME}/.env"
+      else
+        echo "MIGHTY_MARK_TELEGRAM_TOKEN=\"${TELEGRAM_TOKEN}\"" >> "${MARK_HOME}/.env"
+      fi
+      echo "  Updated Telegram token in .env"
+    fi
+    if [ -n "${TELEGRAM_CHAT_ID}" ]; then
+      if grep -q "MIGHTY_MARK_TELEGRAM_CHAT_ID" "${MARK_HOME}/.env"; then
+        sed -i "s|MIGHTY_MARK_TELEGRAM_CHAT_ID=.*|MIGHTY_MARK_TELEGRAM_CHAT_ID=\"${TELEGRAM_CHAT_ID}\"|" "${MARK_HOME}/.env"
+      else
+        echo "MIGHTY_MARK_TELEGRAM_CHAT_ID=\"${TELEGRAM_CHAT_ID}\"" >> "${MARK_HOME}/.env"
+      fi
+      echo "  Updated Telegram chat ID in .env"
+    fi
+  fi
+fi
+
+# ── Step 4: Install cron job ──
+echo "[4/7] Installing cron schedule..."
+
+if [ "$SKIP_CRON" = true ]; then
+  echo "  Skipped (--skip-cron)"
+elif [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would create /etc/cron.d/mighty-mark"
+else
+  cat > /etc/cron.d/mighty-mark <<CRONEOF
+# /etc/cron.d/mighty-mark
+# Mighty Mark — System Health Sentinel
+# CRON_TZ ensures schedules are in America/Chicago regardless of DST
+CRON_TZ=America/Chicago
+
+# Watchdog: every 5 minutes — check gateway heartbeat
+*/5 * * * * root ${MARK_HOME}/watchdog.sh >> ${MARK_HOME}/logs/watchdog-cron.log 2>&1
+
+# Morning Check: daily at 6:00 AM Chicago time (auto-adjusts for CDT/CST)
+0 6 * * * root ${MARK_HOME}/morning-check.sh >> ${MARK_HOME}/logs/morning-cron.log 2>&1
+CRONEOF
+  chmod 644 /etc/cron.d/mighty-mark
+  echo "  Installed: /etc/cron.d/mighty-mark"
+  echo "    Watchdog:      */5 * * * * (every 5 min)"
+  echo "    Morning Check: 0 6 * * *   (daily 6:00 AM CT)"
+fi
+
+# ── Step 5: Install logrotate ──
+echo "[5/7] Installing logrotate..."
+
+if [ "$SKIP_LOGROTATE" = true ]; then
+  echo "  Skipped (--skip-logrotate)"
+elif [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would create /etc/logrotate.d/mighty-mark"
+else
+  cat > /etc/logrotate.d/mighty-mark <<LREOF
+${MARK_HOME}/logs/*.log {
+    daily
+    missingok
+    rotate 14
+    compress
+    delaycompress
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        true
+    endscript
+}
+LREOF
+  chmod 644 /etc/logrotate.d/mighty-mark
+  echo "  Installed: /etc/logrotate.d/mighty-mark (14-day rotation)"
+fi
+
+# ── Step 6: Initialize state ──
+echo "[6/7] Initializing state files..."
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would initialize state files"
+else
+  if [ ! -f "${MARK_HOME}/state/incident-count.json" ]; then
+    echo '{}' > "${MARK_HOME}/state/incident-count.json"
+    echo "  Created: incident-count.json"
+  else
+    echo "  incident-count.json already exists"
+  fi
+fi
+
+# ── Step 7: Test Telegram connectivity ──
+echo "[7/7] Testing Telegram connectivity..."
+
+if [ "$DRY_RUN" = true ]; then
+  echo "  [DRY RUN] Would test Telegram API"
+elif [ -z "${TELEGRAM_TOKEN}" ]; then
+  echo "  Skipped — no Telegram token provided"
+  echo "  Set MIGHTY_MARK_TELEGRAM_TOKEN in ${MARK_HOME}/.env"
+else
+  source "${MARK_HOME}/lib/config.sh"
+  TELEGRAM_BOT_TOKEN="${TELEGRAM_TOKEN}"
+
+  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+    "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/getMe" 2>/dev/null || echo "000")
+
+  if [ "${HTTP_CODE}" = "200" ]; then
+    echo "  Telegram Bot API reachable (HTTP 200)"
+  else
+    echo "  WARNING: Telegram Bot API returned HTTP ${HTTP_CODE}"
+    echo "  Verify the bot token is correct"
+  fi
+fi
+
+# ── Done ──
+echo ""
+echo "╔══════════════════════════════════════════════════╗"
+echo "║   Mighty Mark watchdog installed successfully!   ║"
+echo "╚══════════════════════════════════════════════════╝"
+echo ""
+echo "  Install directory: ${MARK_HOME}"
+echo "  Cron schedule:     /etc/cron.d/mighty-mark"
+echo "  Logrotate:         /etc/logrotate.d/mighty-mark"
+echo "  Logs:              ${MARK_HOME}/logs/"
+echo "  State:             ${MARK_HOME}/state/"
+echo ""
+
+if [ -z "${TELEGRAM_TOKEN}" ]; then
+  echo "  NEXT STEPS:"
+  echo "    1. Create @MightyMarkBot on Telegram"
+  echo "    2. Edit ${MARK_HOME}/.env with your bot token and chat ID"
+  echo "    3. Test: bash ${MARK_HOME}/watchdog.sh"
+  echo ""
+fi
+
+echo "  Manual test: bash ${MARK_HOME}/watchdog.sh"
+echo "  View logs:   tail -f ${MARK_HOME}/logs/watchdog.log"

package/src/watchdog/lib/config.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# /opt/mighty-mark/lib/config.sh
+# Mighty Mark — Shared configuration for all watchdog scripts
+# Sourced by watchdog.sh, morning-check.sh, and install.sh
+
+# ── Paths ──
+MARK_HOME="${MARK_HOME:-/opt/mighty-mark}"
+MARK_LOG_DIR="${MARK_HOME}/logs"
+MARK_STATE_DIR="${MARK_HOME}/state"
+
+# ── OpenClaw ──
+OPENCLAW_BIN="${OPENCLAW_BIN:-/usr/bin/openclaw}"
+OPENCLAW_HOME="${OPENCLAW_HOME:-${HOME}/.clawdbot}"
+OPENCLAW_CONFIG="${OPENCLAW_HOME}/openclaw.json"
+OPENCLAW_SERVICE="openclaw-gateway"
+
+# ── Telegram ──
+# These MUST be set as environment variables or in /opt/mighty-mark/.env
+TELEGRAM_BOT_TOKEN="${MIGHTY_MARK_TELEGRAM_TOKEN:-}"
+TELEGRAM_CHAT_ID="${MIGHTY_MARK_TELEGRAM_CHAT_ID:-}"
+
+# ── Thresholds ──
+MAX_RESTART_ATTEMPTS="${MAX_RESTART_ATTEMPTS:-3}"
+RESTART_COOLDOWN_SECONDS="${RESTART_COOLDOWN_SECONDS:-300}"
+DISK_WARN_PERCENT="${DISK_WARN_PERCENT:-20}"
+MEMORY_WARN_MB="${MEMORY_WARN_MB:-256}"
+CPU_WARN_LOAD="${CPU_WARN_LOAD:-3.0}"
+
+# ── State Files ──
+INCIDENT_COUNT_FILE="${MARK_STATE_DIR}/incident-count.json"
+LAST_CHECK_FILE="${MARK_STATE_DIR}/last-check.json"
+
+# ── Log Files ──
+WATCHDOG_LOG="${MARK_LOG_DIR}/watchdog.log"
+MORNING_LOG="${MARK_LOG_DIR}/morning-check.log"
+GATEWAY_RESTART_LOG="${MARK_LOG_DIR}/gateway-restart.log"
+
+# ── Load .env if present ──
+if [ -f "${MARK_HOME}/.env" ]; then
+  # shellcheck source=/dev/null
+  set -a
+  source "${MARK_HOME}/.env"
+  set +a
+fi

package/src/watchdog/lib/logging.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# /opt/mighty-mark/lib/logging.sh
+# Mighty Mark — Structured logging functions
+
+log_watchdog() {
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [WATCHDOG] $1" >> "${WATCHDOG_LOG:-/opt/mighty-mark/logs/watchdog.log}"
+}
+
+log_morning() {
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [MORNING-CHECK] $1" >> "${MORNING_LOG:-/opt/mighty-mark/logs/morning-check.log}"
+}
+
+log_info() {
+  local component="${1}"
+  local message="${2}"
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [${component}] INFO: ${message}"
+}
+
+log_warn() {
+  local component="${1}"
+  local message="${2}"
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [${component}] WARN: ${message}"
+}
+
+log_error() {
+  local component="${1}"
+  local message="${2}"
+  echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [${component}] ERROR: ${message}" >&2
+}

package/src/watchdog/lib/notify.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# /opt/mighty-mark/lib/notify.sh
+# Mighty Mark — Telegram notification functions
+
+notify_telegram() {
+  local message="$1"
+
+  if [ -z "${TELEGRAM_BOT_TOKEN}" ] || [ -z "${TELEGRAM_CHAT_ID}" ]; then
+    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [NOTIFY] WARN: Telegram credentials not configured, skipping notification"
+    return 1
+  fi
+
+  curl -s -X POST \
+    "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+    -d "chat_id=${TELEGRAM_CHAT_ID}" \
+    -d "text=${message}" \
+    -d "parse_mode=Markdown" \
+    > /dev/null 2>&1
+
+  if [ $? -eq 0 ]; then
+    return 0
+  else
+    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] [NOTIFY] WARN: Telegram notification failed"
+    return 1
+  fi
+}
+
+notify_recovery() {
+  local restart_count="$1"
+  local hostname
+  hostname=$(hostname)
+  notify_telegram "🟡 *MIGHTY MARK RECOVERY*%0A%0AOpenClaw gateway was down and has been restarted.%0ARestart #${restart_count} today.%0A%0AServer: ${hostname}%0ATime: $(date)"
+}
+
+notify_max_restarts() {
+  local max="$1"
+  local hostname
+  hostname=$(hostname)
+  notify_telegram "🔴 *MIGHTY MARK ALERT*%0A%0AOpenClaw gateway has crashed ${max} times today.%0AManual intervention required.%0A%0AServer: ${hostname}%0ATime: $(date)"
+}
+
+notify_restart_failed() {
+  local hostname
+  hostname=$(hostname)
+  notify_telegram "🔴 *MIGHTY MARK ALERT*%0A%0AOpenClaw gateway restart FAILED.%0AManual intervention required.%0A%0AServer: ${hostname}%0ATime: $(date)"
+}

package/src/watchdog/morning-check.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+# /opt/mighty-mark/morning-check.sh
+# Mighty Mark Morning Check — Runs the Level 2 health check via Node.js
+#
+# Cron schedule: 0 6 * * * (with CRON_TZ=America/Chicago)
+#
+# This script is a thin bash wrapper that invokes the TypeScript health
+# engine via npx. The actual check logic lives in the npm package.
+# If Node.js is unavailable, it falls back to a minimal bash-only check.
+
+set -euo pipefail
+
+# ── Load shared config ──
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/lib/config.sh"
+source "${SCRIPT_DIR}/lib/logging.sh"
+source "${SCRIPT_DIR}/lib/notify.sh"
+
+# ── Ensure directories exist ──
+mkdir -p "${MARK_LOG_DIR}" "${MARK_STATE_DIR}"
+
+log_morning "Morning check starting"
+
+# ── Try Node.js health check first ──
+if command -v npx &> /dev/null; then
+  log_morning "Running TypeScript health engine via npx"
+
+  if npx @aiassesstech/mighty-mark check >> "${MORNING_LOG}" 2>&1; then
+    log_morning "Morning check completed successfully (TypeScript engine)"
+    exit 0
+  else
+    log_morning "WARN: TypeScript engine failed, falling back to bash checks"
+  fi
+else
+  log_morning "WARN: npx not found, using bash-only checks"
+fi
+
+# ── Bash Fallback ─────────────────────────────────────────────────
+# Minimal health check when Node.js is unavailable
+
+CHECKS_PASSED=0
+CHECKS_FAILED=0
+CHECKS_TOTAL=0
+FAILURES=""
+
+run_check() {
+  local name="$1"
+  local result="$2"
+  CHECKS_TOTAL=$((CHECKS_TOTAL + 1))
+  if [ "${result}" -eq 0 ]; then
+    CHECKS_PASSED=$((CHECKS_PASSED + 1))
+    log_morning "  PASS: ${name}"
+  else
+    CHECKS_FAILED=$((CHECKS_FAILED + 1))
+    FAILURES="${FAILURES}  - ${name}%0A"
+    log_morning "  FAIL: ${name}"
+  fi
+}
+
+# Check 1: Gateway process
+pgrep -f "${OPENCLAW_SERVICE}" > /dev/null 2>&1
+run_check "Gateway process alive" $?
+
+# Check 2: Config file exists
+test -f "${OPENCLAW_CONFIG}"
+run_check "Gateway config exists" $?
+
+# Check 3: Disk space > 10% free
+DISK_FREE=$(df / | awk 'NR==2{gsub(/%/,""); print 100-$5}')
+test "${DISK_FREE}" -gt 10 2>/dev/null
+run_check "Disk space > 10% free (${DISK_FREE}%)" $?
+
+# Check 4: Memory available
+MEM_AVAIL=$(free -m 2>/dev/null | awk 'NR==2{print $7}' || echo 9999)
+test "${MEM_AVAIL}" -gt 128 2>/dev/null
+run_check "Memory > 128MB available (${MEM_AVAIL}MB)" $?
+
+# Check 5: System uptime > 5 min
+UPTIME_SEC=$(awk '{printf "%d", $1}' /proc/uptime 2>/dev/null || echo 99999)
+test "${UPTIME_SEC}" -gt 300 2>/dev/null
+run_check "System uptime > 5 min" $?
+
+# ── Determine status ──
+if [ "${CHECKS_FAILED}" -gt 0 ]; then
+  STATUS="RED"
+  EMOJI="🔴"
+elif [ "${CHECKS_FAILED}" -eq 0 ] && [ "${CHECKS_TOTAL}" -gt 0 ]; then
+  STATUS="GREEN"
+  EMOJI="🟢"
+else
+  STATUS="YELLOW"
+  EMOJI="🟡"
+fi
+
+log_morning "Morning check complete: ${STATUS} (${CHECKS_PASSED}/${CHECKS_TOTAL} passed)"
+
+# ── Notify via Telegram ──
+HOSTNAME=$(hostname)
+MESSAGE="${EMOJI} *MIGHTY MARK MORNING CHECK* (bash fallback)%0A━━━━━━━━━━━━━━━━━━━━%0A*Status:* ${STATUS}%0A*Results:* ${CHECKS_PASSED}/${CHECKS_TOTAL} passed%0A*Server:* ${HOSTNAME}%0A*Time:* $(date)"
+
+if [ "${CHECKS_FAILED}" -gt 0 ]; then
+  MESSAGE="${MESSAGE}%0A%0A❌ *Failures:*%0A${FAILURES}"
+fi
+
+notify_telegram "${MESSAGE}"
+
+log_morning "Morning check notification sent"

package/src/watchdog/openclaw-gateway.service

@@ -0,0 +1,43 @@
+# /etc/systemd/system/openclaw-gateway.service
+# OpenClaw AI Gateway — Systemd Service
+#
+# Belt-and-suspenders with Mighty Mark watchdog:
+# - Systemd handles fast restarts (Restart=on-failure, RestartSec=30)
+# - Mighty Mark watchdog handles the case where systemd gives up (StartLimitBurst=5)
+#
+# Install:
+#   sudo cp openclaw-gateway.service /etc/systemd/system/
+#   sudo systemctl daemon-reload
+#   sudo systemctl enable openclaw-gateway
+#   sudo systemctl start openclaw-gateway
+
+[Unit]
+Description=OpenClaw AI Gateway
+After=network.target
+Documentation=https://www.aiassesstech.com
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/root
+ExecStart=/usr/bin/openclaw gateway start
+Restart=on-failure
+RestartSec=30
+StartLimitBurst=5
+StartLimitIntervalSec=600
+
+# Environment
+Environment=NODE_ENV=production
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=openclaw-gateway
+
+# Hardening
+NoNewPrivileges=true
+ProtectSystem=strict
+ReadWritePaths=/root/.clawdbot /root/.openclaw /tmp
+
+[Install]
+WantedBy=multi-user.target

package/src/watchdog/watchdog.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# /opt/mighty-mark/watchdog.sh
+# Mighty Mark Watchdog — Keeps the OpenClaw gateway alive
+#
+# Runs via cron every 5 minutes (CRON_TZ=America/Chicago)
+# Zero Node.js dependencies — pure bash + standard Unix tools
+#
+# Behavior:
+#   1. Check if openclaw-gateway process is alive
+#   2. If alive: log heartbeat, exit 0
+#   3. If dead: attempt restart via systemctl
+#   4. If restart succeeds: notify Greg via Telegram
+#   5. If restart fails: notify Greg with escalation alert
+#   6. Max 3 restart attempts per day
+#   7. State persists in /opt/mighty-mark/state/incident-count.json
+
+set -euo pipefail
+
+# ── Load shared config ──
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/lib/config.sh"
+source "${SCRIPT_DIR}/lib/logging.sh"
+source "${SCRIPT_DIR}/lib/notify.sh"
+
+# ── Ensure directories exist ──
+mkdir -p "${MARK_LOG_DIR}" "${MARK_STATE_DIR}"
+
+# ── Functions ──
+
+is_gateway_alive() {
+  if ! pgrep -f "${OPENCLAW_SERVICE}" > /dev/null 2>&1; then
+    return 1
+  fi
+
+  local pid
+  pid=$(pgrep -f "${OPENCLAW_SERVICE}" | head -1)
+  if [ -n "${pid}" ] && [ -d "/proc/${pid}" ]; then
+    local state
+    state=$(awk '/^State:/ {print $2}' "/proc/${pid}/status" 2>/dev/null || echo "?")
+    if [ "${state}" = "Z" ]; then
+      log_watchdog "Gateway process ${pid} is zombie"
+      return 1
+    fi
+    return 0
+  fi
+
+  return 1
+}
+
+get_restart_count_today() {
+  if [ -f "${INCIDENT_COUNT_FILE}" ]; then
+    local today
+    today=$(date +%Y-%m-%d)
+    local count
+    count=$(jq -r --arg d "${today}" '.[$d] // 0' "${INCIDENT_COUNT_FILE}" 2>/dev/null || echo 0)
+    echo "${count}"
+  else
+    echo 0
+  fi
+}
+
+increment_restart_count() {
+  local today
+  today=$(date +%Y-%m-%d)
+  if [ -f "${INCIDENT_COUNT_FILE}" ]; then
+    local tmp
+    tmp=$(mktemp)
+    jq --arg d "${today}" '.[$d] = ((.[$d] // 0) + 1)' "${INCIDENT_COUNT_FILE}" > "${tmp}" && mv "${tmp}" "${INCIDENT_COUNT_FILE}"
+  else
+    echo "{\"${today}\": 1}" > "${INCIDENT_COUNT_FILE}"
+  fi
+}
+
+restart_gateway() {
+  local restart_count
+  restart_count=$(get_restart_count_today)
+
+  if [ "${restart_count}" -ge "${MAX_RESTART_ATTEMPTS}" ]; then
+    log_watchdog "ERROR: Max restart attempts (${MAX_RESTART_ATTEMPTS}) reached today. Manual intervention required."
+    notify_max_restarts "${MAX_RESTART_ATTEMPTS}"
+    return 1
+  fi
+
+  log_watchdog "Attempting gateway restart (attempt $((restart_count + 1))/${MAX_RESTART_ATTEMPTS} today)"
+
+  # Kill any remaining processes
+  pkill -f "${OPENCLAW_SERVICE}" 2>/dev/null || true
+  sleep 2
+
+  # Start gateway
+  if systemctl is-enabled "${OPENCLAW_SERVICE}" > /dev/null 2>&1; then
+    systemctl restart "${OPENCLAW_SERVICE}"
+  else
+    # Fallback: direct start
+    nohup "${OPENCLAW_BIN}" gateway start --config "${OPENCLAW_CONFIG}" \
+      >> "${GATEWAY_RESTART_LOG}" 2>&1 &
+  fi
+
+  # Wait for startup
+  sleep 10
+
+  if is_gateway_alive; then
+    increment_restart_count
+    log_watchdog "Gateway restarted successfully"
+    notify_recovery "$((restart_count + 1))"
+    return 0
+  else
+    increment_restart_count
+    log_watchdog "ERROR: Gateway restart failed"
+    notify_restart_failed
+    return 1
+  fi
+}
+
+# ── Main ──
+log_watchdog "Heartbeat check"
+
+if is_gateway_alive; then
+  log_watchdog "Gateway alive (PID: $(pgrep -f ${OPENCLAW_SERVICE} | head -1))"
+  exit 0
+else
+  log_watchdog "WARNING: Gateway is DOWN"
+  restart_gateway
+fi