nexo-brain 5.3.20 → 5.3.21

package/src/scripts/nexo-immune 2.py

@@ -0,0 +1,936 @@
+#!/usr/bin/env python3
+"""
+NEXO Immune System — Health monitor & auto-repair.
+
+Runs every 30 minutes via LaunchAgent. Checks tokens, LaunchAgents, DBs,
+scripts, logs, disk, and remote server crons. Auto-repairs what it can,
+alerts via notification on NEW failures.
+
+Zero external dependencies. Stdlib + sqlite3 + urllib only.
+"""
+
+import fcntl
+import json
+import os
+import re
+import shlex
+import signal
+import sqlite3
+import ssl
+import subprocess
+import sys
+import time
+from datetime import datetime, date, timedelta
+from pathlib import Path
+
+
+try:
+    from client_preferences import resolve_user_model as _resolve_user_model
+    _USER_MODEL = _resolve_user_model()
+except Exception:
+    _USER_MODEL = ""
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+_script_dir = Path(__file__).resolve().parent
+_repo_src = _script_dir.parent
+NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(_repo_src) if (_repo_src / "server.py").exists() else str(NEXO_HOME)))
+if str(NEXO_CODE) not in sys.path:
+    sys.path.insert(0, str(NEXO_CODE))
+
+from agent_runner import AutomationBackendUnavailableError, run_automation_prompt
+
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+# ─── SSL context for macOS (certifi or system certs) ─────────────────────────
+def _make_ssl_context():
+    """Create an SSL context that works on macOS with Python.org Python."""
+    # Try certifi first (pip-installed)
+    try:
+        import certifi
+        ctx = ssl.create_default_context(cafile=certifi.where())
+        return ctx
+    except ImportError:
+        pass
+    # Try macOS system certificates
+    for ca_path in [
+        "/etc/ssl/cert.pem",
+        "/usr/local/etc/openssl/cert.pem",
+        "/usr/local/etc/openssl@3/cert.pem",
+        "/opt/homebrew/etc/openssl@3/cert.pem",
+    ]:
+        if os.path.exists(ca_path):
+            ctx = ssl.create_default_context(cafile=ca_path)
+            return ctx
+    # Last resort: unverified (still better than crashing)
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+SSL_CTX = _make_ssl_context()
+
+# ─── Paths ────────────────────────────────────────────────────────────────────
+HOME = Path.home()
+CLAUDE_DIR = NEXO_HOME
+COORD_DIR = CLAUDE_DIR / "coordination"
+BRAIN_DIR = CLAUDE_DIR / "brain"
+SCRIPTS_DIR = CLAUDE_DIR / "scripts"
+
+IMMUNE_STATUS = COORD_DIR / "immune-status.json"
+IMMUNE_LOG = COORD_DIR / "immune-log.json"
+LOCK_FILE = COORD_DIR / "immune-process.lock"
+
+# Configure your alert script here (optional)
+# ALERT_SCRIPT = SCRIPTS_DIR / "my-notify.sh"
+
+CLAUDE_MEM_DB = HOME / ".claude-mem" / "claude-mem.db"
+
+LAUNCH_AGENTS_DIR = HOME / "Library" / "LaunchAgents"
+CLAUDE_CLI = HOME / ".local" / "bin" / "claude"
+
+NOW = datetime.now()
+TODAY = date.today()
+
+# ─── Config ───────────────────────────────────────────────────────────────────
+
+# Token checks — configure for your services.
+# Supported types: file_text (read file, optional test_url), json_field (check for refresh_token),
+#                  service_account (check for private_key/client_email), hardcoded (direct URL test)
+TOKEN_CHECKS = [
+    # Example: uncomment and configure for your services
+    # {
+    #     "name": "My API",
+    #     "path": "~/.nexo/my_api_token.txt",
+    #     "type": "file_text",
+    #     "test_url": "https://api.example.com/health?token={token}",
+    # },
+    # {
+    #     "name": "My Service Account",
+    #     "path": "~/.nexo/service-account.json",
+    #     "type": "service_account",
+    # },
+]
+
+EXPECTED_AGENTS = [
+    "com.nexo.immune",
+    "com.nexo.sleep",
+    "com.nexo.synthesis",
+]
+
+# SSH check interval — only every 2 hours, not every 30 min
+SSH_CHECK_INTERVAL_HOURS = 2
+
+# Log size thresholds (bytes)
+LOG_WARN_SIZE = 10 * 1024 * 1024   # 10 MB
+LOG_FAIL_SIZE = 50 * 1024 * 1024   # 50 MB
+LOG_TRUNCATE_SIZE = 50 * 1024 * 1024  # 50 MB — auto-truncate threshold
+
+# Disk thresholds (percentage used)
+DISK_WARN_PCT = 85
+DISK_FAIL_PCT = 95
+
+# Quiet hours — no WhatsApp alerts
+QUIET_START = 23  # 23:00
+QUIET_END = 7     # 07:00
+
+# Skip execution hours (deep night)
+SKIP_START = 0    # 00:00
+SKIP_END = 6      # 06:00
+
+# Max entries in immune-log.json
+MAX_LOG_ENTRIES = 500
+
+# HTTP timeout for token checks
+HTTP_TIMEOUT = 10
+
+# SSH timeout
+SSH_TIMEOUT = 15
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────────────
+
+def load_json(path, default=None):
+    if not path.exists():
+        return default if default is not None else {}
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return default if default is not None else {}
+
+
+def save_json(path, data):
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
+
+
+def is_quiet_hours():
+    """Check if within WhatsApp quiet hours (23:00 - 07:00)."""
+    h = NOW.hour
+    if QUIET_START > QUIET_END:
+        return h >= QUIET_START or h < QUIET_END
+    return QUIET_START <= h < QUIET_END
+
+
+def is_skip_hours():
+    """Check if within skip hours (00:00 - 06:00)."""
+    return SKIP_START <= NOW.hour < SKIP_END
+
+
+def send_alert(title, message):
+    """Send alert notification if not in quiet hours.
+
+    Configure ALERT_SCRIPT at the top of this file to enable.
+    Override this function for custom alerting (email, Slack, etc.).
+    """
+    if is_quiet_hours():
+        print(f"  [QUIET] Suppressed alert: {title}")
+        return False
+    # Default: log only. Configure ALERT_SCRIPT for active notifications.
+    print(f"  [ALERT] {title}: {message}")
+    return True
+
+
+def http_get(url, headers=None, timeout=HTTP_TIMEOUT):
+    """Simple HTTP GET, returns (status_code, body) or (0, error_string)."""
+    try:
+        req = Request(url)
+        if headers:
+            for k, v in headers.items():
+                req.add_header(k, v)
+        with urlopen(req, timeout=timeout, context=SSL_CTX) as resp:
+            body = resp.read().decode("utf-8", errors="replace")
+            return resp.status, body
+    except HTTPError as e:
+        return e.code, str(e)
+    except URLError as e:
+        return 0, str(e.reason)
+    except Exception as e:
+        return 0, str(e)
+
+
+def run_cmd(cmd, timeout=30):
+    """Run a command without invoking a shell. Accepts string or argv list."""
+    try:
+        argv = shlex.split(cmd) if isinstance(cmd, str) else list(cmd)
+        r = subprocess.run(
+            argv, capture_output=True, text=True, timeout=timeout
+        )
+        return r.returncode, r.stdout.strip(), r.stderr.strip()
+    except subprocess.TimeoutExpired:
+        return -1, "", "timeout"
+    except Exception as e:
+        return -1, "", str(e)
+
+
+def pid_alive(pid):
+    """Check if a PID is still running."""
+    try:
+        os.kill(pid, 0)
+        return True
+    except (OSError, ProcessLookupError):
+        return False
+
+
+# ─── Check Functions ──────────────────────────────────────────────────────────
+
+def check_tokens():
+    """Check all configured tokens. Returns list of result dicts."""
+    results = []
+
+    for tc in TOKEN_CHECKS:
+        name = tc["name"]
+        result = {"name": name, "status": "OK", "detail": ""}
+
+        try:
+            if tc["type"] == "file_text":
+                path = Path(tc["path"]).expanduser()
+                if not path.exists():
+                    result["status"] = "FAIL"
+                    result["detail"] = f"Token file missing: {path}"
+                else:
+                    token = path.read_text().strip()
+                    if not token:
+                        result["status"] = "FAIL"
+                        result["detail"] = "Token file empty"
+                    elif "test_url" in tc:
+                        url = tc["test_url"].format(token=token)
+                        code, body = http_get(url)
+                        if code == 200:
+                            result["detail"] = "HTTP 200 OK"
+                        elif code == 190 or (isinstance(body, str) and "expired" in body.lower()):
+                            result["status"] = "FAIL"
+                            result["detail"] = f"Token expired (HTTP {code})"
+                        else:
+                            result["status"] = "FAIL"
+                            result["detail"] = f"HTTP {code}: {body[:200]}"
+
+            elif tc["type"] == "json_field":
+                path = Path(tc["path"]).expanduser()
+                if not path.exists():
+                    result["status"] = "FAIL"
+                    result["detail"] = f"Token file missing: {path}"
+                else:
+                    data = load_json(path, default=None)
+                    if data is None:
+                        result["status"] = "FAIL"
+                        result["detail"] = "Invalid JSON"
+                    elif "refresh_token" not in data:
+                        result["status"] = "FAIL"
+                        result["detail"] = "No refresh_token in JSON"
+                    else:
+                        result["detail"] = "refresh_token present"
+
+            elif tc["type"] == "service_account":
+                path = Path(tc["path"]).expanduser()
+                if not path.exists():
+                    result["status"] = "FAIL"
+                    result["detail"] = f"Service account file missing: {path}"
+                else:
+                    data = load_json(path, default=None)
+                    if data is None:
+                        result["status"] = "FAIL"
+                        result["detail"] = "Invalid JSON"
+                    elif "private_key" not in data or "client_email" not in data:
+                        result["status"] = "FAIL"
+                        result["detail"] = "Missing private_key or client_email"
+                    else:
+                        result["detail"] = f"SA: {data.get('client_email', '?')[:40]}"
+
+            elif tc["type"] == "hardcoded":
+                url = tc["test_url"]
+                headers = {tc["header"]: tc["token"]}
+                code, body = http_get(url, headers=headers)
+                if code == 200:
+                    result["detail"] = "HTTP 200 OK"
+                elif code == 401:
+                    result["status"] = "FAIL"
+                    result["detail"] = "Token unauthorized (401)"
+                else:
+                    result["status"] = "FAIL"
+                    result["detail"] = f"HTTP {code}: {body[:200]}"
+
+        except Exception as e:
+            result["status"] = "FAIL"
+            result["detail"] = f"Exception: {str(e)[:200]}"
+
+        results.append(result)
+
+    return results
+
+
+def check_launch_agents():
+    """Check that expected LaunchAgents are loaded. Auto-repair if not."""
+    results = []
+
+    # Get list of loaded agents
+    rc, stdout, _ = run_cmd("launchctl list")
+    loaded_labels = set()
+    if rc == 0:
+        for line in stdout.splitlines():
+            parts = line.split("\t")
+            if len(parts) >= 3:
+                loaded_labels.add(parts[2])
+
+    for agent in EXPECTED_AGENTS:
+        result = {"name": agent, "status": "OK", "detail": "", "repaired": False}
+
+        if agent in loaded_labels:
+            result["detail"] = "Loaded"
+        else:
+            # Try auto-repair
+            plist = LAUNCH_AGENTS_DIR / f"{agent}.plist"
+            if plist.exists():
+                rc, out, err = run_cmd(f"launchctl load '{plist}'")
+                if rc == 0:
+                    result["status"] = "WARN"
+                    result["detail"] = f"Was unloaded, auto-loaded successfully"
+                    result["repaired"] = True
+                else:
+                    result["status"] = "FAIL"
+                    result["detail"] = f"Unloaded, auto-load failed: {err[:100]}"
+            else:
+                result["status"] = "FAIL"
+                result["detail"] = f"Unloaded, plist not found: {plist}"
+
+        results.append(result)
+
+    return results
+
+
+def check_databases():
+    """Run PRAGMA integrity_check on known databases."""
+    results = []
+
+    dbs = [
+        ("nexo.db", NEXO_HOME / "data" / "nexo.db"),
+        ("cognitive.db", NEXO_HOME / "data" / "cognitive.db"),
+        ("claude-mem.db", CLAUDE_MEM_DB),
+    ]
+
+    for name, path in dbs:
+        result = {"name": name, "status": "OK", "detail": ""}
+
+        if not path.exists():
+            result["status"] = "FAIL"
+            result["detail"] = f"File missing: {path}"
+        else:
+            try:
+                conn = sqlite3.connect(str(path), timeout=5)
+                cursor = conn.execute("PRAGMA integrity_check")
+                check_result = cursor.fetchone()[0]
+                conn.close()
+                if check_result == "ok":
+                    size_mb = path.stat().st_size / (1024 * 1024)
+                    result["detail"] = f"Integrity OK ({size_mb:.1f} MB)"
+                else:
+                    result["status"] = "FAIL"
+                    result["detail"] = f"Integrity failed: {check_result[:200]}"
+            except Exception as e:
+                result["status"] = "FAIL"
+                result["detail"] = f"Error: {str(e)[:200]}"
+
+        results.append(result)
+
+    return results
+
+
+def check_scripts():
+    """Check stale lock files."""
+    results = []
+
+    # Stale lock files (PID dead)
+    lock_files = list(COORD_DIR.glob("*.lock"))
+    for lf in lock_files:
+        if lf == LOCK_FILE:
+            continue  # Skip our own lock
+        result = {"name": f"lock:{lf.name}", "status": "OK", "detail": "", "repaired": False}
+        try:
+            content = lf.read_text().strip()
+            if content and content.isdigit():
+                pid = int(content)
+                if pid_alive(pid):
+                    result["detail"] = f"PID {pid} alive"
+                else:
+                    # Auto-repair: remove stale lock
+                    lf.unlink()
+                    result["status"] = "WARN"
+                    result["detail"] = f"PID {pid} dead — lock removed"
+                    result["repaired"] = True
+            elif content:
+                # Lock file has non-PID content — check if size 0 (normal flock pattern)
+                if lf.stat().st_size == 0:
+                    result["detail"] = "Empty lock (flock pattern)"
+                else:
+                    result["detail"] = f"Non-PID content: {content[:50]}"
+            else:
+                result["detail"] = "Empty lock file"
+        except Exception as e:
+            result["detail"] = f"Error checking: {e}"
+        results.append(result)
+
+    return results
+
+
+def check_logs():
+    """Check log file sizes. Auto-truncate if > 50 MB."""
+    results = []
+
+    # JSON logs to check
+    json_logs = [
+        COORD_DIR / "heartbeat-log.json",
+        COORD_DIR / "reflection-log.json",
+        COORD_DIR / "immune-log.json",
+        COORD_DIR / "ops-board.json",
+        COORD_DIR / "messages.json",
+    ]
+
+    # Text logs to check
+    text_logs = [
+        COORD_DIR / "heartbeat-stdout.log",
+        COORD_DIR / "heartbeat-stderr.log",
+        COORD_DIR / "reflection-stdout.log",
+        COORD_DIR / "reflection-stderr.log",
+        COORD_DIR / "immune-stdout.log",
+        COORD_DIR / "immune-stderr.log",
+    ]
+
+    for log_path in json_logs + text_logs:
+        if not log_path.exists():
+            continue
+
+        result = {"name": log_path.name, "status": "OK", "detail": "", "repaired": False}
+        size = log_path.stat().st_size
+        size_mb = size / (1024 * 1024)
+
+        if size >= LOG_FAIL_SIZE:
+            result["status"] = "FAIL"
+            result["detail"] = f"{size_mb:.1f} MB — exceeds {LOG_FAIL_SIZE // (1024*1024)} MB"
+
+            # Auto-truncate
+            try:
+                if log_path.suffix == ".json":
+                    _truncate_json_log(log_path, keep_entries=200)
+                else:
+                    _truncate_text_log(log_path, keep_lines=1000)
+                new_size = log_path.stat().st_size / (1024 * 1024)
+                result["detail"] += f" -> truncated to {new_size:.1f} MB"
+                result["repaired"] = True
+            except Exception as e:
+                result["detail"] += f" -> truncate failed: {e}"
+
+        elif size >= LOG_WARN_SIZE:
+            result["status"] = "WARN"
+            result["detail"] = f"{size_mb:.1f} MB — approaching limit"
+        else:
+            result["detail"] = f"{size_mb:.2f} MB"
+
+        results.append(result)
+
+    return results
+
+
+def _truncate_json_log(path, keep_entries=200):
+    """Truncate a JSON log file to the last N entries."""
+    data = load_json(path, default=[])
+    if isinstance(data, list) and len(data) > keep_entries:
+        data = data[-keep_entries:]
+        save_json(path, data)
+    elif isinstance(data, dict):
+        # Some logs are dicts with a list value
+        for key in data:
+            if isinstance(data[key], list) and len(data[key]) > keep_entries:
+                data[key] = data[key][-keep_entries:]
+        save_json(path, data)
+
+
+def _truncate_text_log(path, keep_lines=1000):
+    """Truncate a text log to the last N lines."""
+    lines = path.read_text().splitlines()
+    if len(lines) > keep_lines:
+        path.write_text("\n".join(lines[-keep_lines:]) + "\n")
+
+
+def check_disk():
+    """Check disk usage via os.statvfs."""
+    results = []
+    result = {"name": "disk:/", "status": "OK", "detail": ""}
+
+    try:
+        st = os.statvfs("/")
+        total = st.f_frsize * st.f_blocks
+        avail = st.f_frsize * st.f_bavail
+        used = total - avail
+        pct = (used / total) * 100 if total > 0 else 0
+
+        avail_gb = avail / (1024 ** 3)
+        total_gb = total / (1024 ** 3)
+
+        if pct >= DISK_FAIL_PCT:
+            result["status"] = "FAIL"
+            result["detail"] = f"{pct:.1f}% used ({avail_gb:.1f} GB free of {total_gb:.0f} GB)"
+        elif pct >= DISK_WARN_PCT:
+            result["status"] = "WARN"
+            result["detail"] = f"{pct:.1f}% used ({avail_gb:.1f} GB free of {total_gb:.0f} GB)"
+        else:
+            result["detail"] = f"{pct:.1f}% used ({avail_gb:.1f} GB free of {total_gb:.0f} GB)"
+    except Exception as e:
+        result["status"] = "FAIL"
+        result["detail"] = f"Error: {e}"
+
+    results.append(result)
+    return results
+
+
+def check_server_crons():
+    """Check remote server crons via SSH. Only runs every 2 hours.
+
+    Configure SSH_SERVER_CMD below with your server details if you want
+    remote health checks. Leave empty to skip.
+    """
+    results = []
+    result = {"name": "remote-server", "status": "OK", "detail": ""}
+
+    # Configure your SSH health check command here (empty = skip)
+    # Example: 'ssh -p 22 user@myserver.example.com "echo OK"'
+    SSH_SERVER_CMD = ""
+
+    if not SSH_SERVER_CMD:
+        result["detail"] = "No remote server configured (SSH_SERVER_CMD empty)"
+        results.append(result)
+        return results, False
+
+    # Check if we should run (every 2 hours based on last check)
+    status = load_json(IMMUNE_STATUS)
+    last_ssh_str = status.get("last_ssh_check", "")
+    should_run = True
+
+    if last_ssh_str:
+        try:
+            last_ssh = datetime.strptime(last_ssh_str, "%Y-%m-%d %H:%M")
+            hours_ago = (NOW - last_ssh).total_seconds() / 3600
+            if hours_ago < SSH_CHECK_INTERVAL_HOURS:
+                result["detail"] = f"Skipped (last check {hours_ago:.1f}h ago, interval {SSH_CHECK_INTERVAL_HOURS}h)"
+                should_run = False
+        except Exception:
+            pass
+
+    if should_run:
+        rc, stdout, stderr = run_cmd(SSH_SERVER_CMD, timeout=SSH_TIMEOUT)
+
+        if rc == 0:
+            result["detail"] = f"Server OK: {stdout[:100]}"
+        else:
+            result["status"] = "FAIL"
+            err_short = (stderr or "unknown error")[:150]
+            result["detail"] = f"SSH failed (rc={rc}): {err_short}"
+
+    results.append(result)
+    return results, should_run
+
+
+# ─── Alerting ─────────────────────────────────────────────────────────────────
+
+def get_system_uptime_minutes():
+    """Get system uptime in minutes via sysctl."""
+    try:
+        r = subprocess.run(
+            ["sysctl", "-n", "kern.boottime"],
+            capture_output=True, text=True, timeout=5
+        )
+        if r.returncode == 0:
+            # Format: { sec = 1709000000, usec = 0 } ...
+            import re as _re
+            m = _re.search(r'sec\s*=\s*(\d+)', r.stdout)
+            if m:
+                boot_ts = int(m.group(1))
+                return (time.time() - boot_ts) / 60
+    except Exception:
+        pass
+    return 9999  # Assume long uptime if we can't determine
+
+
+def detect_new_failures(current_results, previous_status):
+    """Compare current results with previous to find NEW failures.
+
+    Includes debounce: SSH/server checks need 2 consecutive failures before alerting.
+    Includes boot grace: suppresses all alerts within 10 min of system boot.
+    """
+    # Boot grace period — suppress alerts when network may still be settling
+    uptime = get_system_uptime_minutes()
+    if uptime < 10:
+        print(f"  [GRACE] System uptime {uptime:.0f}min < 10min — suppressing alerts")
+        return []
+
+    prev_checks = {}
+    for category in previous_status.get("checks", {}):
+        for item in previous_status["checks"][category]:
+            key = f"{category}:{item.get('name', '')}"
+            prev_checks[key] = item.get("status", "OK")
+
+    # Load consecutive failure counts for debounce
+    consec_file = COORD_DIR / "immune-consecutive-failures.json"
+    consec = load_json(consec_file, default={})
+
+    new_failures = []
+    for category, items in current_results.items():
+        for item in items:
+            key = f"{category}:{item.get('name', '')}"
+            current_status = item.get("status", "OK")
+            prev_stat = prev_checks.get(key, "OK")
+
+            if current_status in ("FAIL", "WARN"):
+                consec[key] = consec.get(key, 0) + 1
+            else:
+                consec[key] = 0
+
+            # Debounce: server/SSH checks need 2+ consecutive failures
+            is_server_check = category == "server" or "ssh" in key.lower()
+            min_consecutive = 2 if is_server_check else 1
+
+            if current_status == "FAIL" and prev_stat != "FAIL":
+                if consec.get(key, 0) >= min_consecutive:
+                    new_failures.append(item)
+            elif current_status == "WARN" and prev_stat == "OK":
+                if consec.get(key, 0) >= min_consecutive:
+                    new_failures.append(item)
+
+    save_json(consec_file, consec)
+    return new_failures
+
+
+def send_failure_alerts(new_failures):
+    """Send WhatsApp alerts for new failures. Max 1 alert per 30 min."""
+    if not new_failures:
+        return
+
+    # Global alert cooldown — max 1 WhatsApp alert per 30 minutes
+    cooldown_file = COORD_DIR / "immune-last-alert.txt"
+    if cooldown_file.exists():
+        try:
+            last_alert = datetime.strptime(cooldown_file.read_text().strip(), "%Y-%m-%d %H:%M")
+            minutes_since = (NOW - last_alert).total_seconds() / 60
+            if minutes_since < 30:
+                print(f"  [COOLDOWN] Last alert {minutes_since:.0f}min ago — suppressing")
+                return
+        except Exception:
+            pass
+
+    fails = [f for f in new_failures if f["status"] == "FAIL"]
+    warns = [f for f in new_failures if f["status"] == "WARN"]
+
+    sent = False
+    if fails:
+        lines = [f"- {f['name']}: {f['detail']}" for f in fails[:5]]
+        msg = "\n".join(lines)
+        if len(fails) > 5:
+            msg += f"\n... +{len(fails) - 5} more"
+        sent = send_alert(
+            "NEXO Immune FAIL",
+            f"{len(fails)} new failure(s):\n{msg}"
+        )
+
+    if warns and not fails:
+        lines = [f"- {f['name']}: {f['detail']}" for f in warns[:3]]
+        msg = "\n".join(lines)
+        sent = send_alert(
+            "NEXO Immune WARN",
+            f"{len(warns)} new warning(s):\n{msg}"
+        )
+
+    if sent:
+        cooldown_file.write_text(NOW.strftime("%Y-%m-%d %H:%M"))
+
+
+# ─── Main ─────────────────────────────────────────────────────────────────────
+
+def main():
+    print(f"\n{'='*60}")
+    print(f"NEXO Immune System — {NOW.strftime('%Y-%m-%d %H:%M:%S')}")
+    print(f"{'='*60}")
+
+    # Skip hours gate
+    if is_skip_hours():
+        print(f"[SKIP] Hour {NOW.hour} is within skip range ({SKIP_START}:00-{SKIP_END}:00). Exiting.")
+        return
+
+    # Ensure coordination directory exists
+    COORD_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Process lock (fcntl)
+    lock_fd = None
+    try:
+        lock_fd = open(LOCK_FILE, "w")
+        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except (IOError, OSError):
+        print("[LOCKED] Another immune instance is running. Exiting.")
+        if lock_fd:
+            lock_fd.close()
+        return
+
+    try:
+        _run_checks(lock_fd)
+    finally:
+        try:
+            fcntl.flock(lock_fd, fcntl.LOCK_UN)
+            lock_fd.close()
+        except Exception:
+            pass
+
+
+def _run_checks(lock_fd):
+    """Execute all checks and produce report."""
+    previous_status = load_json(IMMUNE_STATUS)
+
+    all_results = {}
+    repairs = []
+
+    # 1. Tokens
+    print("\n[1/7] Checking tokens...")
+    all_results["tokens"] = check_tokens()
+    for r in all_results["tokens"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+
+    # 2. LaunchAgents
+    print("\n[2/7] Checking LaunchAgents...")
+    all_results["agents"] = check_launch_agents()
+    for r in all_results["agents"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+        if r.get("repaired"):
+            repairs.append(f"LaunchAgent {r['name']} reloaded")
+
+    # 3. Databases
+    print("\n[3/7] Checking databases...")
+    all_results["databases"] = check_databases()
+    for r in all_results["databases"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+
+    # 4. Scripts & locks
+    print("\n[4/7] Checking scripts & locks...")
+    all_results["scripts"] = check_scripts()
+    for r in all_results["scripts"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+        if r.get("repaired"):
+            repairs.append(f"Stale lock {r['name']} removed")
+
+    # 5. Logs
+    print("\n[5/7] Checking log sizes...")
+    all_results["logs"] = check_logs()
+    for r in all_results["logs"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+        if r.get("repaired"):
+            repairs.append(f"Log {r['name']} truncated")
+
+    # 6. Disk
+    print("\n[6/7] Checking disk usage...")
+    all_results["disk"] = check_disk()
+    for r in all_results["disk"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+
+    # 7. Server crons
+    print("\n[7/7] Checking server crons...")
+    server_results, ssh_ran = check_server_crons()
+    all_results["server"] = server_results
+    for r in all_results["server"]:
+        icon = "OK" if r["status"] == "OK" else r["status"]
+        print(f"  [{icon}] {r['name']}: {r['detail']}")
+
+    # ─── Summary ──────────────────────────────────────────────────────────
+    counts = {"OK": 0, "WARN": 0, "FAIL": 0}
+    for category_items in all_results.values():
+        for item in category_items:
+            s = item.get("status", "OK")
+            if s in counts:
+                counts[s] += 1
+
+    total = sum(counts.values())
+
+    print(f"\n{'─'*60}")
+    print(f"SUMMARY: {total} checks — {counts['OK']} OK, {counts['WARN']} WARN, {counts['FAIL']} FAIL")
+    if repairs:
+        print(f"AUTO-REPAIRS: {len(repairs)}")
+        for r in repairs:
+            print(f"  - {r}")
+    print(f"{'─'*60}\n")
+
+    # ─── Detect new failures & alert ──────────────────────────────────────
+    new_failures = detect_new_failures(all_results, previous_status)
+    if new_failures:
+        print(f"[ALERT] {len(new_failures)} new failure(s)/warning(s) detected:")
+        for nf in new_failures:
+            print(f"  - [{nf['status']}] {nf['name']}: {nf['detail']}")
+        send_failure_alerts(new_failures)
+    else:
+        print("[OK] No new failures.")
+
+    # ─── Save status ──────────────────────────────────────────────────────
+    status = {
+        "last_run": NOW.strftime("%Y-%m-%d %H:%M"),
+        "counts": counts,
+        "repairs": repairs,
+        "new_failures": len(new_failures),
+        "checks": all_results,
+    }
+    if ssh_ran:
+        status["last_ssh_check"] = NOW.strftime("%Y-%m-%d %H:%M")
+    elif "last_ssh_check" in previous_status:
+        status["last_ssh_check"] = previous_status["last_ssh_check"]
+
+    save_json(IMMUNE_STATUS, status)
+
+    # ─── Append to log ────────────────────────────────────────────────────
+    log_entry = {
+        "ts": NOW.strftime("%Y-%m-%d %H:%M"),
+        "ok": counts["OK"],
+        "warn": counts["WARN"],
+        "fail": counts["FAIL"],
+        "repairs": len(repairs),
+        "new_failures": len(new_failures),
+    }
+
+    log = load_json(IMMUNE_LOG, default=[])
+    if not isinstance(log, list):
+        log = []
+    log.append(log_entry)
+    if len(log) > MAX_LOG_ENTRIES:
+        log = log[-MAX_LOG_ENTRIES:]
+    save_json(IMMUNE_LOG, log)
+
+    print(f"Status saved to {IMMUNE_STATUS}")
+    print(f"Log appended to {IMMUNE_LOG} ({len(log)} entries)")
+
+    # ─── Stage B: CLI interpretation (only when issues found) ────────────
+    if counts["FAIL"] > 0 or counts["WARN"] > 2 or repairs:
+        _run_cli_triage(all_results, repairs, counts)
+
+
+def _run_cli_triage(all_results: dict, repairs: list, counts: dict):
+    """Pass all findings to the configured automation backend for intelligent triage and recommendations."""
+    triage_file = COORD_DIR / "immune-triage.md"
+    findings_json = json.dumps({
+        "timestamp": NOW.strftime("%Y-%m-%d %H:%M"),
+        "counts": counts,
+        "repairs": repairs,
+        "checks": all_results,
+    }, indent=2, default=str)
+
+    prompt = f"""You are the NEXO Immune System triage analyst.
+
+Below are the raw health check results from a scheduled scan. Your job:
+
+1. Identify which failures are REAL problems vs transient/expected
+2. Group related issues (e.g. SSH failure + server cron failure = same root cause)
+3. Prioritize: what needs attention NOW vs can wait
+4. For each real issue, suggest a specific remediation action
+5. Note any patterns across recent runs if visible
+
+Write a concise triage report to: {triage_file}
+
+Format:
+## Immune Triage — YYYY-MM-DD HH:MM
+
+### Critical (act now)
+- ...
+
+### Monitor (watch next run)
+- ...
+
+### Resolved (auto-repaired)
+- ...
+
+### Patterns
+- ...
+
+Raw findings:
+{findings_json}
+
+Write the report. Be concise — max 40 lines."""
+
+    print("\n[TRIAGE] Running CLI interpretation...")
+    try:
+        result = run_automation_prompt(
+            prompt,
+            model=_USER_MODEL or "opus",
+            timeout=21600,
+            output_format="text",
+            allowed_tools="Read,Write,Edit,Glob,Grep,Bash,mcp__nexo__*",
+        )
+        if result.returncode == 0:
+            print(f"[TRIAGE] Report written to {triage_file}")
+        else:
+            print(f"[TRIAGE] CLI exited {result.returncode}: {result.stderr[:200]}")
+    except AutomationBackendUnavailableError as e:
+        print(f"[TRIAGE] Skipping triage: {e}")
+    except subprocess.TimeoutExpired:
+        print("[TRIAGE] CLI timed out (120s)")
+    except Exception as e:
+        print(f"[TRIAGE] Error: {e}")
+
+
+if __name__ == "__main__":
+    main()