nexo-brain 2.4.0 → 2.5.0

package/src/doctor/__init__.py

@@ -0,0 +1 @@
+"""NEXO Doctor — unified modular diagnostic system."""

package/src/doctor/formatters.py

@@ -0,0 +1,52 @@
+"""Doctor output formatters — text and JSON."""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+
+from doctor.models import DoctorReport
+
+
+def format_report(report: DoctorReport, fmt: str = "text") -> str:
+    """Format a DoctorReport as text or JSON."""
+    if fmt == "json":
+        return json.dumps(asdict(report), indent=2, ensure_ascii=False)
+    return _format_text(report)
+
+
+def _format_text(report: DoctorReport) -> str:
+    """Human-friendly text output."""
+    lines = []
+
+    # Header
+    icon = {"healthy": "✓", "degraded": "⚠", "critical": "✗"}.get(report.overall_status, "?")
+    lines.append(f"NEXO Doctor — {icon} {report.overall_status.upper()}")
+    lines.append(f"  {report.counts.get('healthy', 0)} healthy, "
+                 f"{report.counts.get('degraded', 0)} degraded, "
+                 f"{report.counts.get('critical', 0)} critical "
+                 f"({report.duration_ms}ms)")
+    lines.append("")
+
+    # Group by tier
+    current_tier = None
+    for check in report.checks:
+        if check.tier != current_tier:
+            current_tier = check.tier
+            lines.append(f"── {current_tier.upper()} ──")
+
+        icon = {"healthy": "✓", "degraded": "⚠", "critical": "✗"}.get(check.status, "?")
+        fixed = " [FIXED]" if check.fixed else ""
+        lines.append(f"  {icon} {check.summary}{fixed}")
+
+        if check.status != "healthy":
+            for ev in check.evidence:
+                lines.append(f"    → {ev}")
+            if check.repair_plan:
+                lines.append("    Fix:")
+                for step in check.repair_plan:
+                    lines.append(f"      • {step}")
+            if check.escalation_prompt:
+                lines.append(f"    Escalation: {check.escalation_prompt}")
+
+    lines.append("")
+    return "\n".join(lines)

package/src/doctor/models.py

@@ -0,0 +1,44 @@
+"""Doctor data models — check results and report structure."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DoctorCheck:
+    id: str
+    tier: str
+    status: str  # healthy, degraded, critical
+    severity: str  # info, warn, error
+    summary: str
+    evidence: list[str] = field(default_factory=list)
+    repair_plan: list[str] = field(default_factory=list)
+    escalation_prompt: str = ""
+    fixed: bool = False
+
+
+@dataclass
+class DoctorReport:
+    overall_status: str  # healthy, degraded, critical
+    counts: dict = field(default_factory=dict)
+    checks: list[DoctorCheck] = field(default_factory=list)
+    duration_ms: int = 0
+
+    def add(self, check: DoctorCheck):
+        self.checks.append(check)
+
+    def compute_status(self):
+        """Compute overall status from individual checks."""
+        statuses = [c.status for c in self.checks]
+        if "critical" in statuses:
+            self.overall_status = "critical"
+        elif "degraded" in statuses:
+            self.overall_status = "degraded"
+        else:
+            self.overall_status = "healthy"
+        self.counts = {
+            "healthy": statuses.count("healthy"),
+            "degraded": statuses.count("degraded"),
+            "critical": statuses.count("critical"),
+            "total": len(statuses),
+        }

package/src/doctor/orchestrator.py

@@ -0,0 +1,42 @@
+"""Doctor orchestrator — runs providers by tier, aggregates results."""
+from __future__ import annotations
+
+import time
+
+from doctor.models import DoctorReport
+from doctor.providers.boot import run_boot_checks
+from doctor.providers.runtime import run_runtime_checks
+from doctor.providers.deep import run_deep_checks
+
+
+_TIER_RUNNERS = {
+    "boot": run_boot_checks,
+    "runtime": run_runtime_checks,
+    "deep": run_deep_checks,
+}
+
+_TIER_ORDER = ["boot", "runtime", "deep"]
+
+
+def run_doctor(tier: str = "boot", fix: bool = False) -> DoctorReport:
+    """Run diagnostic checks for the specified tier(s).
+
+    Args:
+        tier: "boot", "runtime", "deep", or "all"
+        fix: If True, apply deterministic fixes where possible
+    """
+    report = DoctorReport(overall_status="healthy")
+    start = time.monotonic()
+
+    tiers = _TIER_ORDER if tier == "all" else [tier]
+
+    for t in tiers:
+        runner = _TIER_RUNNERS.get(t)
+        if runner:
+            checks = runner(fix=fix)
+            for check in checks:
+                report.add(check)
+
+    report.compute_status()
+    report.duration_ms = int((time.monotonic() - start) * 1000)
+    return report

package/src/doctor/providers/__init__.py

@@ -0,0 +1 @@
+"""Doctor check providers — boot, runtime, deep."""

package/src/doctor/providers/boot.py

@@ -0,0 +1,206 @@
+"""Boot tier checks — fast, native, no repair. Target <100ms."""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from doctor.models import DoctorCheck
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+
+
+def check_db_exists() -> DoctorCheck:
+    """Check that the main database file exists and is readable."""
+    db_path = NEXO_HOME / "data" / "nexo.db"
+    if db_path.is_file():
+        size_kb = db_path.stat().st_size / 1024
+        return DoctorCheck(
+            id="boot.db_exists",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary=f"Database exists ({size_kb:.0f} KB)",
+            evidence=[str(db_path)],
+        )
+    return DoctorCheck(
+        id="boot.db_exists",
+        tier="boot",
+        status="critical",
+        severity="error",
+        summary="Database file not found",
+        evidence=[f"Expected: {db_path}"],
+        repair_plan=["Run nexo-brain to initialize the database"],
+        escalation_prompt="NEXO database missing — server cannot start without it.",
+    )
+
+
+def check_required_dirs() -> DoctorCheck:
+    """Check that required NEXO_HOME directories exist."""
+    required = ["data", "scripts", "plugins", "crons", "hooks", "coordination", "operations", "logs"]
+    missing = [d for d in required if not (NEXO_HOME / d).is_dir()]
+
+    if not missing:
+        return DoctorCheck(
+            id="boot.required_dirs",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary=f"All {len(required)} required directories present",
+        )
+
+    return DoctorCheck(
+        id="boot.required_dirs",
+        tier="boot",
+        status="degraded" if len(missing) < 3 else "critical",
+        severity="warn" if len(missing) < 3 else "error",
+        summary=f"{len(missing)} required director{'y' if len(missing) == 1 else 'ies'} missing",
+        evidence=[f"Missing: {d}" for d in missing],
+        repair_plan=[f"mkdir -p {NEXO_HOME / d}" for d in missing],
+    )
+
+
+def check_disk_space() -> DoctorCheck:
+    """Check disk free space on NEXO_HOME partition."""
+    try:
+        usage = shutil.disk_usage(str(NEXO_HOME))
+        free_gb = usage.free / (1024 ** 3)
+        pct_free = (usage.free / usage.total) * 100
+
+        if free_gb < 1:
+            return DoctorCheck(
+                id="boot.disk_space",
+                tier="boot",
+                status="critical",
+                severity="error",
+                summary=f"Very low disk space: {free_gb:.1f} GB free ({pct_free:.0f}%)",
+                evidence=[f"Total: {usage.total / (1024**3):.0f} GB, Free: {free_gb:.1f} GB"],
+                repair_plan=["Free up disk space — NEXO needs at least 1 GB for normal operation"],
+                escalation_prompt="Disk space critically low — backups and logs may fail.",
+            )
+        elif free_gb < 5:
+            return DoctorCheck(
+                id="boot.disk_space",
+                tier="boot",
+                status="degraded",
+                severity="warn",
+                summary=f"Low disk space: {free_gb:.1f} GB free ({pct_free:.0f}%)",
+                evidence=[f"Total: {usage.total / (1024**3):.0f} GB, Free: {free_gb:.1f} GB"],
+            )
+        return DoctorCheck(
+            id="boot.disk_space",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary=f"Disk space OK: {free_gb:.0f} GB free ({pct_free:.0f}%)",
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="boot.disk_space",
+            tier="boot",
+            status="degraded",
+            severity="warn",
+            summary=f"Could not check disk space: {e}",
+        )
+
+
+def check_wrapper_scripts() -> DoctorCheck:
+    """Check that cron wrapper script exists."""
+    wrapper = NEXO_HOME / "scripts" / "nexo-cron-wrapper.sh"
+    if wrapper.is_file():
+        return DoctorCheck(
+            id="boot.wrapper_scripts",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary="Cron wrapper script present",
+        )
+    return DoctorCheck(
+        id="boot.wrapper_scripts",
+        tier="boot",
+        status="degraded",
+        severity="warn",
+        summary="Cron wrapper script missing",
+        evidence=[f"Expected: {wrapper}"],
+        repair_plan=["Run nexo-brain to reinstall wrapper scripts"],
+    )
+
+
+def check_python_runtime() -> DoctorCheck:
+    """Check Python interpreter is suitable."""
+    version = sys.version_info
+    if version >= (3, 10):
+        return DoctorCheck(
+            id="boot.python_runtime",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary=f"Python {version.major}.{version.minor}.{version.micro}",
+        )
+    return DoctorCheck(
+        id="boot.python_runtime",
+        tier="boot",
+        status="degraded",
+        severity="warn",
+        summary=f"Python {version.major}.{version.minor} — 3.10+ recommended",
+        evidence=[sys.executable],
+    )
+
+
+def check_config_parse() -> DoctorCheck:
+    """Check schedule.json parses correctly if present."""
+    schedule_file = NEXO_HOME / "config" / "schedule.json"
+    if not schedule_file.exists():
+        return DoctorCheck(
+            id="boot.config_parse",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary="No schedule.json (using defaults)",
+        )
+    try:
+        import json
+        json.loads(schedule_file.read_text())
+        return DoctorCheck(
+            id="boot.config_parse",
+            tier="boot",
+            status="healthy",
+            severity="info",
+            summary="schedule.json parses OK",
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="boot.config_parse",
+            tier="boot",
+            status="degraded",
+            severity="warn",
+            summary=f"schedule.json parse error: {e}",
+            repair_plan=["Fix JSON syntax in schedule.json or delete to use defaults"],
+        )
+
+
+def run_boot_checks(fix: bool = False) -> list[DoctorCheck]:
+    """Run all boot-tier checks."""
+    checks = [
+        check_db_exists(),
+        check_required_dirs(),
+        check_disk_space(),
+        check_wrapper_scripts(),
+        check_python_runtime(),
+        check_config_parse(),
+    ]
+
+    if fix:
+        for check in checks:
+            if check.id == "boot.required_dirs" and check.status != "healthy":
+                # Deterministic fix: create missing directories
+                for plan in check.repair_plan:
+                    if plan.startswith("mkdir"):
+                        dir_path = plan.split("mkdir -p ")[-1]
+                        Path(dir_path).mkdir(parents=True, exist_ok=True)
+                check.fixed = True
+                check.status = "healthy"
+                check.summary += " (fixed)"
+
+    return checks

package/src/doctor/providers/deep.py

@@ -0,0 +1,292 @@
+"""Deep tier checks — read existing artifacts for richer validation. Target <60s."""
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+
+from doctor.models import DoctorCheck
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+
+# Freshness thresholds
+SELF_AUDIT_FRESHNESS = 86400 * 2  # 2 days (runs daily)
+PREFLIGHT_FRESHNESS = 86400  # 1 day
+WATCHDOG_SMOKE_FRESHNESS = 86400  # 1 day
+
+
+def _file_age_seconds(path: Path) -> float | None:
+    try:
+        if path.is_file():
+            return time.time() - path.stat().st_mtime
+    except Exception:
+        pass
+    return None
+
+
+def _load_json(path: Path) -> dict:
+    return json.loads(path.read_text())
+
+
+def check_self_audit_summary() -> DoctorCheck:
+    """Check latest self-audit summary exists and is recent."""
+    summary_file = NEXO_HOME / "logs" / "self-audit-summary.json"
+    age = _file_age_seconds(summary_file)
+
+    if age is None:
+        return DoctorCheck(
+            id="deep.self_audit",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary="Self-audit summary not found",
+            evidence=[f"Expected: {summary_file}"],
+            repair_plan=["Check if daily self-audit cron is installed"],
+        )
+
+    age_hours = age / 3600
+    if age > SELF_AUDIT_FRESHNESS:
+        return DoctorCheck(
+            id="deep.self_audit",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Self-audit summary stale ({age_hours:.0f}h old)",
+            evidence=[f"Last modified {age_hours:.0f} hours ago, threshold {SELF_AUDIT_FRESHNESS // 3600}h"],
+        )
+
+    try:
+        data = _load_json(summary_file)
+        counts = data.get("counts") or {}
+        error_count = int(counts.get("error", 0) or 0)
+        warn_count = int(counts.get("warn", 0) or 0)
+        findings = data.get("findings") or []
+        if error_count > 0:
+            status = "critical"
+            severity = "error"
+        elif warn_count > 0:
+            status = "degraded"
+            severity = "warn"
+        else:
+            status = "healthy"
+            severity = "info"
+        return DoctorCheck(
+            id="deep.self_audit",
+            tier="deep",
+            status=status,
+            severity=severity,
+            summary=(
+                f"Self-audit: {len(findings)} findings "
+                f"({error_count} error, {warn_count} warn; {age_hours:.0f}h ago)"
+            ),
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="deep.self_audit",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Self-audit summary unreadable ({age_hours:.0f}h ago)",
+            evidence=[str(e)],
+        )
+
+
+def check_schema_version() -> DoctorCheck:
+    """Check DB schema version is present and reasonable."""
+    try:
+        import sqlite3
+        db_path = NEXO_HOME / "data" / "nexo.db"
+        if not db_path.is_file():
+            return DoctorCheck(
+                id="deep.schema_version",
+                tier="deep",
+                status="degraded",
+                severity="warn",
+                summary="No database to check schema",
+            )
+        conn = sqlite3.connect(str(db_path), timeout=2)
+        version = conn.execute("PRAGMA user_version").fetchone()[0]
+        conn.close()
+        return DoctorCheck(
+            id="deep.schema_version",
+            tier="deep",
+            status="healthy",
+            severity="info",
+            summary=f"DB schema version: {version}",
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="deep.schema_version",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Schema check failed: {e}",
+        )
+
+
+def check_preflight_summary() -> DoctorCheck:
+    """Check runtime preflight summary."""
+    summary_file = NEXO_HOME / "logs" / "runtime-preflight-summary.json"
+    age = _file_age_seconds(summary_file)
+
+    if age is None:
+        return DoctorCheck(
+            id="deep.preflight",
+            tier="deep",
+            status="healthy",
+            severity="info",
+            summary="No preflight summary (optional)",
+        )
+
+    age_hours = age / 3600
+    if age > PREFLIGHT_FRESHNESS:
+        return DoctorCheck(
+            id="deep.preflight",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Preflight summary stale ({age_hours:.0f}h old)",
+        )
+    try:
+        data = _load_json(summary_file)
+        ok = data.get("ok")
+        checks = data.get("checks") or {}
+        errors = data.get("errors") or []
+        if ok is True:
+            return DoctorCheck(
+                id="deep.preflight",
+                tier="deep",
+                status="healthy",
+                severity="info",
+                summary=f"Runtime preflight OK ({len(checks)} checks, {age_hours:.0f}h ago)",
+            )
+        return DoctorCheck(
+            id="deep.preflight",
+            tier="deep",
+            status="critical",
+            severity="error",
+            summary=f"Runtime preflight failed ({len(errors)} errors, {age_hours:.0f}h ago)",
+            evidence=errors[:5],
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="deep.preflight",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Preflight summary unreadable ({age_hours:.0f}h ago)",
+            evidence=[str(e)],
+        )
+
+
+def check_watchdog_smoke() -> DoctorCheck:
+    """Check watchdog smoke summary."""
+    summary_file = NEXO_HOME / "logs" / "watchdog-smoke-summary.json"
+    age = _file_age_seconds(summary_file)
+
+    if age is None:
+        return DoctorCheck(
+            id="deep.watchdog_smoke",
+            tier="deep",
+            status="healthy",
+            severity="info",
+            summary="No watchdog smoke summary (optional)",
+        )
+
+    age_hours = age / 3600
+    if age > WATCHDOG_SMOKE_FRESHNESS:
+        return DoctorCheck(
+            id="deep.watchdog_smoke",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Watchdog smoke summary stale ({age_hours:.0f}h old)",
+        )
+
+    try:
+        data = _load_json(summary_file)
+        ok = data.get("ok")
+        findings = data.get("findings") or []
+        error_count = sum(1 for finding in findings if finding.get("severity") == "ERROR")
+        if ok is True:
+            return DoctorCheck(
+                id="deep.watchdog_smoke",
+                tier="deep",
+                status="healthy",
+                severity="info",
+                summary=f"Watchdog smoke OK ({len(findings)} findings, {age_hours:.0f}h ago)",
+            )
+        return DoctorCheck(
+            id="deep.watchdog_smoke",
+            tier="deep",
+            status="critical",
+            severity="error",
+            summary=f"Watchdog smoke failed ({error_count} errors, {age_hours:.0f}h ago)",
+            evidence=[finding.get("msg", "") for finding in findings[:5]],
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="deep.watchdog_smoke",
+            tier="deep",
+            status="degraded",
+            severity="warn",
+            summary=f"Watchdog smoke summary unreadable ({age_hours:.0f}h ago)",
+            evidence=[str(e)],
+        )
+
+
+def check_learning_count() -> DoctorCheck:
+    """Check learning count as a health proxy."""
+    try:
+        import sqlite3
+        db_path = NEXO_HOME / "data" / "nexo.db"
+        if not db_path.is_file():
+            return DoctorCheck(
+                id="deep.learning_count",
+                tier="deep",
+                status="healthy",
+                severity="info",
+                summary="No DB to check learnings",
+            )
+        conn = sqlite3.connect(str(db_path), timeout=2)
+        tables = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
+        ).fetchone()
+        if not tables:
+            conn.close()
+            return DoctorCheck(
+                id="deep.learning_count",
+                tier="deep",
+                status="healthy",
+                severity="info",
+                summary="No learnings table yet",
+            )
+        count = conn.execute("SELECT COUNT(*) FROM learnings WHERE archived=0").fetchone()[0]
+        conn.close()
+        return DoctorCheck(
+            id="deep.learning_count",
+            tier="deep",
+            status="healthy",
+            severity="info",
+            summary=f"{count} active learnings in memory",
+        )
+    except Exception as e:
+        return DoctorCheck(
+            id="deep.learning_count",
+            tier="deep",
+            status="healthy",
+            severity="info",
+            summary=f"Learning check skipped: {e}",
+        )
+
+
+def run_deep_checks(fix: bool = False) -> list[DoctorCheck]:
+    """Run all deep-tier checks. Read-only."""
+    return [
+        check_self_audit_summary(),
+        check_schema_version(),
+        check_preflight_summary(),
+        check_watchdog_smoke(),
+        check_learning_count(),
+    ]