nexo-brain 3.0.1 → 3.1.0

package/src/doctor/orchestrator.py

@@ -1,9 +1,11 @@
 """Doctor orchestrator — runs providers by tier, aggregates results."""
 from __future__ import annotations
 
+import sys
 import time
+import traceback
 
-from doctor.models import DoctorReport
+from doctor.models import DoctorCheck, DoctorReport
 from doctor.providers.boot import run_boot_checks
 from doctor.providers.runtime import run_runtime_checks
 from doctor.providers.deep import run_deep_checks
@@ -17,6 +19,8 @@ _TIER_RUNNERS = {
 
 _TIER_ORDER = ["boot", "runtime", "deep"]
 
+VALID_TIERS = frozenset(_TIER_ORDER) | {"all"}
+
 
 def run_doctor(tier: str = "boot", fix: bool = False) -> DoctorReport:
     """Run diagnostic checks for the specified tier(s).
@@ -28,14 +32,40 @@ def run_doctor(tier: str = "boot", fix: bool = False) -> DoctorReport:
     report = DoctorReport(overall_status="healthy")
     start = time.monotonic()
 
+    if tier not in VALID_TIERS:
+        report.add(DoctorCheck(
+            id="orchestrator.invalid_tier",
+            tier="orchestrator",
+            status="critical",
+            severity="error",
+            summary=f"Unknown tier '{tier}' — valid options: {', '.join(sorted(VALID_TIERS))}",
+        ))
+        report.compute_status()
+        report.duration_ms = int((time.monotonic() - start) * 1000)
+        return report
+
     tiers = _TIER_ORDER if tier == "all" else [tier]
 
     for t in tiers:
         runner = _TIER_RUNNERS.get(t)
-        if runner:
+        if not runner:
+            continue
+        try:
             checks = runner(fix=fix)
             for check in checks:
                 report.add(check)
+        except Exception as exc:
+            tb = traceback.format_exception(type(exc), exc, exc.__traceback__)
+            last_frame = tb[-1].strip() if tb else str(exc)
+            report.add(DoctorCheck(
+                id=f"orchestrator.{t}_crashed",
+                tier=t,
+                status="critical",
+                severity="error",
+                summary=f"{t} tier checks crashed: {type(exc).__name__}: {exc}",
+                evidence=[last_frame],
+                repair_plan=[f"Investigate {t} provider — exception during check execution"],
+            ))
 
     report.compute_status()
     report.duration_ms = int((time.monotonic() - start) * 1000)

package/src/doctor/providers/boot.py

@@ -6,7 +6,7 @@ import shutil
 import sys
 from pathlib import Path
 
-from doctor.models import DoctorCheck
+from doctor.models import DoctorCheck, safe_check
 
 NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
 
@@ -209,12 +209,12 @@ def check_config_parse() -> DoctorCheck:
 def run_boot_checks(fix: bool = False) -> list[DoctorCheck]:
     """Run all boot-tier checks."""
     checks = [
-        check_db_exists(),
-        check_required_dirs(),
-        check_disk_space(),
-        check_wrapper_scripts(),
-        check_python_runtime(),
-        check_config_parse(),
+        safe_check(check_db_exists),
+        safe_check(check_required_dirs),
+        safe_check(check_disk_space),
+        safe_check(check_wrapper_scripts),
+        safe_check(check_python_runtime),
+        safe_check(check_config_parse),
     ]
 
     if fix:

package/src/doctor/providers/deep.py

@@ -6,7 +6,7 @@ import os
 import time
 from pathlib import Path
 
-from doctor.models import DoctorCheck
+from doctor.models import DoctorCheck, safe_check
 
 NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
 
@@ -106,8 +106,10 @@ def check_schema_version() -> DoctorCheck:
                 summary="No database to check schema",
             )
         conn = sqlite3.connect(str(db_path), timeout=2)
-        version = conn.execute("PRAGMA user_version").fetchone()[0]
-        conn.close()
+        try:
+            version = conn.execute("PRAGMA user_version").fetchone()[0]
+        finally:
+            conn.close()
         return DoctorCheck(
             id="deep.schema_version",
             tier="deep",
@@ -250,20 +252,21 @@ def check_learning_count() -> DoctorCheck:
                 summary="No DB to check learnings",
             )
         conn = sqlite3.connect(str(db_path), timeout=2)
-        tables = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
-        ).fetchone()
-        if not tables:
+        try:
+            tables = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
+            ).fetchone()
+            if not tables:
+                return DoctorCheck(
+                    id="deep.learning_count",
+                    tier="deep",
+                    status="healthy",
+                    severity="info",
+                    summary="No learnings table yet",
+                )
+            count = conn.execute("SELECT COUNT(*) FROM learnings WHERE archived=0").fetchone()[0]
+        finally:
             conn.close()
-            return DoctorCheck(
-                id="deep.learning_count",
-                tier="deep",
-                status="healthy",
-                severity="info",
-                summary="No learnings table yet",
-            )
-        count = conn.execute("SELECT COUNT(*) FROM learnings WHERE archived=0").fetchone()[0]
-        conn.close()
         return DoctorCheck(
             id="deep.learning_count",
             tier="deep",
@@ -284,9 +287,9 @@ def check_learning_count() -> DoctorCheck:
 def run_deep_checks(fix: bool = False) -> list[DoctorCheck]:
     """Run all deep-tier checks. Read-only."""
     return [
-        check_self_audit_summary(),
-        check_schema_version(),
-        check_preflight_summary(),
-        check_watchdog_smoke(),
-        check_learning_count(),
+        safe_check(check_self_audit_summary),
+        safe_check(check_schema_version),
+        safe_check(check_preflight_summary),
+        safe_check(check_watchdog_smoke),
+        safe_check(check_learning_count),
     ]

package/src/doctor/providers/runtime.py

@@ -25,7 +25,7 @@ from client_preferences import (
     resolve_client_runtime_profile,
 )
 from cron_recovery import should_run_at_load
-from doctor.models import DoctorCheck
+from doctor.models import DoctorCheck, safe_check
 
 NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
 NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parents[2])))
@@ -406,21 +406,22 @@ def _load_active_conditioned_learnings() -> list[dict]:
         import sqlite3
 
         conn = sqlite3.connect(str(db_path), timeout=2)
-        conn.row_factory = sqlite3.Row
-        table = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
-        ).fetchone()
-        if not table:
+        try:
+            conn.row_factory = sqlite3.Row
+            table = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
+            ).fetchone()
+            if not table:
+                return []
+            rows = conn.execute(
+                """SELECT id, title, applies_to
+                   FROM learnings
+                   WHERE status = 'active' AND COALESCE(applies_to, '') != ''
+                   ORDER BY updated_at DESC, id DESC"""
+            ).fetchall()
+            return [dict(row) for row in rows]
+        finally:
             conn.close()
-            return []
-        rows = conn.execute(
-            """SELECT id, title, applies_to
-               FROM learnings
-               WHERE status = 'active' AND COALESCE(applies_to, '') != ''
-               ORDER BY updated_at DESC, id DESC"""
-        ).fetchall()
-        conn.close()
-        return [dict(row) for row in rows]
     except Exception:
         return []
 
@@ -595,22 +596,23 @@ def _open_protocol_debt_summary(*debt_types: str) -> dict:
 
     try:
         conn = sqlite3.connect(str(db_path), timeout=2)
-        conn.row_factory = sqlite3.Row
-        table = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='protocol_debt'"
-        ).fetchone()
-        if not table:
+        try:
+            conn.row_factory = sqlite3.Row
+            table = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='protocol_debt'"
+            ).fetchone()
+            if not table:
+                return summary
+            placeholders = ",".join("?" for _ in debt_types)
+            rows = conn.execute(
+                f"""SELECT debt_type, COUNT(*) AS total
+                    FROM protocol_debt
+                    WHERE status = 'open' AND debt_type IN ({placeholders})
+                    GROUP BY debt_type""",
+                tuple(debt_types),
+            ).fetchall()
+        finally:
             conn.close()
-            return summary
-        placeholders = ",".join("?" for _ in debt_types)
-        rows = conn.execute(
-            f"""SELECT debt_type, COUNT(*) AS total
-                FROM protocol_debt
-                WHERE status = 'open' AND debt_type IN ({placeholders})
-                GROUP BY debt_type""",
-            tuple(debt_types),
-        ).fetchall()
-        conn.close()
     except Exception:
         return summary
 
@@ -625,11 +627,12 @@ def _client_assumption_regressions() -> list[str]:
     src_root = NEXO_CODE if (NEXO_CODE / "server.py").is_file() else (NEXO_CODE / "src")
     if not src_root.is_dir():
         return []
+    src_root = src_root.resolve()
     backup_root = (NEXO_HOME / "backups").resolve()
     contrib_root = (NEXO_HOME / "contrib").resolve()
-    allowed_claude_projects = {
-        (src_root / "scripts" / "deep-sleep" / "collect.py").resolve(),
-        Path(__file__).resolve(),
+    allowed_relative_paths = {
+        Path("scripts") / "deep-sleep" / "collect.py",
+        Path("doctor") / "providers" / "runtime.py",
     }
     offenders: list[str] = []
     for path in src_root.rglob("*.py"):
@@ -648,8 +651,12 @@ def _client_assumption_regressions() -> list[str]:
                 continue
         except Exception:
             pass
-        if ".claude/projects" in text and resolved not in allowed_claude_projects:
-            offenders.append(f"{path.relative_to(NEXO_CODE)} hardcodes ~/.claude/projects")
+        try:
+            relative_path = resolved.relative_to(src_root)
+        except Exception:
+            relative_path = path.relative_to(src_root)
+        if ".claude/projects" in text and relative_path not in allowed_relative_paths:
+            offenders.append(f"{relative_path} hardcodes ~/.claude/projects")
     collect_path = src_root / "scripts" / "deep-sleep" / "collect.py"
     try:
         collect_text = collect_path.read_text()
@@ -1172,14 +1179,16 @@ def check_stale_sessions() -> DoctorCheck:
                 summary="No DB to check sessions",
             )
         conn = sqlite3.connect(str(db_path), timeout=2)
-        conn.row_factory = sqlite3.Row
-        cutoff = time.time() - 7200
-        day_ago = time.time() - 86400
-        rows = conn.execute(
-            "SELECT COUNT(*) as cnt FROM sessions WHERE last_update_epoch < ? AND last_update_epoch > ?",
-            (cutoff, day_ago),
-        ).fetchone()
-        conn.close()
+        try:
+            conn.row_factory = sqlite3.Row
+            cutoff = time.time() - 7200
+            day_ago = time.time() - 86400
+            rows = conn.execute(
+                "SELECT COUNT(*) as cnt FROM sessions WHERE last_update_epoch < ? AND last_update_epoch > ?",
+                (cutoff, day_ago),
+            ).fetchone()
+        finally:
+            conn.close()
         count = rows["cnt"] if rows else 0
         if count > 0:
             return DoctorCheck(
@@ -1221,24 +1230,25 @@ def check_cron_freshness() -> DoctorCheck:
                 summary="No DB to check cron runs",
             )
         conn = sqlite3.connect(str(db_path), timeout=2)
-        # Check if cron_runs table exists
-        tables = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='cron_runs'"
-        ).fetchone()
-        if not tables:
+        try:
+            # Check if cron_runs table exists
+            tables = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='cron_runs'"
+            ).fetchone()
+            if not tables:
+                return DoctorCheck(
+                    id="runtime.cron_freshness",
+                    tier="runtime",
+                    status="healthy",
+                    severity="info",
+                    summary="No cron_runs table yet",
+                )
+            # Latest run per cron
+            rows = conn.execute(
+                "SELECT cron_id, MAX(started_at) as last_run FROM cron_runs GROUP BY cron_id"
+            ).fetchall()
+        finally:
             conn.close()
-            return DoctorCheck(
-                id="runtime.cron_freshness",
-                tier="runtime",
-                status="healthy",
-                severity="info",
-                summary="No cron_runs table yet",
-            )
-        # Latest run per cron
-        rows = conn.execute(
-            "SELECT cron_id, MAX(started_at) as last_run FROM cron_runs GROUP BY cron_id"
-        ).fetchall()
-        conn.close()
 
         stale = []
         expectations = _cron_expectations()
@@ -2167,32 +2177,36 @@ def check_protocol_compliance() -> DoctorCheck:
         db_path = NEXO_HOME / "data" / "nexo.db"
         if db_path.is_file():
             conn = sqlite3.connect(str(db_path), timeout=2)
-            conn.row_factory = sqlite3.Row
-            tables = {
-                row["name"]
-                for row in conn.execute(
-                    "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('protocol_tasks', 'protocol_debt')"
-                ).fetchall()
-            }
-            if {"protocol_tasks", "protocol_debt"}.issubset(tables):
-                window = "-7 days"
-                tasks = conn.execute(
-                    """SELECT * FROM protocol_tasks
-                       WHERE opened_at >= datetime('now', ?)
-                       ORDER BY opened_at DESC""",
-                    (window,),
-                ).fetchall()
-                debt_rows = conn.execute(
-                    """SELECT severity, debt_type, COUNT(*) AS total
-                       FROM protocol_debt
-                       WHERE status = 'open' AND created_at >= datetime('now', ?)
-                       GROUP BY severity, debt_type
-                       ORDER BY total DESC, debt_type ASC""",
-                    (window,),
-                ).fetchall()
+            try:
+                conn.row_factory = sqlite3.Row
+                tables = {
+                    row["name"]
+                    for row in conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('protocol_tasks', 'protocol_debt')"
+                    ).fetchall()
+                }
+                tasks = None
+                debt_rows = None
+                if {"protocol_tasks", "protocol_debt"}.issubset(tables):
+                    window = "-7 days"
+                    tasks = conn.execute(
+                        """SELECT * FROM protocol_tasks
+                           WHERE opened_at >= datetime('now', ?)
+                           ORDER BY opened_at DESC""",
+                        (window,),
+                    ).fetchall()
+                    debt_rows = conn.execute(
+                        """SELECT severity, debt_type, COUNT(*) AS total
+                           FROM protocol_debt
+                           WHERE status = 'open' AND created_at >= datetime('now', ?)
+                           GROUP BY severity, debt_type
+                           ORDER BY total DESC, debt_type ASC""",
+                        (window,),
+                    ).fetchall()
+            finally:
                 conn.close()
 
-                if tasks or debt_rows:
+            if tasks is not None and debt_rows is not None and (tasks or debt_rows):
                     closed_tasks = [row for row in tasks if row["status"] != "open"]
                     verify_required = [row for row in closed_tasks if row["must_verify"] and row["status"] == "done"]
                     verify_ok = [row for row in verify_required if (row["close_evidence"] or "").strip()]
@@ -2410,11 +2424,13 @@ def check_state_watchers() -> DoctorCheck:
     if db_path.is_file():
         try:
             conn = sqlite3.connect(str(db_path))
-            row = conn.execute(
-                "SELECT COUNT(*) FROM state_watchers WHERE status = 'active'"
-            ).fetchone()
-            conn.close()
-            active_watchers = int(row[0] or 0) if row else 0
+            try:
+                row = conn.execute(
+                    "SELECT COUNT(*) FROM state_watchers WHERE status = 'active'"
+                ).fetchone()
+                active_watchers = int(row[0] or 0) if row else 0
+            finally:
+                conn.close()
         except Exception:
             active_watchers = 0
 
@@ -2518,37 +2534,38 @@ def check_automation_telemetry(days: int = 7) -> DoctorCheck:
 
     try:
         conn = sqlite3.connect(str(db_path), timeout=2)
-        conn.row_factory = sqlite3.Row
-        table = conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='automation_runs'"
-        ).fetchone()
-        if not table:
-            conn.close()
-            return DoctorCheck(
-                id="runtime.automation_telemetry",
-                tier="runtime",
-                status="degraded",
-                severity="warn",
-                summary="Automation telemetry schema is missing",
-                evidence=["table automation_runs not found"],
-                repair_plan=["Run NEXO migrations before trusting automation cost/parity metrics"],
-                escalation_prompt="Shared automation runs are happening without the telemetry table that release metrics depend on.",
-            )
+        try:
+            conn.row_factory = sqlite3.Row
+            table = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='automation_runs'"
+            ).fetchone()
+            if not table:
+                return DoctorCheck(
+                    id="runtime.automation_telemetry",
+                    tier="runtime",
+                    status="degraded",
+                    severity="warn",
+                    summary="Automation telemetry schema is missing",
+                    evidence=["table automation_runs not found"],
+                    repair_plan=["Run NEXO migrations before trusting automation cost/parity metrics"],
+                    escalation_prompt="Shared automation runs are happening without the telemetry table that release metrics depend on.",
+                )
 
-        row = conn.execute(
-            """
-            SELECT
-                COUNT(*) AS runs,
-                SUM(CASE WHEN (input_tokens + cached_input_tokens + output_tokens) > 0 THEN 1 ELSE 0 END) AS usage_runs,
-                SUM(CASE WHEN total_cost_usd IS NOT NULL THEN 1 ELSE 0 END) AS cost_runs,
-                SUM(CASE WHEN cost_source = 'pricing_unavailable' THEN 1 ELSE 0 END) AS pricing_gaps,
-                GROUP_CONCAT(DISTINCT backend) AS backends
-            FROM automation_runs
-            WHERE created_at >= datetime('now', ?)
-            """,
-            (f"-{days} days",),
-        ).fetchone()
-        conn.close()
+            row = conn.execute(
+                """
+                SELECT
+                    COUNT(*) AS runs,
+                    SUM(CASE WHEN (input_tokens + cached_input_tokens + output_tokens) > 0 THEN 1 ELSE 0 END) AS usage_runs,
+                    SUM(CASE WHEN total_cost_usd IS NOT NULL THEN 1 ELSE 0 END) AS cost_runs,
+                    SUM(CASE WHEN cost_source = 'pricing_unavailable' THEN 1 ELSE 0 END) AS pricing_gaps,
+                    GROUP_CONCAT(DISTINCT backend) AS backends
+                FROM automation_runs
+                WHERE created_at >= datetime('now', ?)
+                """,
+                (f"-{days} days",),
+            ).fetchone()
+        finally:
+            conn.close()
     except Exception as exc:
         return DoctorCheck(
             id="runtime.automation_telemetry",
@@ -2623,22 +2640,22 @@ def check_automation_telemetry(days: int = 7) -> DoctorCheck:
 def run_runtime_checks(fix: bool = False) -> list[DoctorCheck]:
     """Run all runtime-tier checks. Read-only by default."""
     return [
-        check_immune_status(),
-        check_watchdog_status(),
-        check_stale_sessions(),
-        check_cron_freshness(),
-        check_client_backend_preferences(),
-        check_client_bootstrap_parity(fix=fix),
-        check_codex_session_parity(),
-        check_codex_conditioned_file_discipline(),
-        check_claude_desktop_shared_brain(),
-        check_transcript_source_parity(),
-        check_client_assumption_regressions(),
-        check_protocol_compliance(),
-        check_automation_telemetry(),
-        check_state_watchers(),
-        check_release_artifact_sync(),
-        check_launchagent_integrity(fix=fix),
-        check_personal_script_registry(fix=fix),
-        check_skill_health(fix=fix),
+        safe_check(check_immune_status),
+        safe_check(check_watchdog_status),
+        safe_check(check_stale_sessions),
+        safe_check(check_cron_freshness),
+        safe_check(check_client_backend_preferences),
+        safe_check(check_client_bootstrap_parity, fix=fix),
+        safe_check(check_codex_session_parity),
+        safe_check(check_codex_conditioned_file_discipline),
+        safe_check(check_claude_desktop_shared_brain),
+        safe_check(check_transcript_source_parity),
+        safe_check(check_client_assumption_regressions),
+        safe_check(check_protocol_compliance),
+        safe_check(check_automation_telemetry),
+        safe_check(check_state_watchers),
+        safe_check(check_release_artifact_sync),
+        safe_check(check_launchagent_integrity, fix=fix),
+        safe_check(check_personal_script_registry, fix=fix),
+        safe_check(check_skill_health, fix=fix),
     ]