@smilintux/skcapstone 0.1.0 → 0.2.3

package/src/skcapstone/self_healing.py

@@ -0,0 +1,341 @@
+"""
+Self-Healing Doctor — auto-diagnosing, auto-remediating agent health.
+
+Extends the existing doctor.py diagnostics with auto-fix capabilities.
+Follows the TrusteeMonitor escalation pattern:
+    diagnose → auto-fix → re-check → escalate if still broken.
+
+Registered auto-fixes:
+    - Missing home dirs → mkdir -p
+    - Missing memory index → rebuild from memory/**/*.json
+    - Missing sync backends → write default sync-manifest.json
+    - LLM unreachable → re-probe backends, switch fallback
+    - Config corrupt → reset to defaults
+    - Dead worker thread → restart
+    - Dead inotify thread → restart observer
+    - Stale model profiles → flag for human review
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger("skcapstone.self_healing")
+
+
+class SelfHealingDoctor:
+    """Auto-diagnosing, auto-remediating health monitor.
+
+    Runs diagnostics from doctor.py plus consciousness-specific checks.
+    Auto-fixes what it can, escalates what it can't.
+
+    Args:
+        home: Agent home directory.
+        consciousness_loop: Optional reference to the running loop.
+    """
+
+    def __init__(
+        self,
+        home: Path,
+        consciousness_loop: Any = None,
+    ) -> None:
+        self._home = home
+        self._consciousness = consciousness_loop
+        self._last_report: dict[str, Any] = {}
+
+    def diagnose_and_heal(self) -> dict[str, Any]:
+        """Run diagnostics, auto-fix what we can, escalate the rest.
+
+        Returns:
+            Dict with checks_run, checks_passed, auto_fixed,
+            still_broken, escalated counts and details.
+        """
+        checks_run = 0
+        checks_passed = 0
+        auto_fixed = 0
+        still_broken = 0
+        escalated_items: list[str] = []
+        details: list[dict[str, Any]] = []
+
+        # Run all check groups
+        check_methods = [
+            self._check_home_dirs,
+            self._check_memory_index,
+            self._check_sync_manifest,
+            self._check_consciousness_health,
+            self._check_profile_freshness,
+        ]
+
+        for check_fn in check_methods:
+            try:
+                result = check_fn()
+                checks_run += 1
+                if result["status"] == "ok":
+                    checks_passed += 1
+                elif result["status"] == "fixed":
+                    checks_passed += 1
+                    auto_fixed += 1
+                elif result["status"] == "broken":
+                    still_broken += 1
+                    escalated_items.append(result.get("name", "unknown"))
+                details.append(result)
+            except Exception as exc:
+                checks_run += 1
+                still_broken += 1
+                details.append({
+                    "name": check_fn.__name__,
+                    "status": "error",
+                    "error": str(exc),
+                })
+
+        report = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "checks_run": checks_run,
+            "checks_passed": checks_passed,
+            "auto_fixed": auto_fixed,
+            "still_broken": still_broken,
+            "escalated": escalated_items,
+            "details": details,
+        }
+        self._last_report = report
+
+        # Escalate if needed
+        if escalated_items:
+            self._escalate(escalated_items)
+
+        return report
+
+    @property
+    def last_report(self) -> dict[str, Any]:
+        """Most recent diagnostic report."""
+        return self._last_report
+
+    # -------------------------------------------------------------------
+    # Check methods — each returns {"name", "status", "message"}
+    # -------------------------------------------------------------------
+
+    def _check_home_dirs(self) -> dict[str, Any]:
+        """Ensure all required home subdirectories exist."""
+        required = [
+            "identity", "memory", "trust", "security", "sync", "config",
+            "soul", "logs",
+        ]
+        missing = []
+        for subdir in required:
+            path = self._home / subdir
+            if not path.exists():
+                missing.append(subdir)
+
+        if not missing:
+            return {"name": "home_dirs", "status": "ok", "message": "All dirs present"}
+
+        # Auto-fix: create missing dirs
+        for subdir in missing:
+            (self._home / subdir).mkdir(parents=True, exist_ok=True)
+            logger.info("Auto-created missing dir: %s", subdir)
+
+        return {
+            "name": "home_dirs",
+            "status": "fixed",
+            "message": f"Created {len(missing)} missing dirs: {missing}",
+        }
+
+    def _check_memory_index(self) -> dict[str, Any]:
+        """Check if memory index exists and is valid."""
+        memory_dir = self._home / "memory"
+        index_path = memory_dir / "index.json"
+
+        if not memory_dir.exists():
+            return {"name": "memory_index", "status": "ok", "message": "No memory dir"}
+
+        if index_path.exists():
+            try:
+                data = json.loads(index_path.read_text(encoding="utf-8"))
+                if isinstance(data, (list, dict)):
+                    return {"name": "memory_index", "status": "ok", "message": "Index valid"}
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        # Auto-fix: rebuild index from memory files
+        entries = []
+        for layer_dir in ("short-term", "mid-term", "long-term"):
+            layer_path = memory_dir / layer_dir
+            if layer_path.exists():
+                for f in layer_path.glob("*.json"):
+                    try:
+                        entry = json.loads(f.read_text(encoding="utf-8"))
+                        entries.append({
+                            "memory_id": entry.get("memory_id", f.stem),
+                            "layer": layer_dir,
+                            "tags": entry.get("tags", []),
+                        })
+                    except Exception as exc:
+                        logger.debug("Skipping malformed memory file %s: %s", f, exc)
+                        continue
+
+        index_path.write_text(
+            json.dumps(entries, indent=2), encoding="utf-8"
+        )
+        logger.info("Rebuilt memory index with %d entries", len(entries))
+
+        return {
+            "name": "memory_index",
+            "status": "fixed",
+            "message": f"Rebuilt index with {len(entries)} entries",
+        }
+
+    def _check_sync_manifest(self) -> dict[str, Any]:
+        """Check sync-manifest.json exists."""
+        sync_dir = self._home / "sync"
+        manifest_path = sync_dir / "sync-manifest.json"
+
+        if manifest_path.exists():
+            return {"name": "sync_manifest", "status": "ok", "message": "Manifest present"}
+
+        if not sync_dir.exists():
+            return {"name": "sync_manifest", "status": "ok", "message": "No sync dir"}
+
+        # Auto-fix: write default manifest
+        default_manifest = {
+            "version": 1,
+            "backends": ["syncthing"],
+            "auto_push": True,
+            "auto_pull": True,
+        }
+        sync_dir.mkdir(parents=True, exist_ok=True)
+        manifest_path.write_text(
+            json.dumps(default_manifest, indent=2), encoding="utf-8"
+        )
+        logger.info("Wrote default sync-manifest.json")
+
+        return {
+            "name": "sync_manifest",
+            "status": "fixed",
+            "message": "Created default sync manifest",
+        }
+
+    def _check_consciousness_health(self) -> dict[str, Any]:
+        """Check consciousness loop health."""
+        if not self._consciousness:
+            return {
+                "name": "consciousness",
+                "status": "ok",
+                "message": "Consciousness loop not loaded (disabled)",
+            }
+
+        issues: list[str] = []
+
+        # Check at least one backend is available
+        backends = self._consciousness._bridge.available_backends
+        if not any(backends.values()):
+            # Auto-fix: re-probe
+            self._consciousness._bridge._probe_available_backends()
+            backends = self._consciousness._bridge.available_backends
+            if any(backends.values()):
+                logger.info("Re-probed backends — found available: %s",
+                            [k for k, v in backends.items() if v])
+            else:
+                issues.append("No LLM backends reachable")
+
+        # Check inotify thread
+        observer = self._consciousness._observer
+        if observer and hasattr(observer, "is_alive") and not observer.is_alive():
+            # Auto-fix: restart observer
+            try:
+                self._consciousness._run_inotify_restart()
+                logger.info("Restarted inotify observer")
+            except Exception as exc:
+                logger.debug("Inotify restart failed: %s", exc)
+                issues.append("Inotify thread dead — restart failed")
+
+        if issues:
+            return {
+                "name": "consciousness",
+                "status": "broken",
+                "message": "; ".join(issues),
+            }
+
+        return {
+            "name": "consciousness",
+            "status": "ok",
+            "message": "Consciousness loop healthy",
+        }
+
+    def _check_profile_freshness(self) -> dict[str, Any]:
+        """Flag model profiles older than 90 days as needing review."""
+        try:
+            from skcapstone.prompt_adapter import PromptAdapter
+            adapter = PromptAdapter()
+            stale: list[str] = []
+            now = datetime.now(timezone.utc)
+
+            for profile in adapter.profiles:
+                if not profile.last_updated:
+                    stale.append(profile.family)
+                    continue
+                try:
+                    updated = datetime.fromisoformat(profile.last_updated)
+                    if hasattr(updated, "tzinfo") and updated.tzinfo is None:
+                        updated = updated.replace(tzinfo=timezone.utc)
+                    age_days = (now - updated).days
+                    if age_days > 90:
+                        stale.append(f"{profile.family} ({age_days}d)")
+                except (ValueError, TypeError):
+                    stale.append(profile.family)
+
+            if stale:
+                logger.warning(
+                    "Model profiles older than 90 days (manual update recommended): %s",
+                    ", ".join(stale),
+                )
+                return {
+                    "name": "profile_freshness",
+                    "status": "ok",  # Informational, not broken
+                    "message": f"Stale profiles (>90d): {', '.join(stale)}",
+                    "stale_profiles": stale,
+                }
+
+            return {
+                "name": "profile_freshness",
+                "status": "ok",
+                "message": "All profiles fresh",
+            }
+        except Exception as exc:
+            return {
+                "name": "profile_freshness",
+                "status": "ok",
+                "message": f"Could not check profiles: {exc}",
+            }
+
+    # -------------------------------------------------------------------
+    # Escalation
+    # -------------------------------------------------------------------
+
+    def _escalate(self, items: list[str]) -> None:
+        """Escalate unresolved issues via SKChat.
+
+        Args:
+            items: List of check names that are still broken.
+        """
+        try:
+            from skcapstone.mcp_tools._helpers import _get_agent_name
+            agent_name = _get_agent_name(self._home)
+            message = (
+                f"Self-healing alert from {agent_name}: "
+                f"{len(items)} issue(s) could not be auto-fixed: {', '.join(items)}"
+            )
+            logger.warning("Escalating: %s", message)
+
+            # Try to send via SKChat if available
+            try:
+                from skchat.messenger import AgentMessenger
+                messenger = AgentMessenger.from_config()
+                messenger.send("chef", message)
+            except Exception as exc:
+                logger.debug("SKChat escalation failed (best-effort): %s", exc)
+        except Exception as exc:
+            logger.debug("Escalation failed: %s", exc)

package/src/skcapstone/service_health.py

@@ -0,0 +1,228 @@
+"""Service URL health check mechanism.
+
+Pings all known services in the sovereign stack and returns structured
+status reports.  Each check uses urllib.request with a 3-second timeout
+so the full sweep completes in bounded time even when services are down.
+
+Usage (library):
+    from skcapstone.service_health import check_all_services
+    results = check_all_services()
+    for svc in results:
+        print(f"{svc['name']}: {svc['status']} ({svc['latency_ms']}ms)")
+
+Usage (scheduled task):
+    from skcapstone.service_health import make_service_health_task
+    callback = make_service_health_task()
+    scheduler.register("service_health_check", 300, callback)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import socket
+import time
+import urllib.error
+import urllib.request
+from typing import Any
+
+logger = logging.getLogger("skcapstone.service_health")
+
+# Default timeout per service check (seconds).
+CHECK_TIMEOUT = 3
+
+
+# ---------------------------------------------------------------------------
+# Individual service checks
+# ---------------------------------------------------------------------------
+
+
+def _http_check(
+    name: str,
+    url: str,
+    *,
+    headers: dict[str, str] | None = None,
+    version_key: str | None = None,
+) -> dict[str, Any]:
+    """Perform an HTTP GET health check against *url*.
+
+    Args:
+        name: Human-readable service name.
+        url: Full URL to probe (e.g. ``http://localhost:6333/healthz``).
+        headers: Optional extra HTTP headers (e.g. API keys).
+        version_key: If set, extract this key from the JSON response as version.
+
+    Returns:
+        Status dict with name, url, status, latency_ms, version, error.
+    """
+    result: dict[str, Any] = {
+        "name": name,
+        "url": url,
+        "status": "unknown",
+        "latency_ms": 0,
+        "version": None,
+        "error": None,
+    }
+    req = urllib.request.Request(url, headers=headers or {})
+    t0 = time.monotonic()
+    try:
+        with urllib.request.urlopen(req, timeout=CHECK_TIMEOUT) as resp:
+            latency = (time.monotonic() - t0) * 1000
+            result["latency_ms"] = round(latency, 1)
+            result["status"] = "up"
+
+            if version_key:
+                try:
+                    body = json.loads(resp.read().decode("utf-8"))
+                    result["version"] = body.get(version_key)
+                except Exception:
+                    pass
+    except urllib.error.HTTPError as exc:
+        latency = (time.monotonic() - t0) * 1000
+        result["latency_ms"] = round(latency, 1)
+        # A non-2xx response still means the service is reachable.
+        if exc.code < 500:
+            result["status"] = "up"
+        else:
+            result["status"] = "down"
+            result["error"] = f"HTTP {exc.code}"
+    except Exception as exc:
+        latency = (time.monotonic() - t0) * 1000
+        result["latency_ms"] = round(latency, 1)
+        result["status"] = "down"
+        result["error"] = str(exc)[:200]
+    return result
+
+
+def _tcp_check(name: str, host: str, port: int) -> dict[str, Any]:
+    """Perform a raw TCP connect check.
+
+    Args:
+        name: Human-readable service name.
+        host: Hostname or IP to connect to.
+        port: TCP port number.
+
+    Returns:
+        Status dict with name, url, status, latency_ms, version, error.
+    """
+    url = f"tcp://{host}:{port}"
+    result: dict[str, Any] = {
+        "name": name,
+        "url": url,
+        "status": "unknown",
+        "latency_ms": 0,
+        "version": None,
+        "error": None,
+    }
+    t0 = time.monotonic()
+    try:
+        sock = socket.create_connection((host, port), timeout=CHECK_TIMEOUT)
+        latency = (time.monotonic() - t0) * 1000
+        sock.close()
+        result["latency_ms"] = round(latency, 1)
+        result["status"] = "up"
+    except Exception as exc:
+        latency = (time.monotonic() - t0) * 1000
+        result["latency_ms"] = round(latency, 1)
+        result["status"] = "down"
+        result["error"] = str(exc)[:200]
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Aggregate check
+# ---------------------------------------------------------------------------
+
+
+def check_all_services() -> list[dict[str, Any]]:
+    """Ping every known service and return a list of status dicts.
+
+    Environment variables override default URLs:
+        SKMEMORY_SKVECTOR_URL   — Qdrant REST base (default http://localhost:6333)
+        SKMEMORY_SKGRAPH_HOST   — FalkorDB host   (default localhost)
+        SKMEMORY_SKGRAPH_PORT   — FalkorDB port   (default 6379)
+        SYNCTHING_API_URL       — Syncthing REST   (default http://localhost:8384)
+        SYNCTHING_API_KEY       — Syncthing API key (optional)
+        SKCAPSTONE_DAEMON_URL   — Daemon HTTP base (default http://localhost:9383)
+        SKCHAT_DAEMON_URL       — SKChat daemon    (default http://localhost:9385)
+
+    Returns:
+        List of dicts, each containing: name, url, status ("up"|"down"|"unknown"),
+        latency_ms, version, error.
+    """
+    results: list[dict[str, Any]] = []
+
+    # -- SKVector (Qdrant) --------------------------------------------------
+    qdrant_base = os.environ.get("SKMEMORY_SKVECTOR_URL", "http://localhost:6333")
+    qdrant_url = qdrant_base.rstrip("/") + "/healthz"
+    results.append(_http_check("skvector (Qdrant)", qdrant_url))
+
+    # -- SKGraph (FalkorDB) — TCP check on Redis protocol port ---------------
+    graph_host = os.environ.get("SKMEMORY_SKGRAPH_HOST", "localhost")
+    graph_port = int(os.environ.get("SKMEMORY_SKGRAPH_PORT", "6379"))
+    results.append(_tcp_check("skgraph (FalkorDB)", graph_host, graph_port))
+
+    # -- Syncthing -----------------------------------------------------------
+    syncthing_base = os.environ.get("SYNCTHING_API_URL", "http://localhost:8384")
+    syncthing_url = syncthing_base.rstrip("/") + "/rest/system/status"
+    syncthing_headers: dict[str, str] = {}
+    api_key = os.environ.get("SYNCTHING_API_KEY", "")
+    if api_key:
+        syncthing_headers["X-API-Key"] = api_key
+    results.append(
+        _http_check(
+            "syncthing",
+            syncthing_url,
+            headers=syncthing_headers,
+            version_key="version",
+        )
+    )
+
+    # -- skcapstone daemon ---------------------------------------------------
+    daemon_base = os.environ.get("SKCAPSTONE_DAEMON_URL", "http://localhost:9383")
+    daemon_url = daemon_base.rstrip("/") + "/health"
+    results.append(_http_check("skcapstone daemon", daemon_url))
+
+    # -- skchat daemon -------------------------------------------------------
+    chat_base = os.environ.get("SKCHAT_DAEMON_URL", "http://localhost:9385")
+    chat_url = chat_base.rstrip("/") + "/health"
+    results.append(_http_check("skchat daemon", chat_url))
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Scheduled-task factory
+# ---------------------------------------------------------------------------
+
+
+def make_service_health_task() -> callable:
+    """Return a zero-arg callback suitable for TaskScheduler.register().
+
+    Runs check_all_services() and logs results.  Down services are logged
+    at WARNING level; all-up is logged at DEBUG level.
+    """
+
+    def _run() -> None:
+        results = check_all_services()
+        down = [r for r in results if r["status"] == "down"]
+        if down:
+            names = ", ".join(r["name"] for r in down)
+            logger.warning(
+                "Service health: %d/%d down — %s", len(down), len(results), names
+            )
+            for r in down:
+                logger.warning(
+                    "  %s (%s): %s", r["name"], r["url"], r["error"] or "unreachable"
+                )
+        else:
+            up_count = sum(1 for r in results if r["status"] == "up")
+            logger.debug(
+                "Service health: %d/%d up, %d unknown",
+                up_count,
+                len(results),
+                len(results) - up_count,
+            )
+
+    return _run