@agentunion/kite 1.0.0

package/extensions/services/watchdog/monitor.py

@@ -0,0 +1,420 @@
+"""
+Watchdog monitor — periodic health checks + resource monitoring.
+Launcher handles process-level crashes; Watchdog handles app-level failures
+(process alive but /health unhealthy) and resource anomalies (CPU/memory).
+"""
+
+import asyncio
+import json
+import time
+from datetime import datetime, timezone
+
+import httpx
+
+
+# Module health states
+HEALTHY = "healthy"
+UNHEALTHY = "unhealthy"
+UNKNOWN = "unknown"
+
+# Resource states
+NORMAL = "normal"
+WARNING = "warning"
+CRITICAL = "critical"
+
+
+class ModuleStatus:
+    """Track health and resource state for a single module."""
+
+    def __init__(self, module_id: str, api_endpoint: str, health_endpoint: str, pid: int = None):
+        self.module_id = module_id
+        self.api_endpoint = api_endpoint
+        self.health_endpoint = health_endpoint
+        self.pid = pid
+        # Health check state
+        self.state: str = UNKNOWN
+        self.fail_count: int = 0
+        self.last_check: float = 0
+        self.last_healthy: float = 0
+        self.last_error: str = ""
+        self.restarted_count: int = 0
+        # Resource monitoring state
+        self.resource_state: str = NORMAL
+        self.resource_warning_count: int = 0
+        self.resource_critical_count: int = 0
+        self.critical_since: float = 0  # when critical started
+        self.memory_samples: list[float] = []  # last 5 memory_rss samples
+        self.recovery_since: float = 0  # when recovery observation started
+        self.last_metrics: dict = {}
+
+
+class HealthMonitor:
+    """Periodically check module health and resources, track failures, trigger restarts."""
+
+    # Thresholds
+    HEALTH_TIMEOUT = 5           # HTTP timeout per health check
+    FAIL_THRESHOLD = 3           # consecutive failures before restart
+    MAX_RESTARTS = 3             # max restarts before giving up on a module
+    ALERT_AFTER_RESTARTS = 2     # publish alert after this many restarts
+
+    # Resource thresholds
+    MEMORY_WARNING = 70          # memory_percent > 70% → warning
+    MEMORY_CRITICAL = 85         # memory_percent > 85% → critical
+    CPU_WARNING = 80             # cpu_percent > 80% → warning
+    CPU_CRITICAL = 95            # cpu_percent > 95% → critical
+    CRITICAL_TIMEOUT = 30        # seconds in critical before restart
+    RECOVERY_OBSERVE = 60        # seconds to observe after recovery
+
+    # Check intervals per resource state
+    INTERVALS = {NORMAL: 15, WARNING: 5, CRITICAL: 2}
+
+    def __init__(self, own_token: str, registry_url: str, launcher_url: str,
+                 publish_event=None):
+        self.own_token = own_token
+        self.registry_url = registry_url
+        self.launcher_url = launcher_url
+        self.publish_event = publish_event  # async callable(event_dict)
+        self.modules: dict[str, ModuleStatus] = {}
+        self._running = False
+        self._psutil = None  # lazy import
+
+    # ── Module discovery ──
+
+    async def discover_modules(self):
+        """Fetch monitored modules from Launcher API + Registry health endpoints."""
+        # Step 1: Get module list with monitor/pid from Launcher API
+        monitored = {}  # name -> {pid, running}
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(
+                    f"{self.launcher_url}/launcher/modules", timeout=5,
+                )
+                if resp.status_code == 200:
+                    for m in resp.json():
+                        name = m.get("name", "")
+                        if name == "watchdog" or not m.get("monitor", True):
+                            continue
+                        if m.get("actual_state", "").startswith("running"):
+                            monitored[name] = m.get("pid")
+        except Exception as e:
+            print(f"[watchdog] Launcher API failed: {e}")
+            return
+
+        # Step 2: Get health endpoints from Registry
+        health_map = {}  # name -> {api_endpoint, health_endpoint}
+        headers = {"Authorization": f"Bearer {self.own_token}"}
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(
+                    f"{self.registry_url}/lookup",
+                    params={"field": "health_endpoint"},
+                    headers=headers, timeout=5,
+                )
+                if resp.status_code == 200:
+                    for entry in resp.json():
+                        mid = entry.get("module", "")
+                        if mid in monitored:
+                            health_map[mid] = {
+                                "api_endpoint": entry.get("api_endpoint", ""),
+                                "health_endpoint": entry.get("value", "/health"),
+                            }
+        except Exception:
+            pass
+
+        # Step 3: Sync module list
+        seen = set()
+        for mid, pid in monitored.items():
+            seen.add(mid)
+            h = health_map.get(mid, {})
+            if mid not in self.modules:
+                self.modules[mid] = ModuleStatus(
+                    module_id=mid,
+                    api_endpoint=h.get("api_endpoint", ""),
+                    health_endpoint=h.get("health_endpoint", "/health"),
+                    pid=pid,
+                )
+            else:
+                self.modules[mid].pid = pid
+
+        for mid in list(self.modules):
+            if mid not in seen:
+                del self.modules[mid]
+
+    # ── Health check ──
+
+    async def _check_one(self, status: ModuleStatus):
+        """Check a single module's /health endpoint."""
+        url = f"{status.api_endpoint}{status.health_endpoint}"
+        status.last_check = time.time()
+
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(url, timeout=self.HEALTH_TIMEOUT)
+                if resp.status_code == 200:
+                    body = resp.json()
+                    if body.get("status") == "healthy":
+                        await self._mark_healthy(status)
+                        return
+                    status.last_error = f"unhealthy response: {body.get('status')}"
+                else:
+                    status.last_error = f"HTTP {resp.status_code}"
+        except Exception as e:
+            status.last_error = str(e)
+
+        await self._mark_unhealthy(status)
+
+    async def _mark_healthy(self, status: ModuleStatus):
+        """Module responded healthy — reset failure tracking."""
+        was_unhealthy = status.state == UNHEALTHY
+        status.state = HEALTHY
+        status.fail_count = 0
+        status.last_healthy = time.time()
+        status.last_error = ""
+
+        if was_unhealthy:
+            print(f"[watchdog] {status.module_id} recovered")
+            await self._publish("watchdog.module.recovered", {
+                "module_id": status.module_id,
+            })
+
+    async def _mark_unhealthy(self, status: ModuleStatus):
+        """Module failed health check — increment counter, maybe restart."""
+        status.state = UNHEALTHY
+        status.fail_count += 1
+        print(f"[watchdog] {status.module_id} unhealthy ({status.fail_count}/{self.FAIL_THRESHOLD}): {status.last_error}")
+
+        await self._publish("watchdog.module.unhealthy", {
+            "module_id": status.module_id,
+            "fail_count": status.fail_count,
+            "error": status.last_error,
+        })
+
+        # Restart if threshold reached
+        if (status.fail_count >= self.FAIL_THRESHOLD
+                and status.restarted_count < self.MAX_RESTARTS):
+            await self._restart_module(status)
+
+    # ── Restart via Launcher API ──
+
+    async def _restart_module(self, status: ModuleStatus):
+        """Restart a module via Launcher API."""
+        mid = status.module_id
+        print(f"[watchdog] Restarting {mid} (attempt {status.restarted_count + 1}/{self.MAX_RESTARTS})")
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    f"{self.launcher_url}/launcher/modules/{mid}/restart",
+                    json={"reason": "resource_critical" if status.resource_state == CRITICAL else "restart"},
+                    timeout=15,
+                )
+                if resp.status_code == 200:
+                    status.restarted_count += 1
+                    status.fail_count = 0
+                    print(f"[watchdog] {mid} restart requested")
+                    if status.restarted_count >= self.ALERT_AFTER_RESTARTS:
+                        await self._publish("watchdog.alert", {
+                            "module_id": mid,
+                            "restarted_count": status.restarted_count,
+                            "message": f"{mid} has been restarted {status.restarted_count} times",
+                        })
+                else:
+                    print(f"[watchdog] {mid} restart failed: HTTP {resp.status_code}")
+        except Exception as e:
+            print(f"[watchdog] {mid} restart error: {e}")
+
+    # ── Resource monitoring ──
+
+    def _collect_metrics(self, status: ModuleStatus) -> dict | None:
+        """Collect CPU/memory metrics for a module via psutil."""
+        if not status.pid:
+            return None
+        if not self._psutil:
+            try:
+                import psutil
+                self._psutil = psutil
+            except ImportError:
+                return None
+        try:
+            proc = self._psutil.Process(status.pid)
+            return {
+                "memory_rss": round(proc.memory_info().rss / 1024 / 1024, 1),
+                "memory_percent": round(proc.memory_percent(), 1),
+                "cpu_percent": round(proc.cpu_percent(interval=0.1), 1),
+                "thread_count": proc.num_threads(),
+            }
+        except Exception:
+            return None
+
+    async def _check_resources(self, status: ModuleStatus):
+        """Collect metrics and evaluate resource state."""
+        metrics = self._collect_metrics(status)
+        if not metrics:
+            return
+        status.last_metrics = metrics
+
+        # Track memory samples for leak detection
+        status.memory_samples.append(metrics["memory_rss"])
+        if len(status.memory_samples) > 5:
+            status.memory_samples.pop(0)
+
+        # Determine raw level from thresholds
+        mem_pct = metrics["memory_percent"]
+        cpu_pct = metrics["cpu_percent"]
+        if mem_pct > self.MEMORY_CRITICAL or cpu_pct > self.CPU_CRITICAL:
+            raw_level = CRITICAL
+        elif mem_pct > self.MEMORY_WARNING or cpu_pct > self.CPU_WARNING:
+            raw_level = WARNING
+        else:
+            raw_level = NORMAL
+
+        # Memory leak detection: 3+ consecutive rises, total >20%
+        if len(status.memory_samples) >= 3:
+            recent = status.memory_samples[-3:]
+            if all(recent[i] < recent[i+1] for i in range(len(recent)-1)):
+                if recent[-1] > recent[0] * 1.2:
+                    raw_level = CRITICAL
+
+        await self._update_resource_state(status, raw_level, metrics)
+
+    async def _update_resource_state(self, status: ModuleStatus, level: str, metrics: dict):
+        """Update resource state with consecutive-count logic."""
+        now = time.time()
+        old_state = status.resource_state
+
+        if level == CRITICAL:
+            status.resource_critical_count += 1
+            status.resource_warning_count = 0
+            status.recovery_since = 0
+            if status.resource_critical_count >= 3 and old_state != CRITICAL:
+                status.resource_state = CRITICAL
+                status.critical_since = now
+                await self._publish("watchdog.module.resource_critical", {
+                    "module_id": status.module_id, "metrics": metrics,
+                })
+            elif old_state == CRITICAL:
+                # Already critical — check timeout
+                if now - status.critical_since >= self.CRITICAL_TIMEOUT:
+                    await self._resource_restart(status)
+        elif level == WARNING:
+            status.resource_warning_count += 1
+            status.resource_critical_count = 0
+            status.recovery_since = 0
+            if status.resource_warning_count >= 2 and old_state == NORMAL:
+                status.resource_state = WARNING
+                await self._publish("watchdog.module.resource_warning", {
+                    "module_id": status.module_id, "metrics": metrics,
+                })
+        else:  # NORMAL
+            status.resource_warning_count = 0
+            status.resource_critical_count = 0
+            if old_state != NORMAL:
+                if not status.recovery_since:
+                    status.recovery_since = now
+                elif now - status.recovery_since >= self.RECOVERY_OBSERVE:
+                    status.resource_state = NORMAL
+                    status.recovery_since = 0
+                    await self._publish("watchdog.module.resource_recovered", {
+                        "module_id": status.module_id,
+                    })
+
+    async def _resource_restart(self, status: ModuleStatus):
+        """Restart module due to resource critical timeout."""
+        if status.restarted_count >= self.MAX_RESTARTS:
+            await self._publish("watchdog.alert", {
+                "module_id": status.module_id,
+                "message": f"{status.module_id} exceeded max restarts (resource_critical)",
+            })
+            return
+        print(f"[watchdog] Resource critical timeout for {status.module_id}, restarting...")
+        status.resource_state = NORMAL
+        status.resource_critical_count = 0
+        status.critical_since = 0
+        await self._restart_module(status)
+
+    # ── Event publishing ──
+
+    async def _publish(self, event_type: str, data: dict):
+        """Publish event via callback if available."""
+        if not self.publish_event:
+            return
+        try:
+            await self.publish_event({
+                "event": event_type,
+                "data": data,
+            })
+        except Exception:
+            pass
+
+    # ── Incoming event handler ──
+
+    async def handle_event(self, msg: dict):
+        """Handle events from Event Hub (module.started / module.stopped)."""
+        event_type = msg.get("event", "")
+        data = msg.get("data", {})
+        module_id = data.get("module_id", "")
+
+        if not module_id or module_id == "watchdog":
+            return
+
+        if event_type == "module.started":
+            print(f"[watchdog] Received module.started: {module_id}")
+            # Trigger immediate discovery to pick up the new module
+            await self.discover_modules()
+
+        elif event_type == "module.stopped":
+            print(f"[watchdog] Received module.stopped: {module_id}")
+            # Remove from tracking — it's gone
+            self.modules.pop(module_id, None)
+
+    # ── Main loop ──
+
+    async def run(self):
+        """Main monitoring loop with dynamic intervals per resource state."""
+        self._running = True
+        print("[watchdog] Monitor started")
+        last_discover = 0
+
+        while self._running:
+            now = time.time()
+            # Rediscover every 30s
+            if now - last_discover >= 30:
+                await self.discover_modules()
+                last_discover = now
+
+            if self.modules:
+                tasks = []
+                for s in self.modules.values():
+                    tasks.append(self._check_one(s))
+                    tasks.append(self._check_resources(s))
+                await asyncio.gather(*tasks, return_exceptions=True)
+
+            # Sleep = shortest interval needed by any module
+            interval = self._min_interval()
+            await asyncio.sleep(interval)
+
+    def _min_interval(self) -> float:
+        """Return the shortest check interval needed across all modules."""
+        if not self.modules:
+            return self.INTERVALS[NORMAL]
+        return min(self.INTERVALS.get(s.resource_state, 15) for s in self.modules.values())
+
+    def stop(self):
+        self._running = False
+
+    # ── Stats ──
+
+    def get_status(self) -> dict:
+        """Return current health and resource status of all monitored modules."""
+        return {
+            mid: {
+                "state": s.state,
+                "fail_count": s.fail_count,
+                "restarted_count": s.restarted_count,
+                "last_check": s.last_check,
+                "last_healthy": s.last_healthy,
+                "last_error": s.last_error,
+                "resource_state": s.resource_state,
+                "metrics": s.last_metrics,
+            }
+            for mid, s in self.modules.items()
+        }

package/extensions/services/watchdog/server.py

@@ -0,0 +1,167 @@
+"""
+Watchdog HTTP server.
+Exposes /health and /status endpoints. Runs the monitor loop on startup.
+Connects to Event Hub via WebSocket for event publishing and subscription.
+"""
+
+import asyncio
+import json
+import time
+import uuid
+
+import httpx
+import websockets
+from fastapi import FastAPI
+
+from .monitor import HealthMonitor
+
+
+class WatchdogServer:
+
+    def __init__(self, monitor: HealthMonitor, token: str = "",
+                 event_hub_ws: str = ""):
+        self.monitor = monitor
+        self.token = token
+        self.event_hub_ws = event_hub_ws
+        self._monitor_task: asyncio.Task | None = None
+        self._ws_task: asyncio.Task | None = None
+        self._heartbeat_task: asyncio.Task | None = None
+        self._ws: object | None = None
+        self._start_time = time.time()
+        self.app = self._create_app()
+
+        # Wire up publish callback on monitor
+        self.monitor.publish_event = self._publish_event
+
+    def _create_app(self) -> FastAPI:
+        app = FastAPI(title="Kite Watchdog", docs_url=None, redoc_url=None)
+        server = self
+
+        @app.on_event("startup")
+        async def _startup():
+            server._monitor_task = asyncio.create_task(server.monitor.run())
+            server._heartbeat_task = asyncio.create_task(server._heartbeat_loop())
+            if server.event_hub_ws:
+                server._ws_task = asyncio.create_task(server._ws_loop())
+            server._test_task = asyncio.create_task(server._test_event_loop())
+
+        @app.on_event("shutdown")
+        async def _shutdown():
+            server.monitor.stop()
+            if server._monitor_task:
+                server._monitor_task.cancel()
+            if server._heartbeat_task:
+                server._heartbeat_task.cancel()
+            if server._ws_task:
+                server._ws_task.cancel()
+            if hasattr(server, '_test_task') and server._test_task:
+                server._test_task.cancel()
+            if server._ws:
+                await server._ws.close()
+
+        @app.get("/health")
+        async def health():
+            return {
+                "status": "healthy",
+                "details": {
+                    "monitored_modules": len(server.monitor.modules),
+                    "event_hub_connected": server._ws is not None,
+                    "uptime_seconds": round(time.time() - server._start_time),
+                },
+            }
+
+        @app.get("/status")
+        async def status():
+            return server.monitor.get_status()
+
+        return app
+
+    # ── Event Hub WebSocket client ──
+
+    async def _ws_loop(self):
+        """Connect to Event Hub, subscribe, and listen. Reconnect on failure."""
+        while True:
+            try:
+                await self._ws_connect()
+            except asyncio.CancelledError:
+                return
+            except Exception as e:
+                print(f"[watchdog] Event Hub connection error: {e}")
+            self._ws = None
+            await asyncio.sleep(5)  # reconnect delay
+
+    async def _ws_connect(self):
+        """Single WebSocket session: connect, subscribe, receive loop."""
+        url = f"{self.event_hub_ws}?token={self.token}"
+        async with websockets.connect(url) as ws:
+            self._ws = ws
+            print("[watchdog] Connected to Event Hub")
+
+            # Subscribe to module lifecycle events
+            await ws.send(json.dumps({
+                "type": "subscribe",
+                "events": ["module.started", "module.stopped"],
+            }))
+
+            # Receive loop
+            async for raw in ws:
+                try:
+                    msg = json.loads(raw)
+                except (json.JSONDecodeError, TypeError):
+                    continue
+
+                msg_type = msg.get("type", "")
+                if msg_type == "event":
+                    await self.monitor.handle_event(msg)
+                elif msg_type == "ack":
+                    pass  # publish confirmed
+                elif msg_type == "error":
+                    print(f"[watchdog] Event Hub error: {msg.get('message')}")
+
+    async def _publish_event(self, event: dict):
+        """Publish an event to Event Hub via WebSocket."""
+        if not self._ws:
+            return
+        from datetime import datetime, timezone
+        msg = {
+            "type": "event",
+            "event_id": str(uuid.uuid4()),
+            "event": event.get("event", ""),
+            "source": "watchdog",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "data": event.get("data", {}),
+        }
+        try:
+            await self._ws.send(json.dumps(msg))
+        except Exception as e:
+            print(f"[watchdog] Failed to publish event: {e}")
+
+    # ── Heartbeat to Registry ──
+
+    async def _heartbeat_loop(self):
+        """Send heartbeat to Registry every 30 seconds."""
+        while True:
+            await asyncio.sleep(30)
+            try:
+                async with httpx.AsyncClient() as client:
+                    await client.post(
+                        f"{self.monitor.registry_url}/modules",
+                        json={"action": "heartbeat", "module_id": "watchdog"},
+                        headers={"Authorization": f"Bearer {self.monitor.own_token}"},
+                        timeout=5,
+                    )
+            except Exception:
+                pass
+
+    async def _test_event_loop(self):
+        """Publish a test event every 5 seconds."""
+        from datetime import datetime, timezone
+        while True:
+            await asyncio.sleep(5)
+            await self._publish_event({
+                "event": "watchdog.test",
+                "data": {
+                    "message": "heartbeat from watchdog",
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                },
+            })

package/main.py

@@ -0,0 +1,17 @@
+import secrets
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from core.launcher.entry import Launcher
+
+
+def main():
+    token = secrets.token_hex(32)
+    print(f"[main] KITE_TOKEN generated ({len(token)} chars)")
+    Launcher(kite_token=token).run()
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@agentunion/kite",
+  "version": "1.0.0",
+  "description": "Kite framework launcher — start Kite from anywhere",
+  "bin": {
+    "kite": "./cli.js"
+  },
+  "files": [
+    "cli.js",
+    "main.py",
+    "__main__.py",
+    "__init__.py",
+    "core/**",
+    "extensions/**"
+  ],
+  "engines": {
+    "node": ">=16"
+  },
+  "os": ["win32", "linux", "darwin"],
+  "keywords": ["kite", "framework", "launcher"],
+  "license": "MIT",
+  "author": "agentcp",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/agentcp/kite.git"
+  }
+}