@agentunion/kite 1.0.7 → 1.3.0

package/extensions/services/watchdog/entry.py

@@ -1,100 +1,302 @@
 """
 Watchdog entry point.
-Reads boot_info from stdin, registers to Registry, starts health monitor.
+Connects to Kernel via WebSocket JSON-RPC 2.0, registers, subscribes to events,
+runs health monitor loop, handles incoming RPC requests.
 """
 
+import asyncio
+import builtins
 import json
 import os
-import socket
+import re
 import sys
+import threading
+import time
+import traceback
+import uuid
+from datetime import datetime, timezone
 
-import httpx
-import uvicorn
+import websockets
 
-# Ensure project root is on sys.path (set by main.py or cli.js)
+
+# ── Module configuration ──
+MODULE_NAME = "watchdog"
+
+
+def _fmt_elapsed(t0: float) -> str:
+    """Format elapsed time since t0: <1s → 'NNNms', >=1s → 'N.Ns', >=10s → 'NNs'."""
+    d = time.monotonic() - t0
+    if d < 1:
+        return f"{d * 1000:.0f}ms"
+    if d < 10:
+        return f"{d:.1f}s"
+    return f"{d:.0f}s"
+
+
+# ── Safe stdout/stderr: ignore BrokenPipeError after Launcher closes stdio ──
+
+class _SafeWriter:
+    """Wraps a stream to silently swallow BrokenPipeError on write/flush."""
+    def __init__(self, stream):
+        self._stream = stream
+
+    def write(self, s):
+        try:
+            self._stream.write(s)
+        except (BrokenPipeError, OSError):
+            pass
+
+    def flush(self):
+        try:
+            self._stream.flush()
+        except (BrokenPipeError, OSError):
+            pass
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+sys.stdout = _SafeWriter(sys.stdout)
+sys.stderr = _SafeWriter(sys.stderr)
+
+
+# ── Timestamped print + log file writer ──
+
+_builtin_print = builtins.print
+_start_ts = time.monotonic()
+_last_ts = time.monotonic()
+_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
+_log_lock = threading.Lock()
+_log_latest_path = None
+_log_daily_path = None
+_log_daily_date = ""
+_log_dir = None
+_crash_log_path = None
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+def _resolve_daily_log_path():
+    """Resolve daily log path based on current date."""
+    global _log_daily_path, _log_daily_date
+    if not _log_dir:
+        return
+    today = datetime.now().strftime("%Y-%m-%d")
+    if today == _log_daily_date and _log_daily_path:
+        return
+    month_dir = os.path.join(_log_dir, today[:7])
+    os.makedirs(month_dir, exist_ok=True)
+    _log_daily_path = os.path.join(month_dir, f"{today}.log")
+    _log_daily_date = today
+
+def _write_log(plain_line: str):
+    """Write a plain-text line to both latest.log and daily log."""
+    with _log_lock:
+        if _log_latest_path:
+            try:
+                with open(_log_latest_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+        _resolve_daily_log_path()
+        if _log_daily_path:
+            try:
+                with open(_log_daily_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+
+
+def _write_crash(exc_type, exc_value, exc_tb, thread_name=None, severity="critical", handled=False):
+    """Write crash record to crashes.jsonl + daily crash archive."""
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "module": MODULE_NAME,
+        "thread": thread_name or threading.current_thread().name,
+        "exception_type": exc_type.__name__ if exc_type else "Unknown",
+        "exception_message": str(exc_value),
+        "traceback": "".join(traceback.format_exception(exc_type, exc_value, exc_tb)),
+        "severity": severity,
+        "handled": handled,
+        "process_id": os.getpid(),
+        "platform": sys.platform,
+        "runtime_version": f"Python {sys.version.split()[0]}",
+    }
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            record["context"] = {
+                "function": last.name,
+                "file": os.path.basename(last.filename),
+                "line": last.lineno,
+            }
+
+    line = json.dumps(record, ensure_ascii=False) + "\n"
+
+    if _crash_log_path:
+        try:
+            with open(_crash_log_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+    if _log_dir:
+        try:
+            today = datetime.now().strftime("%Y-%m-%d")
+            archive_dir = os.path.join(_log_dir, "crashes", today[:7])
+            os.makedirs(archive_dir, exist_ok=True)
+            archive_path = os.path.join(archive_dir, f"{today}.jsonl")
+            with open(archive_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+
+def _print_crash_summary(exc_type, exc_tb, thread_name=None):
+    """Print crash summary to console (red highlight)."""
+    RED = "\033[91m"
+    RESET = "\033[0m"
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            location = f"{os.path.basename(last.filename)}:{last.lineno}"
+        else:
+            location = "unknown"
+    else:
+        location = "unknown"
+
+    prefix = f"[{MODULE_NAME}]"
+    if thread_name:
+        _builtin_print(f"{prefix} {RED}线程 {thread_name} 崩溃: "
+                       f"{exc_type.__name__} in {location}{RESET}")
+    else:
+        _builtin_print(f"{prefix} {RED}崩溃: {exc_type.__name__} in {location}{RESET}")
+    if _crash_log_path:
+        _builtin_print(f"{prefix} 崩溃日志: {_crash_log_path}")
+
+def _setup_exception_hooks():
+    """Set up global exception hooks."""
+    _orig_excepthook = sys.excepthook
+
+    def _excepthook(exc_type, exc_value, exc_tb):
+        _write_crash(exc_type, exc_value, exc_tb, severity="critical", handled=False)
+        _print_crash_summary(exc_type, exc_tb)
+        _orig_excepthook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _excepthook
+
+    if hasattr(threading, "excepthook"):
+        def _thread_excepthook(args):
+            _write_crash(args.exc_type, args.exc_value, args.exc_traceback,
+                        thread_name=args.thread.name if args.thread else "unknown",
+                        severity="error", handled=False)
+            _print_crash_summary(args.exc_type, args.exc_traceback,
+                                thread_name=args.thread.name if args.thread else None)
+
+        threading.excepthook = _thread_excepthook
+
+def _tprint(*args, **kwargs):
+    """Timestamped print that adds [timestamp] HH:MM:SS.mmm +delta prefix."""
+    global _last_ts
+    now = time.monotonic()
+    elapsed = now - _start_ts
+    delta = now - _last_ts
+    _last_ts = now
+
+    if elapsed < 1:
+        elapsed_str = f"{elapsed * 1000:.0f}ms"
+    elif elapsed < 100:
+        elapsed_str = f"{elapsed:.1f}s"
+    else:
+        elapsed_str = f"{elapsed:.0f}s"
+
+    if delta < 0.001:
+        delta_str = ""
+    elif delta < 1:
+        delta_str = f"+{delta * 1000:.0f}ms"
+    elif delta < 100:
+        delta_str = f"+{delta:.1f}s"
+    else:
+        delta_str = f"+{delta:.0f}s"
+
+    ts = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+
+    _builtin_print(*args, **kwargs)
+
+    if _log_latest_path or _log_daily_path:
+        sep = kwargs.get("sep", " ")
+        end = kwargs.get("end", "\n")
+        text = sep.join(str(a) for a in args)
+        prefix = f"[{elapsed_str:>6}] {ts} {delta_str:>8} "
+        _write_log(prefix + _strip_ansi(text) + end)
+
+builtins.print = _tprint
+
+# Ensure project root is on sys.path
 _project_root = os.environ.get("KITE_PROJECT") or os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 if _project_root not in sys.path:
     sys.path.insert(0, _project_root)
 
 from extensions.services.watchdog.monitor import HealthMonitor
-from extensions.services.watchdog.server import WatchdogServer
-
-
-def _get_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("127.0.0.1", 0))
-        return s.getsockname()[1]
-
-
-def _register_to_registry(token: str, registry_url: str, port: int):
-    payload = {
-        "action": "register",
-        "module_id": "watchdog",
-        "module_type": "service",
-        "name": "Watchdog",
-        "api_endpoint": f"http://127.0.0.1:{port}",
-        "health_endpoint": "/health",
-        "events_publish": {
-            "watchdog.module.unhealthy": {"description": "Module failed health check"},
-            "watchdog.module.recovered": {"description": "Module recovered from unhealthy"},
-            "watchdog.alert": {"description": "Module restarted too many times"},
-        },
-        "events_subscribe": [
-            "module.started",
-            "module.stopped",
-        ],
-    }
-    headers = {"Authorization": f"Bearer {token}"}
-    resp = httpx.post(
-        f"{registry_url}/modules",
-        json=payload, headers=headers, timeout=5,
-    )
-    if resp.status_code == 200:
-        print("[watchdog] Registered to Registry")
-    else:
-        print(f"[watchdog] WARNING: Registry returned {resp.status_code}")
-
 
-def _get_launcher_url(token: str, registry_url: str) -> str:
-    """Discover Launcher API endpoint from Registry."""
-    headers = {"Authorization": f"Bearer {token}"}
-    try:
-        resp = httpx.get(
-            f"{registry_url}/get/launcher.api_endpoint",
-            headers=headers, timeout=5,
-        )
-        if resp.status_code == 200:
-            return resp.json()
-    except Exception:
-        pass
-    return ""
 
+def _read_stdin_kite_message(expected_type: str, timeout: float = 10) -> dict | None:
+    """Read a single kite message of expected type from stdin with timeout."""
+    result = [None]
 
-def _get_event_hub_ws(token: str, registry_url: str) -> str:
-    """Discover Event Hub WebSocket endpoint from Registry, with retry."""
-    import time
-    headers = {"Authorization": f"Bearer {token}"}
-    deadline = time.time() + 10
-    while time.time() < deadline:
+    def _read():
         try:
-            resp = httpx.get(
-                f"{registry_url}/get/event_hub.metadata.ws_endpoint",
-                headers=headers, timeout=3,
-            )
-            if resp.status_code == 200:
-                val = resp.json()
-                if val:
-                    return val
+            line = sys.stdin.readline().strip()
+            if line:
+                msg = json.loads(line)
+                if isinstance(msg, dict) and msg.get("kite") == expected_type:
+                    result[0] = msg
         except Exception:
             pass
-        time.sleep(1)
-    return ""
+
+    t = threading.Thread(target=_read, daemon=True)
+    t.start()
+    t.join(timeout=timeout)
+    return result[0]
+
+
+# Global WS reference for publish_event callback
+_ws_global = None
 
 
-def main():
-    # Kite environment
-    kite_instance = os.environ.get("KITE_INSTANCE", "")
-    is_debug = os.environ.get("KITE_DEBUG") == "1"
+
+async def main():
+    global _ws_global
+    # Initialize log file paths
+    global _log_dir, _log_latest_path, _crash_log_path
+    module_data = os.environ.get("KITE_MODULE_DATA")
+    if module_data:
+        _log_dir = os.path.join(module_data, "log")
+        os.makedirs(_log_dir, exist_ok=True)
+        suffix = os.environ.get("KITE_INSTANCE_SUFFIX", "")
+
+        _log_latest_path = os.path.join(_log_dir, f"latest{suffix}.log")
+        try:
+            with open(_log_latest_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _log_latest_path = None
+
+        _crash_log_path = os.path.join(_log_dir, f"crashes{suffix}.jsonl")
+        try:
+            with open(_crash_log_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _crash_log_path = None
+
+        _resolve_daily_log_path()
+
+    _setup_exception_hooks()
+
+    _t0 = time.monotonic()
 
     # Read boot_info from stdin (only token)
     token = ""
@@ -106,42 +308,206 @@ def main():
     except Exception:
         pass
 
-    # Read registry_port from environment variable
-    registry_port = int(os.environ.get("KITE_REGISTRY_PORT", "0"))
+    # Read kernel_port: env first (fast path), stdin fallback (parallel start)
+    kernel_port = int(os.environ.get("KITE_KERNEL_PORT", "0"))
+    if not kernel_port:
+        msg = _read_stdin_kite_message("kernel_port", timeout=10)
+        if msg:
+            kernel_port = int(msg.get("kernel_port", 0))
 
-    if not token or not registry_port:
-        print("[watchdog] ERROR: Missing token or KITE_REGISTRY_PORT")
+    if not token or not kernel_port:
+        print("[watchdog] ERROR: Missing token or kernel_port")
         sys.exit(1)
 
-    print(f"[watchdog] Token received ({len(token)} chars), registry port: {registry_port}")
+    print(f"[watchdog] Token received ({len(token)} chars), kernel port: {kernel_port} ({_fmt_elapsed(_t0)})")
 
-    registry_url = f"http://127.0.0.1:{registry_port}"
-    port = _get_free_port()
+    # Connect to Kernel WebSocket
+    ws_url = f"ws://127.0.0.1:{kernel_port}/ws?token={token}&id=watchdog"
+    print(f"[watchdog] Connecting to Kernel: {ws_url}")
 
-    # Register to Registry
-    _register_to_registry(token, registry_url, port)
+    try:
+        async with websockets.connect(ws_url, open_timeout=5, ping_interval=None, ping_timeout=None, close_timeout=10) as ws:
+            _ws_global = ws
+            print(f"[watchdog] Connected to Kernel ({_fmt_elapsed(_t0)})")
+
+            # Subscribe to events
+            await _rpc_call(ws, "event.subscribe", {
+                "events": [
+                    "system.ready",
+                    "module.started",
+                    "module.stopped",
+                    "module.exiting",
+                    "module.ready",
+                    "module.shutdown",
+                ],
+            })
+            print(f"[watchdog] Subscribed to events ({_fmt_elapsed(_t0)})")
+
+            # Register to Kernel Registry via RPC
+            await _rpc_call(ws, "registry.register", {
+                "module_id": "watchdog",
+                "module_type": "service",
+                "events_publish": {
+                    "watchdog.module.unhealthy": {},
+                    "watchdog.module.recovered": {},
+                    "watchdog.alert": {},
+                },
+                "events_subscribe": [
+                    "system.ready",
+                    "module.started",
+                    "module.stopped",
+                    "module.exiting",
+                    "module.ready",
+                    "module.shutdown",
+                ],
+            })
+            print(f"[watchdog] Registered to Kernel ({_fmt_elapsed(_t0)})")
 
-    # Discover Launcher URL
-    launcher_url = _get_launcher_url(token, registry_url)
-    if not launcher_url:
-        print("[watchdog] WARNING: Could not discover Launcher URL, restart disabled")
+            # Create monitor with RPC callback
+            monitor = HealthMonitor(
+                own_token=token,
+                kernel_port=kernel_port,
+            )
+            monitor.publish_event = lambda event: asyncio.create_task(_publish_event(ws, event))
+            monitor.rpc_call = lambda method, params: _rpc_call(ws, method, params)
+
+            # Publish module.ready
+            await _rpc_call(ws, "event.publish", {
+                "event_id": str(uuid.uuid4()),
+                "event": "module.ready",
+                "data": {
+                    "module_id": "watchdog",
+                    "graceful_shutdown": True,
+                },
+            })
+            print(f"[watchdog] module.ready published ({_fmt_elapsed(_t0)})")
 
-    # Discover Event Hub WebSocket URL
-    event_hub_ws = _get_event_hub_ws(token, registry_url)
-    if not event_hub_ws:
-        print("[watchdog] WARNING: Could not discover Event Hub WS, events disabled")
+            # Start monitor loop in background
+            monitor_task = asyncio.create_task(monitor.run())
 
-    # Create monitor and server
-    monitor = HealthMonitor(
-        own_token=token,
-        registry_url=registry_url,
-        launcher_url=launcher_url,
-    )
-    server = WatchdogServer(monitor, token=token, event_hub_ws=event_hub_ws)
+            # Message loop: handle incoming RPC + events
+            async for raw in ws:
+                try:
+                    msg = json.loads(raw)
+                except (json.JSONDecodeError, TypeError):
+                    continue
 
-    print(f"[watchdog] Starting on port {port}")
-    uvicorn.run(server.app, host="127.0.0.1", port=port, log_level="warning")
+                try:
+                    has_method = "method" in msg
+                    has_id = "id" in msg
+
+                    if has_method and not has_id:
+                        # Event Notification
+                        await _handle_event_notification(msg, monitor)
+                    elif has_method and has_id:
+                        # Incoming RPC request
+                        await _handle_rpc_request(ws, msg, monitor)
+                    # Ignore RPC responses (we don't await them in this simple impl)
+                except Exception as e:
+                    print(f"[watchdog] 消息处理异常（已忽略）: {e}")
+
+    except Exception as e:
+        _write_crash(type(e), e, e.__traceback__, severity="critical", handled=True)
+        _print_crash_summary(type(e), e.__traceback__)
+        sys.exit(1)
+
+
+
+async def _rpc_call(ws, method: str, params: dict = None):
+    """Send a JSON-RPC 2.0 request (fire-and-forget, no response awaited)."""
+    msg = {"jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": method}
+    if params:
+        msg["params"] = params
+    await ws.send(json.dumps(msg))
+
+
+async def _publish_event(ws, event: dict):
+    """Publish an event via RPC event.publish."""
+    await _rpc_call(ws, "event.publish", {
+        "event_id": str(uuid.uuid4()),
+        "event": event.get("event", ""),
+        "data": event.get("data", {}),
+    })
+
+
+async def _handle_event_notification(msg: dict, monitor: HealthMonitor):
+    """Handle an event notification (JSON-RPC 2.0 Notification with method='event')."""
+    params = msg.get("params", {})
+    event_type = params.get("event", "")
+    data = params.get("data", {})
+
+    # Special handling for module.shutdown targeting watchdog
+    if event_type == "module.shutdown" and data.get("module_id") == "watchdog":
+        await _handle_shutdown(monitor)
+        return
+
+    # Forward to monitor
+    await monitor.handle_event(msg)
+
+
+async def _handle_rpc_request(ws, msg: dict, monitor: HealthMonitor):
+    """Handle an incoming RPC request (watchdog.* methods)."""
+    rpc_id = msg.get("id", "")
+    method = msg.get("method", "")
+    params = msg.get("params", {})
+
+    handlers = {
+        "health": lambda p: _rpc_health(monitor),
+        "status": lambda p: _rpc_status(monitor),
+    }
+    handler = handlers.get(method)
+    if handler:
+        try:
+            result = await handler(params)
+            await ws.send(json.dumps({"jsonrpc": "2.0", "id": rpc_id, "result": result}))
+        except Exception as e:
+            await ws.send(json.dumps({
+                "jsonrpc": "2.0", "id": rpc_id,
+                "error": {"code": -32603, "message": str(e)},
+            }))
+    else:
+        await ws.send(json.dumps({
+            "jsonrpc": "2.0", "id": rpc_id,
+            "error": {"code": -32601, "message": f"Method not found: {method}"},
+        }))
+
+
+async def _rpc_health(monitor: HealthMonitor) -> dict:
+    """RPC handler for watchdog.health."""
+    return {
+        "status": "healthy",
+        "details": {
+            "monitored_modules": len(monitor.modules),
+            "uptime_seconds": round(time.time() - _start_ts),
+        },
+    }
+
+
+async def _rpc_status(monitor: HealthMonitor) -> dict:
+    """RPC handler for watchdog.status."""
+    return monitor.get_status()
+
+
+async def _handle_shutdown(monitor: HealthMonitor):
+    """Handle module.shutdown event — ack, cleanup, ready, exit."""
+    print("[watchdog] Received shutdown request")
+    # Step 1: Send ack
+    await _publish_event(_ws_global, {
+        "event": "module.shutdown.ack",
+        "data": {"module_id": "watchdog", "estimated_cleanup": 2},
+    })
+    # Step 2: Cleanup
+    monitor.stop()
+    # Step 3: Send ready
+    await _publish_event(_ws_global, {
+        "event": "module.shutdown.ready",
+        "data": {"module_id": "watchdog"},
+    })
+    print("[watchdog] Shutdown ready, exiting")
+    # Step 4: Exit
+    sys.exit(0)
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.run(main())
+

package/extensions/services/watchdog/module.md

@@ -13,6 +13,9 @@ events:
 subscriptions:
   - module.started
   - module.stopped
+  - module.exiting
+  - module.ready
+  - module.shutdown
 ---
 
 # Watchdog（保活模块）