@agentunion/kite 1.0.7 → 1.2.0

package/core/event_hub/entry.py

@@ -1,20 +1,240 @@
 """
 Event Hub entry point.
-Reads token from stdin boot_info, reads launcher_ws_token from stdin second line,
-starts FastAPI server, outputs ws_endpoint via stdout, waits for Launcher to connect,
-then registers to Registry after stdio disconnect.
+Reads token from stdin boot_info, then reads launcher_ws_token + registry_port from
+stdin kite messages (in any order), starts FastAPI server, outputs ws_endpoint via
+stdout, waits for Launcher to connect, then registers to Registry after stdio disconnect.
 """
 
+import builtins
 import json
 import os
 import re
 import socket
 import sys
 import threading
+import time
+import traceback
+from datetime import datetime, timezone
 
 import uvicorn
 
-# Ensure project root is on sys.path (set by main.py or cli.js)
+
+# ── Module configuration ──
+MODULE_NAME = "event_hub"
+
+
+def _fmt_elapsed(t0: float) -> str:
+    """Format elapsed time since t0: <1s → 'NNNms', >=1s → 'N.Ns', >=10s → 'NNs'."""
+    d = time.monotonic() - t0
+    if d < 1:
+        return f"{d * 1000:.0f}ms"
+    if d < 10:
+        return f"{d:.1f}s"
+    return f"{d:.0f}s"
+
+
+# ── Safe stdout/stderr: ignore BrokenPipeError after Launcher closes stdio ──
+
+class _SafeWriter:
+    """Wraps a stream to silently swallow BrokenPipeError on write/flush."""
+    def __init__(self, stream):
+        self._stream = stream
+
+    def write(self, s):
+        try:
+            self._stream.write(s)
+        except (BrokenPipeError, OSError):
+            pass
+
+    def flush(self):
+        try:
+            self._stream.flush()
+        except (BrokenPipeError, OSError):
+            pass
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+sys.stdout = _SafeWriter(sys.stdout)
+sys.stderr = _SafeWriter(sys.stderr)
+
+
+# ── Timestamped print + log file writer ──
+# Independent implementation per module (no shared code dependency)
+
+_builtin_print = builtins.print
+_start_ts = time.monotonic()
+_last_ts = time.monotonic()
+_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
+_log_lock = threading.Lock()
+_log_latest_path = None
+_log_daily_path = None
+_log_daily_date = ""
+_log_dir = None
+_crash_log_path = None
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+def _resolve_daily_log_path():
+    """Resolve daily log path based on current date."""
+    global _log_daily_path, _log_daily_date
+    if not _log_dir:
+        return
+    today = datetime.now().strftime("%Y-%m-%d")
+    if today == _log_daily_date and _log_daily_path:
+        return
+    month_dir = os.path.join(_log_dir, today[:7])
+    os.makedirs(month_dir, exist_ok=True)
+    _log_daily_path = os.path.join(month_dir, f"{today}.log")
+    _log_daily_date = today
+
+def _write_log(plain_line: str):
+    """Write a plain-text line to both latest.log and daily log."""
+    with _log_lock:
+        if _log_latest_path:
+            try:
+                with open(_log_latest_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+        _resolve_daily_log_path()
+        if _log_daily_path:
+            try:
+                with open(_log_daily_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+
+def _write_crash(exc_type, exc_value, exc_tb, thread_name=None, severity="critical", handled=False):
+    """Write crash record to crashes.jsonl + daily crash archive."""
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "module": MODULE_NAME,
+        "thread": thread_name or threading.current_thread().name,
+        "exception_type": exc_type.__name__ if exc_type else "Unknown",
+        "exception_message": str(exc_value),
+        "traceback": "".join(traceback.format_exception(exc_type, exc_value, exc_tb)),
+        "severity": severity,
+        "handled": handled,
+        "process_id": os.getpid(),
+        "platform": sys.platform,
+        "runtime_version": f"Python {sys.version.split()[0]}",
+    }
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            record["context"] = {
+                "function": last.name,
+                "file": os.path.basename(last.filename),
+                "line": last.lineno,
+            }
+
+    line = json.dumps(record, ensure_ascii=False) + "\n"
+
+    if _crash_log_path:
+        try:
+            with open(_crash_log_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+    if _log_dir:
+        try:
+            today = datetime.now().strftime("%Y-%m-%d")
+            archive_dir = os.path.join(_log_dir, "crashes", today[:7])
+            os.makedirs(archive_dir, exist_ok=True)
+            archive_path = os.path.join(archive_dir, f"{today}.jsonl")
+            with open(archive_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+def _print_crash_summary(exc_type, exc_tb, thread_name=None):
+    """Print crash summary to console (red highlight)."""
+    RED = "\033[91m"
+    RESET = "\033[0m"
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            location = f"{os.path.basename(last.filename)}:{last.lineno}"
+        else:
+            location = "unknown"
+    else:
+        location = "unknown"
+
+    prefix = f"[{MODULE_NAME}]"
+    if thread_name:
+        _builtin_print(f"{prefix} {RED}线程 {thread_name} 崩溃: "
+                       f"{exc_type.__name__} in {location}{RESET}")
+    else:
+        _builtin_print(f"{prefix} {RED}崩溃: {exc_type.__name__} in {location}{RESET}")
+    if _crash_log_path:
+        _builtin_print(f"{prefix} 崩溃日志: {_crash_log_path}")
+
+def _setup_exception_hooks():
+    """Set up global exception hooks."""
+    _orig_excepthook = sys.excepthook
+
+    def _excepthook(exc_type, exc_value, exc_tb):
+        _write_crash(exc_type, exc_value, exc_tb, severity="critical", handled=False)
+        _print_crash_summary(exc_type, exc_tb)
+        _orig_excepthook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _excepthook
+
+    if hasattr(threading, "excepthook"):
+        def _thread_excepthook(args):
+            _write_crash(args.exc_type, args.exc_value, args.exc_traceback,
+                        thread_name=args.thread.name if args.thread else "unknown",
+                        severity="error", handled=False)
+            _print_crash_summary(args.exc_type, args.exc_traceback,
+                                thread_name=args.thread.name if args.thread else None)
+
+        threading.excepthook = _thread_excepthook
+
+def _tprint(*args, **kwargs):
+    """Timestamped print that adds [timestamp] HH:MM:SS.mmm +delta prefix."""
+    global _last_ts
+    now = time.monotonic()
+    elapsed = now - _start_ts
+    delta = now - _last_ts
+    _last_ts = now
+
+    if elapsed < 1:
+        elapsed_str = f"{elapsed * 1000:.0f}ms"
+    elif elapsed < 100:
+        elapsed_str = f"{elapsed:.1f}s"
+    else:
+        elapsed_str = f"{elapsed:.0f}s"
+
+    if delta < 0.001:
+        delta_str = ""
+    elif delta < 1:
+        delta_str = f"+{delta * 1000:.0f}ms"
+    elif delta < 100:
+        delta_str = f"+{delta:.1f}s"
+    else:
+        delta_str = f"+{delta:.0f}s"
+
+    ts = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+
+    _builtin_print(*args, **kwargs)
+
+    if _log_latest_path or _log_daily_path:
+        sep = kwargs.get("sep", " ")
+        end = kwargs.get("end", "\n")
+        text = sep.join(str(a) for a in args)
+        prefix = f"[{elapsed_str:>6}] {ts} {delta_str:>8} "
+        _write_log(prefix + _strip_ansi(text) + end)
+
+builtins.print = _tprint
+
+# Ensure project root is on sys.path
 _this_dir = os.path.dirname(os.path.abspath(__file__))
 _project_root = os.environ.get("KITE_PROJECT") or os.path.dirname(os.path.dirname(_this_dir))
 if _project_root not in sys.path:
@@ -59,29 +279,68 @@ def _bind_port(preferred: int, host: str) -> int:
         return s.getsockname()[1]
 
 
-def _read_stdin_kite_message() -> dict | None:
-    """Read a structured kite message from stdin (second line after boot_info).
-    Uses a short timeout thread to avoid blocking forever if Launcher doesn't send.
+def _read_stdin_kite_messages(expected: set[str], timeout: float = 10) -> dict[str, dict]:
+    """Read multiple structured kite messages from stdin until all expected types received.
+
+    Args:
+        expected: set of kite message types to wait for (e.g. {"launcher_ws_token", "registry_port"})
+        timeout: total timeout in seconds
+
+    Returns:
+        dict mapping kite type -> message dict. Missing types are absent from the result.
     """
-    result = [None]
+    collected: dict[str, dict] = {}
+    remaining = set(expected)
 
     def _read():
-        try:
-            line = sys.stdin.readline().strip()
-            if line:
+        while remaining:
+            try:
+                line = sys.stdin.readline().strip()
+                if not line:
+                    return  # stdin closed
                 msg = json.loads(line)
                 if isinstance(msg, dict) and "kite" in msg:
-                    result[0] = msg
-        except Exception:
-            pass
+                    kite_type = msg["kite"]
+                    collected[kite_type] = msg
+                    remaining.discard(kite_type)
+            except Exception:
+                return  # parse error or stdin closed
 
     t = threading.Thread(target=_read, daemon=True)
     t.start()
-    t.join(timeout=5)
-    return result[0]
+    t.join(timeout=timeout)
+    return collected
 
 
 def main():
+    # Initialize log file paths
+    global _log_dir, _log_latest_path, _crash_log_path
+    module_data = os.environ.get("KITE_MODULE_DATA")
+    if module_data:
+        _log_dir = os.path.join(module_data, "log")
+        os.makedirs(_log_dir, exist_ok=True)
+        suffix = os.environ.get("KITE_INSTANCE_SUFFIX", "")
+
+        _log_latest_path = os.path.join(_log_dir, f"latest{suffix}.log")
+        try:
+            with open(_log_latest_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _log_latest_path = None
+
+        _crash_log_path = os.path.join(_log_dir, f"crashes{suffix}.jsonl")
+        try:
+            with open(_crash_log_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _crash_log_path = None
+
+        _resolve_daily_log_path()
+
+    _setup_exception_hooks()
+
+    _t0 = time.monotonic()
+
     # Kite environment
     kite_instance = os.environ.get("KITE_INSTANCE", "")
     is_debug = os.environ.get("KITE_DEBUG") == "1"
@@ -100,24 +359,36 @@ def main():
         print("[event_hub] 错误: boot_info 中缺少令牌")
         sys.exit(1)
 
-    # Step 2: Read launcher_ws_token from stdin (second line, structured kite message)
+    # Step 2: Check env for registry_port first (fast path for restart / Phase 3.5+ scenarios)
+    registry_port = int(os.environ.get("KITE_REGISTRY_PORT", "0"))
+
+    # Determine which stdin messages to wait for
+    stdin_expected = {"launcher_ws_token"}
+    if not registry_port:
+        stdin_expected.add("registry_port")
+
+    kite_msgs = _read_stdin_kite_messages(stdin_expected, timeout=10)
+
     launcher_ws_token = ""
-    kite_msg = _read_stdin_kite_message()
-    if kite_msg and kite_msg.get("kite") == "launcher_ws_token":
-        launcher_ws_token = kite_msg.get("launcher_ws_token", "")
+    ws_msg = kite_msgs.get("launcher_ws_token")
+    if ws_msg:
+        launcher_ws_token = ws_msg.get("launcher_ws_token", "")
 
     if launcher_ws_token:
         print(f"[event_hub] 已收到启动器 WS 令牌 ({len(launcher_ws_token)} 字符)")
     else:
         print("[event_hub] 警告: 未收到 launcher_ws_token，启动器引导认证已禁用")
 
-    # Step 3: Read registry_port from environment variable
-    registry_port = int(os.environ.get("KITE_REGISTRY_PORT", "0"))
+    # Step 3: registry_port — env already had it, or read from stdin
+    if not registry_port:
+        port_msg = kite_msgs.get("registry_port")
+        if port_msg:
+            registry_port = int(port_msg.get("registry_port", 0))
     if not registry_port:
-        print("[event_hub] 错误: KITE_REGISTRY_PORT 未设置")
+        print("[event_hub] 错误: 未收到 registry_port（stdin 和环境变量均无）")
         sys.exit(1)
 
-    print(f"[event_hub] 已收到令牌 ({len(token)} 字符)，Registry 端口: {registry_port}")
+    print(f"[event_hub] 已收到令牌 ({len(token)} 字符)，Registry 端口: {registry_port} ({_fmt_elapsed(_t0)})")
 
     # Step 4: Read config from own module.md
     md_config = _read_module_md()
@@ -149,8 +420,16 @@ def main():
     # Step 7: Start HTTP + WS server
     # Launcher will connect with launcher_ws_token → Event Hub sends module.ready → stdio disconnect
     # After stdio disconnect, Event Hub registers to Registry (done by server on_launcher_connected callback)
-    print(f"[event_hub] 启动中 {bind_host}:{port}")
-    uvicorn.run(server.app, host=bind_host, port=port, log_level="warning")
+    print(f"[event_hub] 启动中 {bind_host}:{port} ({_fmt_elapsed(_t0)})")
+    try:
+        config = uvicorn.Config(server.app, host=bind_host, port=port, log_level="warning")
+        uvi_server = uvicorn.Server(config)
+        server._uvicorn_server = uvi_server
+        uvi_server.run()
+    except Exception as e:
+        _write_crash(type(e), e, e.__traceback__, severity="critical", handled=True)
+        _print_crash_summary(type(e), e.__traceback__)
+        sys.exit(1)
 
 
 if __name__ == "__main__":

package/core/event_hub/hub.py

@@ -55,6 +55,8 @@ class EventHub:
         self._cnt_dedup = 0
         self._cnt_errors = 0
         self._start_time = time.time()
+        # Shutdown callback: set by EventHubServer to handle own shutdown
+        self._shutdown_cb = None
 
     # ── Connection lifecycle ──
 
@@ -186,6 +188,12 @@ class EventHub:
         await self._route_event(module_id, msg)
         await self._send_ack(ws, event_id)
 
+        # Check if this is a shutdown event targeting event_hub
+        if event_type == "module.shutdown" and self._shutdown_cb:
+            data = msg.get("data", {})
+            if data.get("module_id") == "event_hub":
+                asyncio.create_task(self._shutdown_cb(data))
+
     # ── Routing ──
 
     async def _route_event(self, sender_id: str, msg: dict):

package/core/event_hub/server.py

@@ -5,7 +5,7 @@ FastAPI app: /ws (WebSocket), /health, /stats.
 
 Launcher bootstrap sequence:
   Launcher connects with launcher_ws_token → Event Hub verifies locally →
-  sends module.ready → registers to Registry.
+  registers to Registry → sends module.ready.
 """
 
 import asyncio
@@ -35,7 +35,12 @@ class EventHubServer:
         self._timer_task: asyncio.Task | None = None
         self._launcher_connected = False
         self._registered_to_registry = False
+        self._uvicorn_server = None  # set by entry.py for graceful shutdown
+        self._launcher_ws: WebSocket | None = None  # reference to Launcher's WS
+        self._http_client: httpx.AsyncClient | None = None  # reused for Registry calls
         self.app = self._create_app()
+        # Register shutdown callback on hub
+        self.hub._shutdown_cb = self._handle_shutdown
 
     # ── Token verification ──
 
@@ -49,18 +54,22 @@ class EventHubServer:
         if self.launcher_ws_token and token == self.launcher_ws_token:
             return "launcher"
         # Normal verification via Registry
+        import time as _time
+        _t0 = _time.monotonic()
         try:
-            async with httpx.AsyncClient() as client:
-                resp = await client.post(
-                    f"{self.registry_url}/verify",
-                    json={"token": token},
-                    headers={"Authorization": f"Bearer {self.own_token}"},
-                    timeout=5,
-                )
-                if resp.status_code == 200:
-                    body = resp.json()
-                    if body.get("ok"):
-                        return body.get("module_id")
+            resp = await self._http_client.post(
+                f"{self.registry_url}/verify",
+                json={"token": token},
+                headers={"Authorization": f"Bearer {self.own_token}"},
+                timeout=5,
+            )
+            _t1 = _time.monotonic()
+            if resp.status_code == 200:
+                body = resp.json()
+                if body.get("ok"):
+                    mid = body.get("module_id")
+                    print(f"[event_hub] _verify_token({mid}): {(_t1-_t0)*1000:.0f}ms")
+                    return mid
         except Exception as e:
             print(f"[event_hub] Token verification failed: {e}")
         return None
@@ -68,8 +77,13 @@ class EventHubServer:
     # ── Launcher bootstrap ──
 
     async def _on_launcher_connected(self, ws: WebSocket):
-        """Called on first Launcher WS connect. Sends module.ready, then registers to Registry."""
+        """Called on first Launcher WS connect. Registers to Registry first, then sends module.ready.
+        This order ensures all startup-phase prints are done before Launcher calls close_stdio."""
         self._launcher_connected = True
+        self._launcher_ws = ws
+        # Step 1: Register to Registry (may print log lines)
+        await self._register_to_registry()
+        # Step 2: Send module.ready (Launcher will close_stdio after receiving this)
         msg = {
             "type": "event",
             "event_id": str(uuid.uuid4()),
@@ -79,6 +93,7 @@ class EventHubServer:
             "data": {
                 "module_id": "event_hub",
                 "ws_endpoint": f"ws://{self.advertise_ip}:{self.port}/ws",
+                "graceful_shutdown": True,
             },
         }
         try:
@@ -86,8 +101,48 @@ class EventHubServer:
             print("[event_hub] Sent module.ready to Launcher")
         except Exception as e:
             print(f"[event_hub] Failed to send module.ready: {e}")
-        # Register to Registry in background
-        asyncio.create_task(self._register_to_registry())
+
+    async def _handle_shutdown(self, data: dict):
+        """Handle module.shutdown event targeting event_hub."""
+        print("[event_hub] Received shutdown request")
+        # Send ack and ready via Launcher's WS connection
+        ws = self._launcher_ws
+        if ws:
+            try:
+                ack_msg = {
+                    "type": "event",
+                    "event_id": str(uuid.uuid4()),
+                    "event": "module.shutdown.ack",
+                    "source": "event_hub",
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "data": {"module_id": "event_hub", "estimated_cleanup": 2},
+                }
+                await ws.send_text(json.dumps(ack_msg))
+            except Exception as e:
+                print(f"[event_hub] Failed to send shutdown ack: {e}")
+        # Cleanup
+        if self._timer_task:
+            self._timer_task.cancel()
+        if self._http_client:
+            await self._http_client.aclose()
+        # Send ready
+        if ws:
+            try:
+                ready_msg = {
+                    "type": "event",
+                    "event_id": str(uuid.uuid4()),
+                    "event": "module.shutdown.ready",
+                    "source": "event_hub",
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "data": {"module_id": "event_hub"},
+                }
+                await ws.send_text(json.dumps(ready_msg))
+            except Exception as e:
+                print(f"[event_hub] Failed to send shutdown ready: {e}")
+        print("[event_hub] Shutdown ready, exiting")
+        # Trigger uvicorn exit
+        if self._uvicorn_server:
+            self._uvicorn_server.should_exit = True
 
     async def _register_to_registry(self):
         """Register to Registry. Triggered after Launcher connects."""
@@ -129,6 +184,8 @@ class EventHubServer:
 
         @app.on_event("startup")
         async def _startup():
+            # Initialize HTTP client early to avoid blocking first token verification
+            server._http_client = httpx.AsyncClient()
             server._timer_task = asyncio.create_task(server._timer_loop())
 
         @app.on_event("shutdown")
@@ -148,7 +205,10 @@ class EventHubServer:
                 # causing websockets 15.x clients to get "no close frame received or sent" errors.
                 await ws.accept()
                 print(f"[event_hub] 认证失败: token={token[:8]}... hint={mid_hint}")
-                await ws.close(code=4001, reason="Authentication failed")
+                try:
+                    await ws.close(code=4001, reason="Authentication failed")
+                except Exception:
+                    pass
                 return
 
             await ws.accept()
@@ -168,7 +228,10 @@ class EventHubServer:
             except WebSocketDisconnect:
                 pass
             except Exception as e:
-                print(f"[event_hub] WebSocket error for {module_id}: {e}")
+                # Connection-state errors are normal during shutdown
+                err = str(e).lower()
+                if "not connected" not in err and "closed" not in err:
+                    print(f"[event_hub] WebSocket error for {module_id}: {e}")
             finally:
                 server.hub.remove_connection(module_id)