@agentunion/kite 1.0.6 → 1.2.0

package/core/event_hub/entry.py

@@ -1,112 +1,435 @@
 """
 Event Hub entry point.
-Reads boot_info from stdin, starts FastAPI server, registers to Registry.
+Reads token from stdin boot_info, then reads launcher_ws_token + registry_port from
+stdin kite messages (in any order), starts FastAPI server, outputs ws_endpoint via
+stdout, waits for Launcher to connect, then registers to Registry after stdio disconnect.
 """
 
+import builtins
 import json
 import os
+import re
 import socket
 import sys
+import threading
+import time
+import traceback
+from datetime import datetime, timezone
 
-import httpx
 import uvicorn
 
-# Ensure project root is on sys.path
-_this_dir = os.path.dirname(os.path.abspath(__file__))
-_project_root = os.path.dirname(os.path.dirname(_this_dir))
-if _project_root not in sys.path:
-    sys.path.insert(0, _project_root)
 
-from core.event_hub.hub import EventHub
-from core.event_hub.server import EventHubServer
+# ── Module configuration ──
+MODULE_NAME = "event_hub"
 
 
-def _get_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("127.0.0.1", 0))
-        return s.getsockname()[1]
+def _fmt_elapsed(t0: float) -> str:
+    """Format elapsed time since t0: <1s → 'NNNms', >=1s → 'N.Ns', >=10s → 'NNs'."""
+    d = time.monotonic() - t0
+    if d < 1:
+        return f"{d * 1000:.0f}ms"
+    if d < 10:
+        return f"{d:.1f}s"
+    return f"{d:.0f}s"
+
+
+# ── Safe stdout/stderr: ignore BrokenPipeError after Launcher closes stdio ──
+
+class _SafeWriter:
+    """Wraps a stream to silently swallow BrokenPipeError on write/flush."""
+    def __init__(self, stream):
+        self._stream = stream
 
+    def write(self, s):
+        try:
+            self._stream.write(s)
+        except (BrokenPipeError, OSError):
+            pass
 
-def _register_to_registry(token: str, registry_url: str, port: int, advertise_ip: str = "127.0.0.1"):
-    """Synchronous registration to Registry at startup."""
-    payload = {
-        "action": "register",
-        "module_id": "event_hub",
-        "module_type": "infrastructure",
-        "name": "Event Hub",
-        "api_endpoint": f"http://{advertise_ip}:{port}",
-        "health_endpoint": "/health",
-        "metadata": {
-            "ws_endpoint": f"ws://{advertise_ip}:{port}/ws",
-        },
+    def flush(self):
+        try:
+            self._stream.flush()
+        except (BrokenPipeError, OSError):
+            pass
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+sys.stdout = _SafeWriter(sys.stdout)
+sys.stderr = _SafeWriter(sys.stderr)
+
+
+# ── Timestamped print + log file writer ──
+# Independent implementation per module (no shared code dependency)
+
+_builtin_print = builtins.print
+_start_ts = time.monotonic()
+_last_ts = time.monotonic()
+_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
+_log_lock = threading.Lock()
+_log_latest_path = None
+_log_daily_path = None
+_log_daily_date = ""
+_log_dir = None
+_crash_log_path = None
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+def _resolve_daily_log_path():
+    """Resolve daily log path based on current date."""
+    global _log_daily_path, _log_daily_date
+    if not _log_dir:
+        return
+    today = datetime.now().strftime("%Y-%m-%d")
+    if today == _log_daily_date and _log_daily_path:
+        return
+    month_dir = os.path.join(_log_dir, today[:7])
+    os.makedirs(month_dir, exist_ok=True)
+    _log_daily_path = os.path.join(month_dir, f"{today}.log")
+    _log_daily_date = today
+
+def _write_log(plain_line: str):
+    """Write a plain-text line to both latest.log and daily log."""
+    with _log_lock:
+        if _log_latest_path:
+            try:
+                with open(_log_latest_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+        _resolve_daily_log_path()
+        if _log_daily_path:
+            try:
+                with open(_log_daily_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+
+def _write_crash(exc_type, exc_value, exc_tb, thread_name=None, severity="critical", handled=False):
+    """Write crash record to crashes.jsonl + daily crash archive."""
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "module": MODULE_NAME,
+        "thread": thread_name or threading.current_thread().name,
+        "exception_type": exc_type.__name__ if exc_type else "Unknown",
+        "exception_message": str(exc_value),
+        "traceback": "".join(traceback.format_exception(exc_type, exc_value, exc_tb)),
+        "severity": severity,
+        "handled": handled,
+        "process_id": os.getpid(),
+        "platform": sys.platform,
+        "runtime_version": f"Python {sys.version.split()[0]}",
     }
-    headers = {"Authorization": f"Bearer {token}"}
-    resp = httpx.post(
-        f"{registry_url}/modules",
-        json=payload,
-        headers=headers,
-        timeout=5,
-    )
-    if resp.status_code == 200:
-        print(f"[event_hub] Registered to Registry")
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            record["context"] = {
+                "function": last.name,
+                "file": os.path.basename(last.filename),
+                "line": last.lineno,
+            }
+
+    line = json.dumps(record, ensure_ascii=False) + "\n"
+
+    if _crash_log_path:
+        try:
+            with open(_crash_log_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+    if _log_dir:
+        try:
+            today = datetime.now().strftime("%Y-%m-%d")
+            archive_dir = os.path.join(_log_dir, "crashes", today[:7])
+            os.makedirs(archive_dir, exist_ok=True)
+            archive_path = os.path.join(archive_dir, f"{today}.jsonl")
+            with open(archive_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+def _print_crash_summary(exc_type, exc_tb, thread_name=None):
+    """Print crash summary to console (red highlight)."""
+    RED = "\033[91m"
+    RESET = "\033[0m"
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            location = f"{os.path.basename(last.filename)}:{last.lineno}"
+        else:
+            location = "unknown"
+    else:
+        location = "unknown"
+
+    prefix = f"[{MODULE_NAME}]"
+    if thread_name:
+        _builtin_print(f"{prefix} {RED}线程 {thread_name} 崩溃: "
+                       f"{exc_type.__name__} in {location}{RESET}")
+    else:
+        _builtin_print(f"{prefix} {RED}崩溃: {exc_type.__name__} in {location}{RESET}")
+    if _crash_log_path:
+        _builtin_print(f"{prefix} 崩溃日志: {_crash_log_path}")
+
+def _setup_exception_hooks():
+    """Set up global exception hooks."""
+    _orig_excepthook = sys.excepthook
+
+    def _excepthook(exc_type, exc_value, exc_tb):
+        _write_crash(exc_type, exc_value, exc_tb, severity="critical", handled=False)
+        _print_crash_summary(exc_type, exc_tb)
+        _orig_excepthook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _excepthook
+
+    if hasattr(threading, "excepthook"):
+        def _thread_excepthook(args):
+            _write_crash(args.exc_type, args.exc_value, args.exc_traceback,
+                        thread_name=args.thread.name if args.thread else "unknown",
+                        severity="error", handled=False)
+            _print_crash_summary(args.exc_type, args.exc_traceback,
+                                thread_name=args.thread.name if args.thread else None)
+
+        threading.excepthook = _thread_excepthook
+
+def _tprint(*args, **kwargs):
+    """Timestamped print that adds [timestamp] HH:MM:SS.mmm +delta prefix."""
+    global _last_ts
+    now = time.monotonic()
+    elapsed = now - _start_ts
+    delta = now - _last_ts
+    _last_ts = now
+
+    if elapsed < 1:
+        elapsed_str = f"{elapsed * 1000:.0f}ms"
+    elif elapsed < 100:
+        elapsed_str = f"{elapsed:.1f}s"
+    else:
+        elapsed_str = f"{elapsed:.0f}s"
+
+    if delta < 0.001:
+        delta_str = ""
+    elif delta < 1:
+        delta_str = f"+{delta * 1000:.0f}ms"
+    elif delta < 100:
+        delta_str = f"+{delta:.1f}s"
     else:
-        print(f"[event_hub] WARNING: Registry returned {resp.status_code}")
+        delta_str = f"+{delta:.0f}s"
+
+    ts = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+
+    _builtin_print(*args, **kwargs)
 
+    if _log_latest_path or _log_daily_path:
+        sep = kwargs.get("sep", " ")
+        end = kwargs.get("end", "\n")
+        text = sep.join(str(a) for a in args)
+        prefix = f"[{elapsed_str:>6}] {ts} {delta_str:>8} "
+        _write_log(prefix + _strip_ansi(text) + end)
 
-def _read_test_mode() -> bool:
-    """Read test_mode from module.md frontmatter."""
+builtins.print = _tprint
+
+# Ensure project root is on sys.path
+_this_dir = os.path.dirname(os.path.abspath(__file__))
+_project_root = os.environ.get("KITE_PROJECT") or os.path.dirname(os.path.dirname(_this_dir))
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from core.event_hub.hub import EventHub
+from core.event_hub.server import EventHubServer
+
+
+def _read_module_md() -> dict:
+    """Read preferred_port, advertise_ip from own module.md."""
     md_path = os.path.join(_this_dir, "module.md")
+    result = {"preferred_port": 0, "advertise_ip": "127.0.0.1"}
     try:
         with open(md_path, encoding="utf-8") as f:
             text = f.read()
-        import re, yaml
         m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
         if m:
-            fm = yaml.safe_load(m.group(1)) or {}
-            return bool(fm.get("test_mode", False))
+            try:
+                import yaml
+                fm = yaml.safe_load(m.group(1)) or {}
+            except ImportError:
+                fm = {}
+            result["preferred_port"] = int(fm.get("preferred_port", 0))
+            result["advertise_ip"] = fm.get("advertise_ip", "127.0.0.1")
     except Exception:
         pass
-    return False
+    return result
+
+
+def _bind_port(preferred: int, host: str) -> int:
+    """Try preferred port first, fall back to OS-assigned."""
+    if preferred:
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind((host, preferred))
+                return preferred
+        except OSError:
+            pass
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind((host, 0))
+        return s.getsockname()[1]
+
+
+def _read_stdin_kite_messages(expected: set[str], timeout: float = 10) -> dict[str, dict]:
+    """Read multiple structured kite messages from stdin until all expected types received.
+
+    Args:
+        expected: set of kite message types to wait for (e.g. {"launcher_ws_token", "registry_port"})
+        timeout: total timeout in seconds
+
+    Returns:
+        dict mapping kite type -> message dict. Missing types are absent from the result.
+    """
+    collected: dict[str, dict] = {}
+    remaining = set(expected)
+
+    def _read():
+        while remaining:
+            try:
+                line = sys.stdin.readline().strip()
+                if not line:
+                    return  # stdin closed
+                msg = json.loads(line)
+                if isinstance(msg, dict) and "kite" in msg:
+                    kite_type = msg["kite"]
+                    collected[kite_type] = msg
+                    remaining.discard(kite_type)
+            except Exception:
+                return  # parse error or stdin closed
+
+    t = threading.Thread(target=_read, daemon=True)
+    t.start()
+    t.join(timeout=timeout)
+    return collected
 
 
 def main():
-    # Read boot_info from stdin
+    # Initialize log file paths
+    global _log_dir, _log_latest_path, _crash_log_path
+    module_data = os.environ.get("KITE_MODULE_DATA")
+    if module_data:
+        _log_dir = os.path.join(module_data, "log")
+        os.makedirs(_log_dir, exist_ok=True)
+        suffix = os.environ.get("KITE_INSTANCE_SUFFIX", "")
+
+        _log_latest_path = os.path.join(_log_dir, f"latest{suffix}.log")
+        try:
+            with open(_log_latest_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _log_latest_path = None
+
+        _crash_log_path = os.path.join(_log_dir, f"crashes{suffix}.jsonl")
+        try:
+            with open(_crash_log_path, "w", encoding="utf-8") as f:
+                pass
+        except Exception:
+            _crash_log_path = None
+
+        _resolve_daily_log_path()
+
+    _setup_exception_hooks()
+
+    _t0 = time.monotonic()
+
+    # Kite environment
+    kite_instance = os.environ.get("KITE_INSTANCE", "")
+    is_debug = os.environ.get("KITE_DEBUG") == "1"
+
+    # Step 1: Read token from stdin boot_info
     token = ""
-    registry_port = 0
-    bind_host = "127.0.0.1"
-    advertise_ip = "127.0.0.1"
     try:
         line = sys.stdin.readline().strip()
         if line:
             boot_info = json.loads(line)
             token = boot_info.get("token", "")
-            registry_port = boot_info.get("registry_port", 0)
-            bind_host = boot_info.get("bind", "127.0.0.1")
-            advertise_ip = boot_info.get("advertise_ip", "127.0.0.1")
     except Exception:
         pass
 
-    if not token or not registry_port:
-        print("[event_hub] ERROR: Missing token or registry_port in boot_info")
+    if not token:
+        print("[event_hub] 错误: boot_info 中缺少令牌")
+        sys.exit(1)
+
+    # Step 2: Check env for registry_port first (fast path for restart / Phase 3.5+ scenarios)
+    registry_port = int(os.environ.get("KITE_REGISTRY_PORT", "0"))
+
+    # Determine which stdin messages to wait for
+    stdin_expected = {"launcher_ws_token"}
+    if not registry_port:
+        stdin_expected.add("registry_port")
+
+    kite_msgs = _read_stdin_kite_messages(stdin_expected, timeout=10)
+
+    launcher_ws_token = ""
+    ws_msg = kite_msgs.get("launcher_ws_token")
+    if ws_msg:
+        launcher_ws_token = ws_msg.get("launcher_ws_token", "")
+
+    if launcher_ws_token:
+        print(f"[event_hub] 已收到启动器 WS 令牌 ({len(launcher_ws_token)} 字符)")
+    else:
+        print("[event_hub] 警告: 未收到 launcher_ws_token，启动器引导认证已禁用")
+
+    # Step 3: registry_port — env already had it, or read from stdin
+    if not registry_port:
+        port_msg = kite_msgs.get("registry_port")
+        if port_msg:
+            registry_port = int(port_msg.get("registry_port", 0))
+    if not registry_port:
+        print("[event_hub] 错误: 未收到 registry_port（stdin 和环境变量均无）")
         sys.exit(1)
 
-    print(f"[event_hub] Token received ({len(token)} chars), registry port: {registry_port}")
+    print(f"[event_hub] 已收到令牌 ({len(token)} 字符)，Registry 端口: {registry_port} ({_fmt_elapsed(_t0)})")
 
+    # Step 4: Read config from own module.md
+    md_config = _read_module_md()
+    advertise_ip = md_config["advertise_ip"]
+    preferred_port = md_config["preferred_port"]
+
+    # Step 5: Bind port and create server
+    bind_host = advertise_ip
+    port = _bind_port(preferred_port, bind_host)
     registry_url = f"http://127.0.0.1:{registry_port}"
-    port = _get_free_port()
 
-    # Register to Registry
-    _register_to_registry(token, registry_url, port, advertise_ip)
+    if is_debug:
+        print("[event_hub] 调试模式已启用 (KITE_DEBUG=1)，接受所有令牌")
 
-    # Create hub and server
-    test_mode = _read_test_mode()
-    if test_mode:
-        print("[event_hub] WARNING: test_mode enabled, all tokens accepted")
     hub = EventHub()
-    server = EventHubServer(hub, own_token=token, registry_url=registry_url, test_mode=test_mode)
+    server = EventHubServer(
+        hub,
+        own_token=token,
+        registry_url=registry_url,
+        launcher_ws_token=launcher_ws_token,
+        advertise_ip=advertise_ip,
+        port=port,
+    )
+
+    # Step 6: Output ws_endpoint via stdout (Launcher reads this)
+    ws_endpoint = f"ws://{advertise_ip}:{port}/ws"
+    print(json.dumps({"kite": "ws_endpoint", "ws_endpoint": ws_endpoint}), flush=True)
 
-    print(f"[event_hub] Starting on {bind_host}:{port}")
-    uvicorn.run(server.app, host=bind_host, port=port, log_level="warning")
+    # Step 7: Start HTTP + WS server
+    # Launcher will connect with launcher_ws_token → Event Hub sends module.ready → stdio disconnect
+    # After stdio disconnect, Event Hub registers to Registry (done by server on_launcher_connected callback)
+    print(f"[event_hub] 启动中 {bind_host}:{port} ({_fmt_elapsed(_t0)})")
+    try:
+        config = uvicorn.Config(server.app, host=bind_host, port=port, log_level="warning")
+        uvi_server = uvicorn.Server(config)
+        server._uvicorn_server = uvi_server
+        uvi_server.run()
+    except Exception as e:
+        _write_crash(type(e), e, e.__traceback__, severity="critical", handled=True)
+        _print_crash_summary(type(e), e.__traceback__)
+        sys.exit(1)
 
 
 if __name__ == "__main__":

package/core/event_hub/hub.py

@@ -55,6 +55,8 @@ class EventHub:
         self._cnt_dedup = 0
         self._cnt_errors = 0
         self._start_time = time.time()
+        # Shutdown callback: set by EventHubServer to handle own shutdown
+        self._shutdown_cb = None
 
     # ── Connection lifecycle ──
 
@@ -186,6 +188,12 @@ class EventHub:
         await self._route_event(module_id, msg)
         await self._send_ack(ws, event_id)
 
+        # Check if this is a shutdown event targeting event_hub
+        if event_type == "module.shutdown" and self._shutdown_cb:
+            data = msg.get("data", {})
+            if data.get("module_id") == "event_hub":
+                asyncio.create_task(self._shutdown_cb(data))
+
     # ── Routing ──
 
     async def _route_event(self, sender_id: str, msg: dict):

package/core/event_hub/module.md

@@ -8,7 +8,6 @@ runtime: python
 entry: entry.py
 events: []
 subscriptions: []
-test_mode: true
 ---
 
 # Event Hub