@agentunion/kite 1.0.7 → 1.3.0

package/kernel/entry.py

@@ -0,0 +1,407 @@
+"""
+Kernel entry point.
+Reads boot_info token + tokens dict from stdin, starts unified WebSocket server,
+outputs port via stdout, self-registers, publishes module.ready.
+
+Stdin protocol:
+  Line 1: {"token": "launcher_kite_token"}                     (boot_info)
+  Line 2: {"kite": "tokens", "tokens": {"mod1":"tok1", ...}}   (token mapping)
+
+Stdout protocol:
+  {"kite": "port", "port": N}                                  (Kernel port)
+"""
+
+import builtins
+import json
+import os
+import re
+import sys
+import socket
+import threading
+import time
+import traceback
+from datetime import datetime, timezone
+
+import uvicorn
+
+
+# ── Module configuration ──
+MODULE_NAME = "kernel"
+
+
+def _fmt_elapsed(t0: float) -> str:
+    """Format elapsed time since t0: <1s -> 'NNNms', >=1s -> 'N.Ns', >=10s -> 'NNs'."""
+    d = time.monotonic() - t0
+    if d < 1:
+        return f"{d * 1000:.0f}ms"
+    if d < 10:
+        return f"{d:.1f}s"
+    return f"{d:.0f}s"
+
+
+# ── Safe stdout/stderr: ignore BrokenPipeError after Launcher closes stdio ──
+
+class _SafeWriter:
+    """Wraps a stream to silently swallow BrokenPipeError on write/flush."""
+    def __init__(self, stream):
+        self._stream = stream
+
+    def write(self, s):
+        try:
+            self._stream.write(s)
+        except (BrokenPipeError, OSError):
+            pass
+
+    def flush(self):
+        try:
+            self._stream.flush()
+        except (BrokenPipeError, OSError):
+            pass
+
+    def __getattr__(self, name):
+        return getattr(self._stream, name)
+
+sys.stdout = _SafeWriter(sys.stdout)
+sys.stderr = _SafeWriter(sys.stderr)
+
+
+# ── Timestamped print + log file writer ──
+# Independent implementation per module (no shared code dependency)
+
+_builtin_print = builtins.print
+_start_ts = time.monotonic()
+_last_ts = time.monotonic()
+_ANSI_RE = re.compile(r"\033\[[0-9;]*m")
+_log_lock = threading.Lock()
+_log_latest_path = None
+_log_daily_path = None
+_log_daily_date = ""
+_log_dir = None
+_crash_log_path = None
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+def _resolve_daily_log_path():
+    """Resolve daily log path based on current date."""
+    global _log_daily_path, _log_daily_date
+    if not _log_dir:
+        return
+    today = datetime.now().strftime("%Y-%m-%d")
+    if today == _log_daily_date and _log_daily_path:
+        return
+    month_dir = os.path.join(_log_dir, today[:7])
+    os.makedirs(month_dir, exist_ok=True)
+    _log_daily_path = os.path.join(month_dir, f"{today}.log")
+    _log_daily_date = today
+
+def _write_log(plain_line: str):
+    """Write a plain-text line to both latest.log and daily log."""
+    with _log_lock:
+        if _log_latest_path:
+            try:
+                with open(_log_latest_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+        _resolve_daily_log_path()
+        if _log_daily_path:
+            try:
+                with open(_log_daily_path, "a", encoding="utf-8") as f:
+                    f.write(plain_line)
+            except Exception:
+                pass
+
+def _write_crash(exc_type, exc_value, exc_tb, thread_name=None, severity="critical", handled=False):
+    """Write crash record to crashes.jsonl + daily crash archive."""
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "module": MODULE_NAME,
+        "thread": thread_name or threading.current_thread().name,
+        "exception_type": exc_type.__name__ if exc_type else "Unknown",
+        "exception_message": str(exc_value),
+        "traceback": "".join(traceback.format_exception(exc_type, exc_value, exc_tb)),
+        "severity": severity,
+        "handled": handled,
+        "process_id": os.getpid(),
+        "platform": sys.platform,
+        "runtime_version": f"Python {sys.version.split()[0]}",
+    }
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            record["context"] = {
+                "function": last.name,
+                "file": os.path.basename(last.filename),
+                "line": last.lineno,
+            }
+
+    line = json.dumps(record, ensure_ascii=False) + "\n"
+
+    # 1. Write to crashes.jsonl (current run)
+    if _crash_log_path:
+        try:
+            with open(_crash_log_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+    # 2. Write to daily crash archive
+    if _log_dir:
+        try:
+            today = datetime.now().strftime("%Y-%m-%d")
+            archive_dir = os.path.join(_log_dir, "crashes", today[:7])
+            os.makedirs(archive_dir, exist_ok=True)
+            archive_path = os.path.join(archive_dir, f"{today}.jsonl")
+            with open(archive_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception:
+            pass
+
+def _print_crash_summary(exc_type, exc_tb, thread_name=None):
+    """Print crash summary to console (red highlight)."""
+    RED = "\033[91m"
+    RESET = "\033[0m"
+
+    if exc_tb:
+        tb_entries = traceback.extract_tb(exc_tb)
+        if tb_entries:
+            last = tb_entries[-1]
+            location = f"{os.path.basename(last.filename)}:{last.lineno}"
+        else:
+            location = "unknown"
+    else:
+        location = "unknown"
+
+    prefix = f"[{MODULE_NAME}]"
+    if thread_name:
+        _builtin_print(f"{prefix} {RED}线程 {thread_name} 崩溃: "
+                       f"{exc_type.__name__} in {location}{RESET}")
+    else:
+        _builtin_print(f"{prefix} {RED}崩溃: {exc_type.__name__} in {location}{RESET}")
+    if _crash_log_path:
+        _builtin_print(f"{prefix} 崩溃日志: {_crash_log_path}")
+
+def _setup_exception_hooks():
+    """Set up global exception hooks (sys.excepthook + threading.excepthook)."""
+    _orig_excepthook = sys.excepthook
+
+    def _excepthook(exc_type, exc_value, exc_tb):
+        _write_crash(exc_type, exc_value, exc_tb, severity="critical", handled=False)
+        _print_crash_summary(exc_type, exc_tb)
+        _orig_excepthook(exc_type, exc_value, exc_tb)
+
+    sys.excepthook = _excepthook
+
+    if hasattr(threading, "excepthook"):
+        def _thread_excepthook(args):
+            _write_crash(args.exc_type, args.exc_value, args.exc_traceback,
+                        thread_name=args.thread.name if args.thread else "unknown",
+                        severity="error", handled=False)
+            _print_crash_summary(args.exc_type, args.exc_traceback,
+                                thread_name=args.thread.name if args.thread else None)
+
+        threading.excepthook = _thread_excepthook
+
+def _tprint(*args, **kwargs):
+    """Timestamped print that adds [timestamp] HH:MM:SS.mmm +delta prefix."""
+    global _last_ts
+    now = time.monotonic()
+    elapsed = now - _start_ts
+    delta = now - _last_ts
+    _last_ts = now
+
+    # Format elapsed (cumulative time since start)
+    if elapsed < 1:
+        elapsed_str = f"{elapsed * 1000:.0f}ms"
+    elif elapsed < 100:
+        elapsed_str = f"{elapsed:.1f}s"
+    else:
+        elapsed_str = f"{elapsed:.0f}s"
+
+    # Format delta (time since last print)
+    if delta < 0.001:
+        delta_str = ""
+    elif delta < 1:
+        delta_str = f"+{delta * 1000:.0f}ms"
+    elif delta < 100:
+        delta_str = f"+{delta:.1f}s"
+    else:
+        delta_str = f"+{delta:.0f}s"
+
+    # Current time
+    ts = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+
+    # Print to console (original behavior)
+    _builtin_print(*args, **kwargs)
+
+    # Write to log files with timestamp prefix
+    if _log_latest_path or _log_daily_path:
+        sep = kwargs.get("sep", " ")
+        end = kwargs.get("end", "\n")
+        text = sep.join(str(a) for a in args)
+        prefix = f"[{elapsed_str:>6}] {ts} {delta_str:>8} "
+        _write_log(prefix + _strip_ansi(text) + end)
+
+builtins.print = _tprint
+
+# Ensure project root is on sys.path (set by main.py or cli.js)
+_this_dir = os.path.dirname(os.path.abspath(__file__))
+_project_root = os.environ.get("KITE_PROJECT") or os.path.dirname(os.path.dirname(_this_dir))
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from kernel.server import KernelServer
+
+
+def _read_module_md() -> dict:
+    """Read preferred_port and advertise_ip from own module.md."""
+    md_path = os.path.join(_this_dir, "module.md")
+    result = {"preferred_port": 0, "advertise_ip": "127.0.0.1"}
+    try:
+        with open(md_path, "r", encoding="utf-8") as f:
+            text = f.read()
+        m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
+        if m:
+            try:
+                import yaml
+                fm = yaml.safe_load(m.group(1)) or {}
+            except ImportError:
+                fm = {}
+            result["preferred_port"] = int(fm.get("preferred_port", 0))
+            result["advertise_ip"] = fm.get("advertise_ip", "127.0.0.1")
+    except Exception:
+        pass
+    return result
+
+
+def _bind_port(preferred: int, host: str) -> int:
+    """Try preferred port first, fall back to OS-assigned."""
+    if preferred:
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind((host, preferred))
+                return preferred
+        except OSError:
+            pass
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind((host, 0))
+        return s.getsockname()[1]
+
+
+def _read_stdin_kite_messages(expected: set[str], timeout: float = 10) -> dict[str, dict]:
+    """Read multiple structured kite messages from stdin until all expected types received.
+
+    Args:
+        expected: set of kite message types to wait for (e.g. {"tokens"})
+        timeout: total timeout in seconds
+
+    Returns:
+        dict mapping kite type -> message dict. Missing types are absent from the result.
+    """
+    collected: dict[str, dict] = {}
+    remaining = set(expected)
+
+    def _read():
+        while remaining:
+            try:
+                line = sys.stdin.readline().strip()
+                if not line:
+                    return  # stdin closed
+                msg = json.loads(line)
+                if isinstance(msg, dict) and "kite" in msg:
+                    kite_type = msg["kite"]
+                    collected[kite_type] = msg
+                    remaining.discard(kite_type)
+            except Exception:
+                return  # parse error or stdin closed
+
+    t = threading.Thread(target=_read, daemon=True)
+    t.start()
+    t.join(timeout=timeout)
+    return collected
+
+
+def main():
+    # Initialize log file paths
+    global _log_dir, _log_latest_path, _crash_log_path
+    module_data = os.environ.get("KITE_MODULE_DATA")
+    if module_data:
+        _log_dir = os.path.join(module_data, "log")
+        os.makedirs(_log_dir, exist_ok=True)
+        suffix = os.environ.get("KITE_INSTANCE_SUFFIX", "")
+
+        # latest.log — truncate on each startup
+        _log_latest_path = os.path.join(_log_dir, f"latest{suffix}.log")
+        try:
+            with open(_log_latest_path, "w", encoding="utf-8") as f:
+                pass  # truncate
+        except Exception:
+            _log_latest_path = None
+
+        # crashes.jsonl — truncate on each startup
+        _crash_log_path = os.path.join(_log_dir, f"crashes{suffix}.jsonl")
+        try:
+            with open(_crash_log_path, "w", encoding="utf-8") as f:
+                pass  # truncate
+        except Exception:
+            _crash_log_path = None
+
+        _resolve_daily_log_path()
+
+    # Setup exception hooks
+    _setup_exception_hooks()
+
+    _t0 = time.monotonic()
+
+    # Kite environment
+    is_debug = os.environ.get("KITE_DEBUG") == "1"
+
+    if is_debug:
+        print("[kernel] 调试模式已启用 (KITE_DEBUG=1)，接受所有令牌")
+
+    # Step 1: Read config from own module.md
+    md_config = _read_module_md()
+    advertise_ip = md_config["advertise_ip"]
+    preferred_port = md_config["preferred_port"]
+
+    # Step 2: Generate launcher token
+    import secrets
+    launcher_token = secrets.token_urlsafe(32)
+
+    # Step 3: Create KernelServer with launcher_token
+    server = KernelServer(launcher_token=launcher_token, advertise_ip=advertise_ip)
+
+    # Step 4: Bind port
+    bind_host = advertise_ip
+    port = _bind_port(preferred_port, bind_host)
+    server.port = port
+
+    # Step 5: Output port + launcher_token via stdout
+    print(json.dumps({"kite": "port", "port": port, "token": launcher_token}), flush=True)
+
+    print(f"[kernel] 启动中 {bind_host}:{port} ({_fmt_elapsed(_t0)})")
+
+    # Step 6: Self-register in registry (direct memory write)
+    server.self_register()
+
+    # Step 7: Start uvicorn (module.ready will be published when Launcher subscribes)
+    try:
+        config = uvicorn.Config(server.app, host=bind_host, port=port, log_level="warning")
+        uvi_server = uvicorn.Server(config)
+        server._uvicorn_server = uvi_server
+
+        # module.ready is now published when Launcher subscribes (not at startup)
+        # This ensures Launcher is connected and subscribed before receiving the event
+
+        uvi_server.run()
+    except Exception as e:
+        _write_crash(type(e), e, e.__traceback__, severity="critical", handled=True)
+        _print_crash_summary(type(e), e.__traceback__)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/{core/event_hub/hub.py → kernel/event_hub.py}

@@ -1,7 +1,13 @@
 """
-Core Event Hub logic.
-WebSocket handling, event processing (validate → dedup → route → ack),
-connection and subscription management.
+Kernel Event Hub logic.
+Connection management, subscription matching, event routing (validate -> dedup -> route),
+per-subscriber delivery queues.
+
+Adapted from core/event_hub/hub.py for the merged Kernel module:
+  - Removed handle_message, _handle_event, _send_ack, _send_error (moved to RpcRouter)
+  - Added publish_event() for RPC-driven event publishing
+  - Added publish_internal() for Kernel-originated events (e.g. module.offline)
+  - Event delivery uses JSON-RPC 2.0 Notification format
 """
 
 import asyncio
@@ -14,7 +20,7 @@ try:
 except ImportError:
     orjson = None
 
-from fastapi import WebSocket, WebSocketDisconnect
+from starlette.websockets import WebSocket
 
 from .dedup import EventDedup
 from .router import match_parts
@@ -31,6 +37,7 @@ def _loads(raw: str):
         return orjson.loads(raw)
     return json.loads(raw)
 
+
 QUEUE_MAXSIZE = 10000
 
 
@@ -75,14 +82,14 @@ class EventHub:
         self._senders[module_id] = asyncio.create_task(
             self._sender_loop(module_id, ws, q)
         )
-        print(f"[event_hub] {module_id} connected")
+        print(f"[kernel] {module_id} connected")
 
     async def _close_old(self, module_id: str, ws: WebSocket):
         try:
             await ws.close(code=4000, reason="replaced by new connection")
         except Exception:
             pass
-        print(f"[event_hub] Closed old connection for {module_id}")
+        print(f"[kernel] Closed old connection for {module_id}")
 
     def remove_connection(self, module_id: str):
         """Clean up on disconnect."""
@@ -93,7 +100,7 @@ class EventHub:
         task = self._senders.pop(module_id, None)
         if task:
             task.cancel()
-        print(f"[event_hub] {module_id} disconnected")
+        print(f"[kernel] {module_id} disconnected")
 
     # ── Sender loop (per-subscriber) ──
 
@@ -109,7 +116,7 @@ class EventHub:
                         await ws.send_text(raw)
                     self._cnt_routed += 1
                 except Exception:
-                    print(f"[event_hub] Send failed to {mid}, closing sender")
+                    print(f"[kernel] Send failed to {mid}, closing sender")
                     break
         except asyncio.CancelledError:
             pass
@@ -122,77 +129,72 @@ class EventHub:
         self.subscriptions[module_id].update(
             (p, tuple(p.split("."))) for p in events
         )
-        print(f"[event_hub] {module_id} subscribed: {events}")
+        print(f"[kernel] {module_id} subscribed: {events}")
 
     def handle_unsubscribe(self, module_id: str, events: list[str]):
         subs = self.subscriptions.get(module_id)
         if subs:
             to_remove = {item for item in subs if item[0] in events}
             subs.difference_update(to_remove)
-        print(f"[event_hub] {module_id} unsubscribed: {events}")
-
-    # ── Main message handler ──
-
-    async def handle_message(self, module_id: str, ws: WebSocket, raw: str):
-        try:
-            msg = _loads(raw)
-        except Exception:
-            await self._send_error(ws, "Invalid JSON")
-            return
-
-        msg_type = msg.get("type", "")
-
-        if msg_type == "subscribe":
-            events = msg.get("events", [])
-            if isinstance(events, list) and events:
-                self.handle_subscribe(module_id, events)
-            return
+        print(f"[kernel] {module_id} unsubscribed: {events}")
 
-        if msg_type == "unsubscribe":
-            events = msg.get("events", [])
-            if isinstance(events, list) and events:
-                self.handle_unsubscribe(module_id, events)
-            return
+    # ── Event publishing (called by RpcRouter) ──
 
-        if msg_type == "event":
-            await self._handle_event(module_id, ws, msg)
-            return
-
-        await self._send_error(ws, f"Unknown message type: {msg_type}")
-
-    # ── Event processing ──
-
-    async def _handle_event(self, module_id: str, ws: WebSocket, msg: dict):
-        """Validate → dedup → auto-fill → route → ack."""
+    def publish_event(self, sender_id: str, event_id: str, event_type: str,
+                      data: dict = None, echo: bool = False) -> dict:
+        """Publish an event from a module. Called by RpcRouter for event.publish RPC.
+        Returns {ok: True} on success, or error dict if validation fails."""
         self._cnt_received += 1
 
-        event_id = msg.get("event_id")
-        event_type = msg.get("event")
         if not event_id or not event_type:
             self._cnt_errors += 1
-            await self._send_error(ws, "Missing required field: event_id or event")
-            return
+            return {"ok": False, "error": "Missing required field: event_id or event"}
 
         if self.dedup.is_duplicate(event_id):
             self._cnt_dedup += 1
-            await self._send_ack(ws, event_id)
-            return
+            return {"ok": True}  # silently accept duplicates
+
+        msg = {
+            "jsonrpc": "2.0",
+            "method": "event",
+            "params": {
+                "event_id": event_id,
+                "event": event_type,
+                "source": sender_id,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "data": data or {},
+            },
+        }
+
+        self._route_event(sender_id, msg, event_type, echo)
+        return {"ok": True}
 
-        if not msg.get("source"):
-            msg["source"] = module_id
-        if not msg.get("timestamp"):
-            msg["timestamp"] = datetime.now(timezone.utc).isoformat()
+    def publish_internal(self, event_type: str, data: dict):
+        """Publish a Kernel-originated event (e.g. module.offline, module.registered).
+        No dedup check — internal events are unique by nature."""
+        import uuid
+        event_id = str(uuid.uuid4())
+        self._cnt_received += 1
+
+        msg = {
+            "jsonrpc": "2.0",
+            "method": "event",
+            "params": {
+                "event_id": event_id,
+                "event": event_type,
+                "source": "kernel",
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "data": data,
+            },
+        }
 
-        await self._route_event(module_id, msg)
-        await self._send_ack(ws, event_id)
+        self._route_event("kernel", msg, event_type, echo=False)
 
     # ── Routing ──
 
-    async def _route_event(self, sender_id: str, msg: dict):
+    def _route_event(self, sender_id: str, msg: dict, event_type: str, echo: bool):
         """Enqueue event to all matching subscribers' delivery queues."""
-        event_type = msg["event"]
         e_parts = tuple(event_type.split("."))
-        echo = msg.get("echo", False)
         raw = None  # lazy serialization
 
         for mid, patterns in self.subscriptions.items():
@@ -207,26 +209,12 @@ class EventHub:
                         try:
                             queue.put_nowait(raw)
                         except asyncio.QueueFull:
-                            await queue.put(raw)
+                            # Best-effort: drop if queue is full and we can't await
+                            # (we're not in an async context in _route_event)
+                            pass
                         self._cnt_queued += 1
                     break
 
-    # ── Helpers ──
-
-    @staticmethod
-    async def _send_ack(ws: WebSocket, event_id: str):
-        try:
-            await ws.send_text('{"type":"ack","event_id":"' + event_id + '"}')
-        except Exception:
-            pass
-
-    @staticmethod
-    async def _send_error(ws: WebSocket, message: str):
-        try:
-            await ws.send_json({"type": "error", "message": message})
-        except Exception:
-            pass
-
     # ── Stats ──
 
     def _counters_dict(self) -> dict:

package/kernel/module.md

@@ -0,0 +1,33 @@
+---
+name: kernel
+display_name: Kernel
+type: infrastructure
+state: enabled
+runtime: python
+entry: entry.py
+---
+
+# Kernel
+
+Unified infrastructure module that merges Registry (service discovery) and Event Hub (event routing) into a single process with a WebSocket JSON-RPC 2.0 interface.
+
+## Responsibilities
+
+- **Service Registry**: Module registration, heartbeat TTL, glob lookup, dot-path queries
+- **Event Routing**: NATS-style wildcard subscriptions, per-subscriber queues, 1h dedup
+- **RPC Routing**: JSON-RPC 2.0 dispatch for builtin methods + cross-module forwarding
+- **Token Verification**: In-memory token→module_id resolution (no cross-process HTTP)
+
+## Protocol
+
+Single WebSocket endpoint: `ws://127.0.0.1:{port}/ws?token={TOKEN}&id={MODULE_ID}`
+
+Three frame types on the wire:
+- **RPC Request**: `{jsonrpc:"2.0", id, method, params}` — client→Kernel or Kernel→client (forwarded)
+- **RPC Response**: `{jsonrpc:"2.0", id, result}` or `{jsonrpc:"2.0", id, error}` — response to request
+- **Event Notification**: `{jsonrpc:"2.0", method:"event", params:{event_id, event, source, timestamp, data}}` — no id, no response
+
+## HTTP Endpoints (debug only)
+
+- `GET /health` — combined registry + event hub health
+- `GET /stats` — connections, subscriptions, counters