@agentunion/kite 1.2.0 → 1.3.0

package/{core/registry → kernel}/entry.py

@@ -1,7 +1,14 @@
 """
-Registry entry point.
-Reads token from stdin boot_info, starts HTTP server on dynamic port,
-outputs port via stdout structured message, waits for Event Hub to trigger module.ready.
+Kernel entry point.
+Reads boot_info token + tokens dict from stdin, starts unified WebSocket server,
+outputs port via stdout, self-registers, publishes module.ready.
+
+Stdin protocol:
+  Line 1: {"token": "launcher_kite_token"}                     (boot_info)
+  Line 2: {"kite": "tokens", "tokens": {"mod1":"tok1", ...}}   (token mapping)
+
+Stdout protocol:
+  {"kite": "port", "port": N}                                  (Kernel port)
 """
 
 import builtins
@@ -19,11 +26,11 @@ import uvicorn
 
 
 # ── Module configuration ──
-MODULE_NAME = "registry"
+MODULE_NAME = "kernel"
 
 
 def _fmt_elapsed(t0: float) -> str:
-    """Format elapsed time since t0: <1s → 'NNNms', >=1s → 'N.Ns', >=10s → 'NNs'."""
+    """Format elapsed time since t0: <1s -> 'NNNms', >=1s -> 'N.Ns', >=10s -> 'NNs'."""
     d = time.monotonic() - t0
     if d < 1:
         return f"{d * 1000:.0f}ms"
@@ -59,7 +66,6 @@ sys.stderr = _SafeWriter(sys.stderr)
 
 
 # ── Timestamped print + log file writer ──
-# Implements format: [timestamp] HH:MM:SS.mmm +delta [module_name] message
 # Independent implementation per module (no shared code dependency)
 
 _builtin_print = builtins.print
@@ -247,8 +253,7 @@ _project_root = os.environ.get("KITE_PROJECT") or os.path.dirname(os.path.dirnam
 if _project_root not in sys.path:
     sys.path.insert(0, _project_root)
 
-from core.registry.store import RegistryStore
-from core.registry.server import RegistryServer
+from kernel.server import KernelServer
 
 
 def _read_module_md() -> dict:
@@ -258,7 +263,6 @@ def _read_module_md() -> dict:
     try:
         with open(md_path, "r", encoding="utf-8") as f:
             text = f.read()
-        import re
         m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
         if m:
             try:
@@ -287,6 +291,39 @@ def _bind_port(preferred: int, host: str) -> int:
         return s.getsockname()[1]
 
 
+def _read_stdin_kite_messages(expected: set[str], timeout: float = 10) -> dict[str, dict]:
+    """Read multiple structured kite messages from stdin until all expected types received.
+
+    Args:
+        expected: set of kite message types to wait for (e.g. {"tokens"})
+        timeout: total timeout in seconds
+
+    Returns:
+        dict mapping kite type -> message dict. Missing types are absent from the result.
+    """
+    collected: dict[str, dict] = {}
+    remaining = set(expected)
+
+    def _read():
+        while remaining:
+            try:
+                line = sys.stdin.readline().strip()
+                if not line:
+                    return  # stdin closed
+                msg = json.loads(line)
+                if isinstance(msg, dict) and "kite" in msg:
+                    kite_type = msg["kite"]
+                    collected[kite_type] = msg
+                    remaining.discard(kite_type)
+            except Exception:
+                return  # parse error or stdin closed
+
+    t = threading.Thread(target=_read, daemon=True)
+    t.start()
+    t.join(timeout=timeout)
+    return collected
+
+
 def main():
     # Initialize log file paths
     global _log_dir, _log_latest_path, _crash_log_path
@@ -320,54 +357,45 @@ def main():
     _t0 = time.monotonic()
 
     # Kite environment
-    kite_instance = os.environ.get("KITE_INSTANCE", "")
     is_debug = os.environ.get("KITE_DEBUG") == "1"
 
-    # Step 1: Read token from stdin boot_info (only token)
-    launcher_token = ""
-    try:
-        line = sys.stdin.readline().strip()
-        if line:
-            boot_info = json.loads(line)
-            launcher_token = boot_info.get("token", "")
-    except Exception:
-        pass
-
-    if not launcher_token:
-        print("[registry] 错误: 未通过 stdin boot_info 提供启动器令牌")
-        sys.exit(1)
-
-    print(f"[registry] 已收到启动器令牌 ({len(launcher_token)} 字符) ({_fmt_elapsed(_t0)})")
-
     if is_debug:
-        print("[registry] 调试模式已启用 (KITE_DEBUG=1)，接受所有令牌")
+        print("[kernel] 调试模式已启用 (KITE_DEBUG=1)，接受所有令牌")
 
-    # Step 2: Read config from own module.md
+    # Step 1: Read config from own module.md
     md_config = _read_module_md()
     advertise_ip = md_config["advertise_ip"]
     preferred_port = md_config["preferred_port"]
 
-    # Step 3: Create store and server
-    store = RegistryStore(launcher_token)
-    server = RegistryServer(store, launcher_token=launcher_token, advertise_ip=advertise_ip)
+    # Step 2: Generate launcher token
+    import secrets
+    launcher_token = secrets.token_urlsafe(32)
+
+    # Step 3: Create KernelServer with launcher_token
+    server = KernelServer(launcher_token=launcher_token, advertise_ip=advertise_ip)
 
     # Step 4: Bind port
     bind_host = advertise_ip
     port = _bind_port(preferred_port, bind_host)
+    server.port = port
 
-    # Step 5: Output port via stdout structured message (Launcher reads this)
-    # This message proves HTTP is about to start — Launcher uses it as readiness signal
-    print(json.dumps({"kite": "port", "port": port}), flush=True)
+    # Step 5: Output port + launcher_token via stdout
+    print(json.dumps({"kite": "port", "port": port, "token": launcher_token}), flush=True)
 
-    print(f"[registry] 启动中 {bind_host}:{port} ({_fmt_elapsed(_t0)})")
+    print(f"[kernel] 启动中 {bind_host}:{port} ({_fmt_elapsed(_t0)})")
 
-    # Store port and advertise_ip for module.ready event later
-    server.port = port
+    # Step 6: Self-register in registry (direct memory write)
+    server.self_register()
 
+    # Step 7: Start uvicorn (module.ready will be published when Launcher subscribes)
     try:
         config = uvicorn.Config(server.app, host=bind_host, port=port, log_level="warning")
         uvi_server = uvicorn.Server(config)
         server._uvicorn_server = uvi_server
+
+        # module.ready is now published when Launcher subscribes (not at startup)
+        # This ensures Launcher is connected and subscribed before receiving the event
+
         uvi_server.run()
     except Exception as e:
         _write_crash(type(e), e, e.__traceback__, severity="critical", handled=True)

package/{core/event_hub/hub.py → kernel/event_hub.py}

@@ -1,7 +1,13 @@
 """
-Core Event Hub logic.
-WebSocket handling, event processing (validate → dedup → route → ack),
-connection and subscription management.
+Kernel Event Hub logic.
+Connection management, subscription matching, event routing (validate -> dedup -> route),
+per-subscriber delivery queues.
+
+Adapted from core/event_hub/hub.py for the merged Kernel module:
+  - Removed handle_message, _handle_event, _send_ack, _send_error (moved to RpcRouter)
+  - Added publish_event() for RPC-driven event publishing
+  - Added publish_internal() for Kernel-originated events (e.g. module.offline)
+  - Event delivery uses JSON-RPC 2.0 Notification format
 """
 
 import asyncio
@@ -14,7 +20,7 @@ try:
 except ImportError:
     orjson = None
 
-from fastapi import WebSocket, WebSocketDisconnect
+from starlette.websockets import WebSocket
 
 from .dedup import EventDedup
 from .router import match_parts
@@ -31,6 +37,7 @@ def _loads(raw: str):
         return orjson.loads(raw)
     return json.loads(raw)
 
+
 QUEUE_MAXSIZE = 10000
 
 
@@ -55,8 +62,6 @@ class EventHub:
         self._cnt_dedup = 0
         self._cnt_errors = 0
         self._start_time = time.time()
-        # Shutdown callback: set by EventHubServer to handle own shutdown
-        self._shutdown_cb = None
 
     # ── Connection lifecycle ──
 
@@ -77,14 +82,14 @@ class EventHub:
         self._senders[module_id] = asyncio.create_task(
             self._sender_loop(module_id, ws, q)
         )
-        print(f"[event_hub] {module_id} connected")
+        print(f"[kernel] {module_id} connected")
 
     async def _close_old(self, module_id: str, ws: WebSocket):
         try:
             await ws.close(code=4000, reason="replaced by new connection")
         except Exception:
             pass
-        print(f"[event_hub] Closed old connection for {module_id}")
+        print(f"[kernel] Closed old connection for {module_id}")
 
     def remove_connection(self, module_id: str):
         """Clean up on disconnect."""
@@ -95,7 +100,7 @@ class EventHub:
         task = self._senders.pop(module_id, None)
         if task:
             task.cancel()
-        print(f"[event_hub] {module_id} disconnected")
+        print(f"[kernel] {module_id} disconnected")
 
     # ── Sender loop (per-subscriber) ──
 
@@ -111,7 +116,7 @@ class EventHub:
                         await ws.send_text(raw)
                     self._cnt_routed += 1
                 except Exception:
-                    print(f"[event_hub] Send failed to {mid}, closing sender")
+                    print(f"[kernel] Send failed to {mid}, closing sender")
                     break
         except asyncio.CancelledError:
             pass
@@ -124,83 +129,72 @@ class EventHub:
         self.subscriptions[module_id].update(
             (p, tuple(p.split("."))) for p in events
         )
-        print(f"[event_hub] {module_id} subscribed: {events}")
+        print(f"[kernel] {module_id} subscribed: {events}")
 
     def handle_unsubscribe(self, module_id: str, events: list[str]):
         subs = self.subscriptions.get(module_id)
         if subs:
             to_remove = {item for item in subs if item[0] in events}
             subs.difference_update(to_remove)
-        print(f"[event_hub] {module_id} unsubscribed: {events}")
-
-    # ── Main message handler ──
-
-    async def handle_message(self, module_id: str, ws: WebSocket, raw: str):
-        try:
-            msg = _loads(raw)
-        except Exception:
-            await self._send_error(ws, "Invalid JSON")
-            return
-
-        msg_type = msg.get("type", "")
-
-        if msg_type == "subscribe":
-            events = msg.get("events", [])
-            if isinstance(events, list) and events:
-                self.handle_subscribe(module_id, events)
-            return
+        print(f"[kernel] {module_id} unsubscribed: {events}")
 
-        if msg_type == "unsubscribe":
-            events = msg.get("events", [])
-            if isinstance(events, list) and events:
-                self.handle_unsubscribe(module_id, events)
-            return
+    # ── Event publishing (called by RpcRouter) ──
 
-        if msg_type == "event":
-            await self._handle_event(module_id, ws, msg)
-            return
-
-        await self._send_error(ws, f"Unknown message type: {msg_type}")
-
-    # ── Event processing ──
-
-    async def _handle_event(self, module_id: str, ws: WebSocket, msg: dict):
-        """Validate → dedup → auto-fill → route → ack."""
+    def publish_event(self, sender_id: str, event_id: str, event_type: str,
+                      data: dict = None, echo: bool = False) -> dict:
+        """Publish an event from a module. Called by RpcRouter for event.publish RPC.
+        Returns {ok: True} on success, or error dict if validation fails."""
         self._cnt_received += 1
 
-        event_id = msg.get("event_id")
-        event_type = msg.get("event")
         if not event_id or not event_type:
             self._cnt_errors += 1
-            await self._send_error(ws, "Missing required field: event_id or event")
-            return
+            return {"ok": False, "error": "Missing required field: event_id or event"}
 
         if self.dedup.is_duplicate(event_id):
             self._cnt_dedup += 1
-            await self._send_ack(ws, event_id)
-            return
+            return {"ok": True}  # silently accept duplicates
+
+        msg = {
+            "jsonrpc": "2.0",
+            "method": "event",
+            "params": {
+                "event_id": event_id,
+                "event": event_type,
+                "source": sender_id,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "data": data or {},
+            },
+        }
 
-        if not msg.get("source"):
-            msg["source"] = module_id
-        if not msg.get("timestamp"):
-            msg["timestamp"] = datetime.now(timezone.utc).isoformat()
+        self._route_event(sender_id, msg, event_type, echo)
+        return {"ok": True}
 
-        await self._route_event(module_id, msg)
-        await self._send_ack(ws, event_id)
+    def publish_internal(self, event_type: str, data: dict):
+        """Publish a Kernel-originated event (e.g. module.offline, module.registered).
+        No dedup check — internal events are unique by nature."""
+        import uuid
+        event_id = str(uuid.uuid4())
+        self._cnt_received += 1
 
-        # Check if this is a shutdown event targeting event_hub
-        if event_type == "module.shutdown" and self._shutdown_cb:
-            data = msg.get("data", {})
-            if data.get("module_id") == "event_hub":
-                asyncio.create_task(self._shutdown_cb(data))
+        msg = {
+            "jsonrpc": "2.0",
+            "method": "event",
+            "params": {
+                "event_id": event_id,
+                "event": event_type,
+                "source": "kernel",
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "data": data,
+            },
+        }
+
+        self._route_event("kernel", msg, event_type, echo=False)
 
     # ── Routing ──
 
-    async def _route_event(self, sender_id: str, msg: dict):
+    def _route_event(self, sender_id: str, msg: dict, event_type: str, echo: bool):
         """Enqueue event to all matching subscribers' delivery queues."""
-        event_type = msg["event"]
         e_parts = tuple(event_type.split("."))
-        echo = msg.get("echo", False)
         raw = None  # lazy serialization
 
         for mid, patterns in self.subscriptions.items():
@@ -215,26 +209,12 @@ class EventHub:
                         try:
                             queue.put_nowait(raw)
                         except asyncio.QueueFull:
-                            await queue.put(raw)
+                            # Best-effort: drop if queue is full and we can't await
+                            # (we're not in an async context in _route_event)
+                            pass
                         self._cnt_queued += 1
                     break
 
-    # ── Helpers ──
-
-    @staticmethod
-    async def _send_ack(ws: WebSocket, event_id: str):
-        try:
-            await ws.send_text('{"type":"ack","event_id":"' + event_id + '"}')
-        except Exception:
-            pass
-
-    @staticmethod
-    async def _send_error(ws: WebSocket, message: str):
-        try:
-            await ws.send_json({"type": "error", "message": message})
-        except Exception:
-            pass
-
     # ── Stats ──
 
     def _counters_dict(self) -> dict:

package/kernel/module.md

@@ -0,0 +1,33 @@
+---
+name: kernel
+display_name: Kernel
+type: infrastructure
+state: enabled
+runtime: python
+entry: entry.py
+---
+
+# Kernel
+
+Unified infrastructure module that merges Registry (service discovery) and Event Hub (event routing) into a single process with a WebSocket JSON-RPC 2.0 interface.
+
+## Responsibilities
+
+- **Service Registry**: Module registration, heartbeat TTL, glob lookup, dot-path queries
+- **Event Routing**: NATS-style wildcard subscriptions, per-subscriber queues, 1h dedup
+- **RPC Routing**: JSON-RPC 2.0 dispatch for builtin methods + cross-module forwarding
+- **Token Verification**: In-memory token→module_id resolution (no cross-process HTTP)
+
+## Protocol
+
+Single WebSocket endpoint: `ws://127.0.0.1:{port}/ws?token={TOKEN}&id={MODULE_ID}`
+
+Three frame types on the wire:
+- **RPC Request**: `{jsonrpc:"2.0", id, method, params}` — client→Kernel or Kernel→client (forwarded)
+- **RPC Response**: `{jsonrpc:"2.0", id, result}` or `{jsonrpc:"2.0", id, error}` — response to request
+- **Event Notification**: `{jsonrpc:"2.0", method:"event", params:{event_id, event, source, timestamp, data}}` — no id, no response
+
+## HTTP Endpoints (debug only)
+
+- `GET /health` — combined registry + event hub health
+- `GET /stats` — connections, subscriptions, counters

package/{core/registry/store.py → kernel/registry_store.py}

@@ -1,7 +1,11 @@
 """
-Registry in-memory store.
+Kernel registry in-memory store.
 Manages module records, token verification, heartbeat TTL, lookup and get-by-path.
-No persistence — Registry crash triggers Kite full restart, all modules re-register.
+No persistence — Kernel crash triggers Kite full restart, all modules re-register.
+
+Adapted from core/registry/store.py for the merged Kernel module:
+  - api_endpoint is now optional (modules no longer need HTTP)
+  - Added set_offline() for WebSocket disconnect handling
 """
 
 import fnmatch
@@ -50,7 +54,7 @@ class RegistryStore:
 
     # ── Module lifecycle ──
 
-    _REQUIRED_FIELDS = ("module_id", "module_type", "api_endpoint")
+    _REQUIRED_FIELDS = ("module_id", "module_type")  # api_endpoint now optional
 
     def register_module(self, data: dict) -> dict:
         """Register or update a module. Idempotent — same module_id overwrites."""
@@ -69,7 +73,7 @@ class RegistryStore:
                     if other_mid == mid:
                         continue
                     if tool_name in other_data.get("tools", {}):
-                        print(f"[registry] WARNING: tool '{tool_name}' registered by both '{other_mid}' and '{mid}'")
+                        print(f"[kernel] WARNING: tool '{tool_name}' registered by both '{other_mid}' and '{mid}'")
 
         # Strip action field — it's a request verb, not part of the registration payload
         record = {k: v for k, v in data.items() if k != "action"}
@@ -93,6 +97,11 @@ class RegistryStore:
         self.modules[module_id]["status"] = "online"
         return {"ok": True}
 
+    def set_offline(self, module_id: str):
+        """Mark a module as offline (called on WebSocket disconnect)."""
+        if module_id in self.modules:
+            self.modules[module_id]["status"] = "offline"
+
     def check_ttl(self) -> list[str]:
         """Mark modules as offline if heartbeat expired. Returns list of newly-offline module_ids."""
         now = time.time()