@agentunion/kite 1.5.0 → 1.6.0

package/kernel/rpc_router.py

@@ -1,491 +1,1400 @@
-"""
-JSON-RPC 2.0 message dispatcher for Kernel.
-Routes builtin methods (registry.*, event.*, kernel.*) and forwards
-cross-module RPC requests with timeout tracking.
-"""
-
-import asyncio
-import json
-import time
-import uuid
-from dataclasses import dataclass, field
-from datetime import datetime, timezone
-
-from starlette.websockets import WebSocket
-
-from .registry_store import RegistryStore
-from .event_hub import EventHub
-
-
-# ── JSON-RPC 2.0 error codes ──
-
-PARSE_ERROR = -32700
-INVALID_REQUEST = -32600
-METHOD_NOT_FOUND = -32601
-INVALID_PARAMS = -32602
-INTERNAL_ERROR = -32603
-
-# Kite custom error codes
-MODULE_OFFLINE = -32001
-RPC_TIMEOUT = -32002
-AUTH_FAILED = -32003
-PERMISSION_DENIED = -32004
-DUPLICATE_EVENT = -32005
-
-DEFAULT_FORWARD_TIMEOUT = 5.0  # seconds
-
-
-@dataclass
-class PendingForward:
-    """Tracks a forwarded cross-module RPC awaiting response."""
-    caller_ws: WebSocket
-    original_id: str
-    caller_id: str
-    target_id: str
-    created_at: float = field(default_factory=time.time)
-    timeout_handle: asyncio.TimerHandle | None = None
-
-
-def _result_msg(msg_id: str, result: dict) -> str:
-    return json.dumps({"jsonrpc": "2.0", "id": msg_id, "result": result})
-
-
-def _error_msg(msg_id: str, code: int, message: str, data: dict = None) -> str:
-    err = {"code": code, "message": message}
-    if data:
-        err["data"] = data
-    return json.dumps({"jsonrpc": "2.0", "id": msg_id, "error": err})
-
-
-class RpcRouter:
-    """JSON-RPC 2.0 message dispatcher.
-
-    Handles:
-    - 13 builtin methods (registry.*, event.*, kernel.*)
-    - Cross-module RPC forwarding with timeout
-    - RPC response matching for forwarded calls
-    """
-
-    def __init__(self, registry: RegistryStore, event_hub: EventHub,
-                 connections: dict[str, WebSocket], kernel_server):
-        self.registry = registry
-        self.event_hub = event_hub
-        self.connections = connections
-        self.kernel_server = kernel_server  # Direct reference to KernelServer
-
-        # RPC 统计计数器
-        self._rpc_total = 0  # 总 RPC 调用次数
-        self._rpc_builtin = 0  # 内置方法调用次数
-        self._rpc_forwarded = 0  # 转发调用次数
-        self._rpc_errors = 0  # 错误次数
-
-        # Builtin method dispatch table
-        self.methods: dict[str, callable] = {
-            "registry.register":      self._registry_register,
-            "registry.deregister":    self._registry_deregister,
-            "registry.lookup":        self._registry_lookup,
-            "registry.get":           self._registry_get,
-            "registry.verify":        self._registry_verify,
-            "event.publish":          self._event_publish,
-            "event.subscribe":        self._event_subscribe,
-            "event.unsubscribe":      self._event_unsubscribe,
-            "kernel.ping":            self._kernel_ping,
-            "kernel.stats":           self._kernel_stats,
-            "kernel.health":          self._kernel_health,
-            "kernel.latencies":       self._kernel_latencies,
-            "kernel.generate_tokens": self._kernel_generate_tokens,
-            "kernel.register_tokens": self._kernel_register_tokens,
-        }
-
-        # Pending cross-module forwards: internal_id -> PendingForward
-        self._pending: dict[str, PendingForward] = {}
-
-    # ── Main dispatch ──
-
-    async def dispatch(self, caller_id: str, ws: WebSocket, msg: dict):
-        """Route a parsed JSON-RPC message to builtin handler or cross-module forward."""
-        method = msg.get("method", "")
-        msg_id = msg.get("id")
-
-        # 统计：总调用次数
-        self._rpc_total += 1
-
-        # JSON-RPC Notification (no id) — currently not handled from clients
-        if msg_id is None:
-            return
-
-        params = msg.get("params") or {}
-
-        # Builtin method
-        handler = self.methods.get(method)
-        if handler:
-            # 统计：内置方法调用
-            self._rpc_builtin += 1
-            try:
-                result = await handler(caller_id, params)
-            except Exception as e:
-                # 统计：错误次数
-                self._rpc_errors += 1
-                print(f"[kernel] RPC handler error ({method}): {e}")
-                try:
-                    await ws.send_text(_error_msg(msg_id, INTERNAL_ERROR, str(e)))
-                except Exception:
-                    # Can't send error response, connection closed
-                    pass
-                return
-
-            # Send result (may fail if connection closed during shutdown)
-            try:
-                await ws.send_text(_result_msg(msg_id, result))
-            except Exception as e:
-                # Connection closed during shutdown — this is normal, exit silently
-                pass
-            return
-
-        # Cross-module forward: method prefix is target module_id
-        dot_idx = method.find(".")
-        if dot_idx > 0:
-            target = method[:dot_idx]
-            if target in self.connections:
-                # Check if target module is ready (state machine protection)
-                if not self.registry.is_ready(target):
-                    # 统计：错误次数
-                    self._rpc_errors += 1
-                    await ws.send_text(_error_msg(
-                        msg_id, MODULE_OFFLINE,
-                        f"Module not ready: {target} (status: {self.registry.modules.get(target, {}).get('status', 'unknown')})"))
-                    return
-                # 统计：转发调用
-                self._rpc_forwarded += 1
-                await self._forward(caller_id, ws, msg_id, target, method, params)
-                return
-            # Target not connected — check if registered but offline
-            if target in self.registry.modules:
-                # 统计：错误次数
-                self._rpc_errors += 1
-                await ws.send_text(_error_msg(
-                    msg_id, MODULE_OFFLINE, f"Module offline: {target}"))
-                return
-
-        # Method not found
-        # 统计：错误次数
-        self._rpc_errors += 1
-        await ws.send_text(_error_msg(msg_id, METHOD_NOT_FOUND, f"Method not found: {method}"))
-
-    async def handle_response(self, module_id: str, msg: dict):
-        """Handle an RPC response from a module (matches pending forwards)."""
-        msg_id = msg.get("id")
-        if not msg_id:
-            return
-
-        pending = self._pending.pop(msg_id, None)
-        if not pending:
-            return  # orphan response, ignore
-
-        # Cancel timeout
-        if pending.timeout_handle:
-            pending.timeout_handle.cancel()
-
-        # Forward response to original caller with original ID
-        response = {"jsonrpc": "2.0", "id": pending.original_id}
-        if "result" in msg:
-            response["result"] = msg["result"]
-        elif "error" in msg:
-            response["error"] = msg["error"]
-        else:
-            response["result"] = None
-
-        try:
-            await pending.caller_ws.send_text(json.dumps(response))
-        except Exception:
-            pass  # caller disconnected
-
-    # ── Cross-module forwarding ──
-
-    async def _forward(self, caller_id: str, caller_ws: WebSocket,
-                       original_id: str, target: str, method: str, params: dict):
-        """Forward RPC request to target module with timeout tracking."""
-        internal_id = f"fwd-{uuid.uuid4().hex[:12]}"
-
-        # Extract timeout from params (optional _timeout field)
-        timeout = DEFAULT_FORWARD_TIMEOUT
-        if isinstance(params, dict) and "_timeout" in params:
-            try:
-                timeout = float(params.pop("_timeout"))
-            except (ValueError, TypeError):
-                pass
-
-        # Strip target prefix from method for the forwarded request
-        actual_method = method[len(target) + 1:]  # e.g. "watchdog.get_status" -> "get_status"
-
-        # Inject caller_id into params for permission checking
-        if not isinstance(params, dict):
-            params = {}
-        params["_caller_id"] = caller_id
-
-        # Record pending forward
-        loop = asyncio.get_event_loop()
-        pending = PendingForward(
-            caller_ws=caller_ws,
-            original_id=original_id,
-            caller_id=caller_id,
-            target_id=target,
-        )
-        pending.timeout_handle = loop.call_later(
-            timeout, lambda iid=internal_id: asyncio.ensure_future(self._handle_timeout(iid))
-        )
-        self._pending[internal_id] = pending
-
-        # Send to target
-        fwd_msg = json.dumps({
-            "jsonrpc": "2.0",
-            "id": internal_id,
-            "method": actual_method,
-            "params": params or {},
-        })
-        target_ws = self.connections.get(target)
-        if target_ws:
-            try:
-                await target_ws.send_text(fwd_msg)
-            except Exception:
-                # Target send failed — clean up and error to caller
-                self._pending.pop(internal_id, None)
-                if pending.timeout_handle:
-                    pending.timeout_handle.cancel()
-                await caller_ws.send_text(_error_msg(
-                    original_id, MODULE_OFFLINE, f"Failed to reach module: {target}"))
-        else:
-            # Target disconnected between check and send
-            self._pending.pop(internal_id, None)
-            if pending.timeout_handle:
-                pending.timeout_handle.cancel()
-            await caller_ws.send_text(_error_msg(
-                original_id, MODULE_OFFLINE, f"Module offline: {target}"))
-
-    async def _handle_timeout(self, internal_id: str):
-        """Called when a forwarded RPC times out."""
-        pending = self._pending.pop(internal_id, None)
-        if not pending:
-            return
-        try:
-            await pending.caller_ws.send_text(_error_msg(
-                pending.original_id, RPC_TIMEOUT,
-                f"RPC timeout waiting for {pending.target_id}"))
-        except Exception:
-            pass
-
-    # ── Builtin handlers: registry.* ──
-
-    async def _registry_register(self, caller_id: str, params: dict) -> dict:
-        mid = params.get("module_id")
-        if not mid:
-            raise ValueError("module_id required")
-        # Permission: only Launcher or the module itself
-        if caller_id != "launcher" and caller_id != mid:
-            raise PermissionError(f"Module '{caller_id}' cannot register as '{mid}'")
-
-        print(f"[kernel] registry.register called by {caller_id} for module {mid}")
-        print(f"[kernel]   rpc_methods: {params.get('rpc_methods')}")
-
-        result = self.registry.register_module(params)
-        self.event_hub.publish_internal("module.registered", {"module_id": mid}, source=self.kernel_server.module_id)
-
-        # Only publish registry.updated if content actually changed
-        if result.get("changed", True):
-            from datetime import datetime, timezone
-            self.event_hub.publish_internal("registry.updated", {
-                "module_id": mid,
-                "timestamp": datetime.now(timezone.utc).isoformat(),
-                "action": "register",
-            }, source=self.kernel_server.module_id)
-            print(f"[kernel] registry.updated published for {mid}")
-        else:
-            print(f"[kernel] {mid} re-registered with no changes, skipping registry.updated")
-
-        # When Launcher registers, Kernel publishes its own module.ready
-        if mid == "launcher" and self.kernel_server:
-            self.kernel_server.publish_ready()
-            print(f"[kernel] launcher registered → kernel module.ready published")
-
-        return result
-
-    async def _registry_deregister(self, caller_id: str, params: dict) -> dict:
-        mid = params.get("module_id")
-        if not mid:
-            raise ValueError("module_id required")
-        if caller_id != "launcher" and caller_id != mid:
-            raise PermissionError(f"Module '{caller_id}' cannot deregister '{mid}'")
-
-        self.registry.deregister_module(mid)
-        self.event_hub.publish_internal("module.unregistered", {"module_id": mid}, source=self.kernel_server.module_id)
-
-        # Publish registry.updated event for cache invalidation
-        from datetime import datetime, timezone
-        self.event_hub.publish_internal("registry.updated", {
-            "module_id": mid,
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "action": "deregister",
-        }, source=self.kernel_server.module_id)
-
-        return {}
-
-    async def _registry_lookup(self, caller_id: str, params: dict) -> dict:
-        field = params.get("field")
-        module = params.get("module")
-        value = params.get("value")
-
-        print(f"[kernel] registry.lookup called by {caller_id}: field={field}, module={module}, value={value}")
-
-        results = self.registry.lookup(
-            field=field,
-            module=module,
-            value=value,
-        )
-
-        print(f"[kernel] registry.lookup results: {len(results)} matches")
-        for r in results:
-            print(f"[kernel]   - {r['module']}.{r['field']} = {r['value']}")
-
-        return {
-            "results": results,
-            "last_update_time": self.registry.last_update_time
-        }
-
-    async def _registry_get(self, caller_id: str, params: dict) -> dict:
-        path = params.get("path", "")
-        if not path:
-            raise ValueError("path required")
-        val, found = self.registry.get_by_path(path)
-        if not found:
-            raise KeyError(f"Path not found: {path}")
-        return {"value": val}
-
-    async def _registry_verify(self, caller_id: str, params: dict) -> dict:
-        token = params.get("token", "")
-        module_id = self.registry.verify_token(token)
-        if not module_id:
-            raise PermissionError("Invalid token")
-        return {"module_id": module_id}
-
-    # ── Builtin handlers: event.* ──
-
-    async def _event_publish(self, caller_id: str, params: dict) -> dict:
-        event_id = params.get("event_id", "")
-        event_type = params.get("event", "")
-        data = params.get("data")
-        echo = params.get("echo", False)
-
-        # When a module publishes module.ready, update its status in registry
-        if event_type == "module.ready":
-            mid = (data or {}).get("module_id", caller_id)
-            print(f"[kernel] DEBUG: 收到 module.ready 事件，module_id={mid}, caller_id={caller_id}")
-            self.registry.set_ready(mid)
-            print(f"[kernel] DEBUG: 已调用 set_ready({mid})")
-
-        return self.event_hub.publish_event(caller_id, event_id, event_type, data, echo)
-
-    async def _event_subscribe(self, caller_id: str, params: dict) -> dict:
-        events = params.get("events", [])
-        if not isinstance(events, list) or not events:
-            raise ValueError("events must be a non-empty list")
-        self.event_hub.handle_subscribe(caller_id, events)
-        return {}
-
-    async def _event_unsubscribe(self, caller_id: str, params: dict) -> dict:
-        events = params.get("events", [])
-        if not isinstance(events, list) or not events:
-            raise ValueError("events must be a non-empty list")
-        return self.event_hub.handle_unsubscribe(caller_id, events)
-
-    # ── Builtin handlers: kernel.* ──
-
-    async def _kernel_ping(self, caller_id: str, params: dict) -> dict:
-        return {"pong": True, "timestamp": datetime.now(timezone.utc).isoformat()}
-
-    async def _kernel_stats(self, caller_id: str, params: dict) -> dict:
-        event_stats = self.event_hub.get_stats()
-        return {
-            **event_stats,
-            "rpc": {
-                "total": self._rpc_total,
-                "builtin": self._rpc_builtin,
-                "forwarded": self._rpc_forwarded,
-                "errors": self._rpc_errors
-            }
-        }
-
-    async def _kernel_health(self, caller_id: str, params: dict) -> dict:
-        eh_health = self.event_hub.get_health()
-        return {
-            "status": "healthy",
-            "module_count": len(self.registry.modules),
-            "online_count": sum(
-                1 for m in self.registry.modules.values()
-                if m.get("status") in ("registered", "ready")
-            ),
-            "event_stats": eh_health.get("details", {}),
-        }
-
-    async def _kernel_latencies(self, caller_id: str, params: dict) -> dict:
-        """Get ping/pong latencies for all connected modules.
-
-        Returns:
-            {
-                "latencies": {
-                    "module1": {
-                        "outbound": 12.34,  # ms, Kernel → module
-                        "inbound": 23.45,   # ms, module → Kernel
-                        "status": "ok" | "timeout" | "never"
-                    },
-                    ...
-                }
-            }
-        """
-        result = {}
-        for module_id in self.kernel_server.connections.keys():
-            latency_data = self.kernel_server._pong_latencies.get(module_id, {})
-            status = self.kernel_server._pong_status.get(module_id, "never")
-
-            result[module_id] = {
-                "outbound": latency_data.get("outbound"),
-                "inbound": latency_data.get("inbound"),
-                "status": status,
-            }
-
-        return {"latencies": result}
-
-    async def _kernel_generate_tokens(self, caller_id: str, params: dict) -> dict:
-        """Generate tokens for a list of module names.
-
-        Args:
-            params: {"modules": ["mod1", "mod2", ...]}
-
-        Returns:
-            {"tokens": {"mod1": "token1", "mod2": "token2", ...}}
-        """
-        # Only Launcher may request token generation
-        if caller_id != "launcher":
-            raise PermissionError("Only Launcher may generate tokens")
-
-        modules = params.get("modules", [])
-        if not isinstance(modules, list):
-            raise ValueError("modules must be a list")
-
-        import secrets
-        tokens = {}
-        for module_name in modules:
-            tokens[module_name] = secrets.token_hex(32)
-
-        # Register tokens in registry
-        self.registry.register_tokens(tokens)
-
-        return {"tokens": tokens}
-
-    async def _kernel_register_tokens(self, caller_id: str, params: dict) -> dict:
-        # Only Launcher may register tokens
-        if caller_id != "launcher":
-            raise PermissionError("Only Launcher may register tokens")
-        self.registry.register_tokens(params)
-        return {}
-
-
+"""
+JSON-RPC 2.0 message dispatcher for Kernel.
+Routes builtin methods (registry.*, event.*, kernel.*) and forwards
+cross-module RPC requests with timeout tracking.
+"""
+
+import asyncio
+import json
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from starlette.websockets import WebSocket
+
+from .registry_store import RegistryStore
+from .event_hub import EventHub
+
+
+# ── JSON-RPC 2.0 error codes ──
+
+PARSE_ERROR = -32700
+INVALID_REQUEST = -32600
+METHOD_NOT_FOUND = -32601
+INVALID_PARAMS = -32602
+INTERNAL_ERROR = -32603
+
+# Kite custom error codes
+MODULE_OFFLINE = -32001
+RPC_TIMEOUT = -32002
+AUTH_FAILED = -32003
+PERMISSION_DENIED = -32004
+DUPLICATE_EVENT = -32005
+MODULE_DEGRADED = -32006
+TRACE_DEPTH_EXCEEDED = -32007
+
+DEFAULT_FORWARD_TIMEOUT = 5.0  # seconds
+
+# ── Trace 链路追踪配置 ──
+TRACE_CACHE_TTL = 300  # 5 分钟
+TRACE_CACHE_MAX = 1000  # 最多保留 1000 条 trace
+TRACE_MAX_DEPTH = 20  # 最大调用深度
+
+
+@dataclass
+class TraceSpan:
+    """一次 RPC 调用的追踪记录。"""
+    rpc_id: str
+    trace_id: str
+    caller: str
+    target: str
+    method: str
+    duration: float  # 毫秒
+    status: str  # success | error | timeout | offline
+    error: str | None = None
+    parent_rpc_id: str | None = None
+    depth: int = 1
+    timestamp: str = ""
+
+
+@dataclass
+class PendingForward:
+    """Tracks a forwarded cross-module RPC awaiting response."""
+    caller_ws: WebSocket
+    original_id: str
+    caller_id: str
+    target_id: str
+    method: str  # 完整方法名（如 "watchdog.get_status"）
+    params: dict  # 调用参数
+    trace_id: str = ""  # 链路追踪 ID
+    parent_rpc_id: str | None = None  # 上游 rpc_id
+    depth: int = 1  # 调用深度
+    created_at: float = field(default_factory=time.time)
+    timeout_handle: asyncio.TimerHandle | None = None
+
+
+def _result_msg(msg_id: str, result: dict) -> str:
+    return json.dumps({"jsonrpc": "2.0", "id": msg_id, "result": result})
+
+
+def _error_msg(msg_id: str, code: int, message: str, data: dict = None) -> str:
+    err = {"code": code, "message": message}
+    if data:
+        err["data"] = data
+    return json.dumps({"jsonrpc": "2.0", "id": msg_id, "error": err})
+
+
+class RpcRouter:
+    """JSON-RPC 2.0 message dispatcher.
+
+    Handles:
+    - 13 builtin methods (registry.*, event.*, kernel.*)
+    - Cross-module RPC forwarding with timeout
+    - RPC response matching for forwarded calls
+    """
+
+    def __init__(self, registry: RegistryStore, event_hub: EventHub,
+                 connections: dict[str, WebSocket], kernel_server):
+        self.registry = registry
+        self.event_hub = event_hub
+        self.connections = connections
+        self.kernel_server = kernel_server  # Direct reference to KernelServer
+
+        # RPC 统计计数器
+        self._rpc_total = 0  # 总 RPC 调用次数
+        self._rpc_builtin = 0  # 内置方法调用次数
+        self._rpc_forwarded = 0  # 转发调用次数
+        self._rpc_errors = 0  # 错误次数
+
+        # ── 链路追踪内存缓存 ──
+        # trace_id -> {"spans": [TraceSpan], "created_at": float, "origin": str}
+        self._trace_cache: dict[str, dict] = {}
+
+        # Builtin method dispatch table
+        self.methods: dict[str, callable] = {
+            "registry.register":      self._registry_register,
+            "registry.deregister":    self._registry_deregister,
+            "registry.lookup":        self._registry_lookup,
+            "registry.get":           self._registry_get,
+            "registry.verify":        self._registry_verify,
+            "event.publish":          self._event_publish,
+            "event.subscribe":        self._event_subscribe,
+            "event.unsubscribe":      self._event_unsubscribe,
+            "kernel.ping":            self._kernel_ping,
+            "kernel.stats":           self._kernel_stats,
+            "kernel.health":          self._kernel_health,
+            "kernel.latencies":       self._kernel_latencies,
+            "kernel.generate_tokens": self._kernel_generate_tokens,
+            "kernel.register_tokens": self._kernel_register_tokens,
+            "kernel.get_module_events": self._kernel_get_module_events,
+            "kernel.get_registry_stats": self._kernel_get_registry_stats,
+            "kernel.report_degraded":   self._kernel_report_degraded,
+            "kernel.report_recovered":  self._kernel_report_recovered,
+            "kernel.set_ordering_groups": self._kernel_set_ordering_groups,
+            "kernel.connection_status": self._kernel_connection_status,
+            "kernel.add_connection":    self._kernel_add_connection,
+            "kernel.flush_init":      self._kernel_flush_init,
+            "kernel.instance_pressure": self._kernel_instance_pressure,
+            "kernel.update_module_config": self._kernel_update_module_config,
+            "kernel.set_scaling":        self._kernel_set_scaling,
+            "trace.query":              self._trace_query,
+            "trace.list":               self._trace_list,
+        }
+
+        # Pending cross-module forwards: internal_id -> PendingForward
+        self._pending: dict[str, PendingForward] = {}
+
+    # ── Main dispatch ──
+
+    async def dispatch(self, caller_id: str, ws: WebSocket, msg: dict):
+        """Route a parsed JSON-RPC message to builtin handler or cross-module forward."""
+        method = msg.get("method", "")
+        msg_id = msg.get("id")
+
+        # 统计：总调用次数
+        self._rpc_total += 1
+
+        # JSON-RPC Notification (no id) — currently not handled from clients
+        if msg_id is None:
+            return
+
+        params = msg.get("params") or {}
+
+        # Builtin method
+        handler = self.methods.get(method)
+        if handler:
+            # 统计：内置方法调用
+            self._rpc_builtin += 1
+            try:
+                result = await handler(caller_id, params)
+            except Exception as e:
+                # 统计：错误次数
+                self._rpc_errors += 1
+                print(f"[kernel] RPC handler error ({method}): {e}")
+                try:
+                    await ws.send_text(_error_msg(msg_id, INTERNAL_ERROR, str(e)))
+                except Exception:
+                    # Can't send error response, connection closed
+                    pass
+                return
+
+            # Send result (may fail if connection closed during shutdown)
+            try:
+                await ws.send_text(_result_msg(msg_id, result))
+            except Exception as e:
+                # Connection closed during shutdown — this is normal, exit silently
+                pass
+            return
+
+        # Cross-module forward: method prefix is target module_id
+        dot_idx = method.find(".")
+        if dot_idx > 0:
+            target = method[:dot_idx]
+
+            # Resolve target to an available instance_key
+            # For multi-instance modules, pick one instance randomly
+            target_instance = self._resolve_rpc_target(target)
+
+            if target_instance:
+                # Check if target module is ready (state machine protection)
+                if not self.registry.is_ready(target):
+                    # 统计：错误次数
+                    self._rpc_errors += 1
+                    state = self.registry.get_state(target)
+                    await ws.send_text(_error_msg(
+                        msg_id, MODULE_OFFLINE,
+                        f"Module not ready: {target} (status: {state})",
+                        data={"module_status": state}))
+                    return
+                # 统计：转发调用
+                self._rpc_forwarded += 1
+                await self._forward(caller_id, ws, msg_id, target_instance, method, params,
+                                    target_module=target)
+                return
+            # Target not connected — check if registered but offline
+            if target in self.registry.modules:
+                # 统计：错误次数
+                self._rpc_errors += 1
+                await ws.send_text(_error_msg(
+                    msg_id, MODULE_OFFLINE, f"Module offline: {target}",
+                    data={"module_status": "offline"}))
+                return
+
+        # Method not found
+        # 统计：错误次数
+        self._rpc_errors += 1
+        await ws.send_text(_error_msg(msg_id, METHOD_NOT_FOUND, f"Method not found: {method}"))
+
+    async def handle_response(self, module_id: str, msg: dict):
+        """Handle an RPC response from a module (matches pending forwards)."""
+        msg_id = msg.get("id")
+        if not msg_id:
+            return
+
+        pending = self._pending.pop(msg_id, None)
+        if not pending:
+            return  # orphan response, ignore
+
+        # Cancel timeout
+        if pending.timeout_handle:
+            pending.timeout_handle.cancel()
+
+        # 计算耗时并发出 RPC 调用完成事件
+        duration = (time.time() - pending.created_at) * 1000
+        if "result" in msg:
+            self._publish_rpc_event(msg_id, pending, duration, "success", result=msg["result"])
+        elif "error" in msg:
+            error_msg = msg["error"].get("message", str(msg["error"])) if isinstance(msg["error"], dict) else str(msg["error"])
+            self._publish_rpc_event(msg_id, pending, duration, "error", error=error_msg)
+        else:
+            self._publish_rpc_event(msg_id, pending, duration, "success", result=None)
+
+        # Forward response to original caller with original ID
+        response = {"jsonrpc": "2.0", "id": pending.original_id}
+        if "result" in msg:
+            response["result"] = msg["result"]
+        elif "error" in msg:
+            response["error"] = msg["error"]
+        else:
+            response["result"] = None
+
+        try:
+            await pending.caller_ws.send_text(json.dumps(response))
+        except Exception:
+            # caller 原始连接失败 → 尝试同模块其他 slot
+            caller_slots = self.connections.get(pending.caller_id, {})
+            sent = False
+            for ws in caller_slots.values():
+                if ws is not pending.caller_ws:
+                    try:
+                        await ws.send_text(json.dumps(response))
+                        sent = True
+                        break
+                    except Exception:
+                        continue
+            if not sent:
+                pass  # caller 完全断开
+
+    def _resolve_rpc_target(self, target_module: str) -> str | None:
+        """Resolve a module_id to an available instance_key for RPC forwarding.
+
+        Strategy: random selection across all connected instances.
+        Returns None if no instance is connected.
+        """
+        import random
+        # Get all active instances from EventHub
+        inst_keys = self.event_hub.get_instance_keys(target_module)
+        if not inst_keys:
+            return None
+        # Filter to instances that actually have connections
+        connected = [k for k in inst_keys if k in self.connections and self.connections[k]]
+        if not connected:
+            return None
+        if len(connected) == 1:
+            return connected[0]
+        return random.choice(connected)
+
+    # ── Cross-module forwarding ──
+
+    async def _forward(self, caller_id: str, caller_ws: WebSocket,
+                       original_id: str, target: str, method: str, params: dict,
+                       target_module: str = ""):
+        """Forward RPC request to target module with timeout tracking.
+
+        Args:
+            target: instance_key to route to (e.g. 'assistant' or 'assistant#2')
+            target_module: base module_id for registry lookups. If empty, derived from target.
+        """
+        if not target_module:
+            from .event_hub import parse_instance_key
+            target_module = parse_instance_key(target)[0]
+
+        # 检查目标模块状态（用 base module_id）
+        target_state = self.registry.get_state(target_module)
+        if target_state == "degraded":
+            # 降级状态：检查调用的方法是否在 affected 列表中
+            degraded_info = self.registry.get_degraded_info(target_module)
+            if degraded_info:
+                actual_method = method[len(target_module) + 1:]
+                affected = degraded_info.get("affected", [])
+                if actual_method in affected:
+                    error_response = {
+                        "jsonrpc": "2.0",
+                        "id": original_id,
+                        "error": {
+                            "code": MODULE_DEGRADED,
+                            "message": f"Module '{target}' is degraded, method '{actual_method}' is affected",
+                            "data": {
+                                "module_status": "degraded",
+                                "level": degraded_info.get("level"),
+                                "reason": degraded_info.get("reason"),
+                                "retry_after_ms": degraded_info.get("estimated_recovery_ms"),
+                            }
+                        }
+                    }
+                    try:
+                        await caller_ws.send_text(json.dumps(error_response))
+                    except Exception:
+                        pass
+                    return
+        elif target_state != "ready":
+            # 目标模块不在 ready 状态，立即返回错误
+            error_response = {
+                "jsonrpc": "2.0",
+                "id": original_id,
+                "error": {
+                    "code": MODULE_OFFLINE,
+                    "message": f"Module '{target}' is not ready (state: {target_state})",
+                    "data": {
+                        "module_status": target_state,
+                    }
+                }
+            }
+            try:
+                await caller_ws.send_text(json.dumps(error_response))
+            except Exception:
+                pass
+            return
+
+        internal_id = f"fwd-{uuid.uuid4().hex[:12]}"
+
+        # Extract timeout from params (optional _timeout field)
+        timeout = DEFAULT_FORWARD_TIMEOUT
+        if isinstance(params, dict) and "_timeout" in params:
+            try:
+                timeout = float(params.pop("_timeout"))
+            except (ValueError, TypeError):
+                pass
+
+        # Strip target prefix from method for the forwarded request
+        actual_method = method[len(target_module) + 1:]  # e.g. "watchdog.get_status" -> "get_status"
+
+        # ── 链路追踪：生成或继承 _trace ──
+        if not isinstance(params, dict):
+            params = {}
+        upstream_trace = params.pop("_trace", None)
+
+        if upstream_trace and isinstance(upstream_trace, dict):
+            # 继承上游 trace
+            trace_id = upstream_trace.get("trace_id", str(uuid.uuid4()))
+            parent_rpc_id = upstream_trace.get("parent_rpc_id")
+            origin = upstream_trace.get("origin", caller_id)
+            depth = upstream_trace.get("depth", 0) + 1
+        else:
+            # 新链路
+            trace_id = str(uuid.uuid4())
+            parent_rpc_id = None
+            origin = caller_id
+            depth = 1
+
+        # 深度保护
+        if depth > TRACE_MAX_DEPTH:
+            await caller_ws.send_text(_error_msg(
+                original_id, TRACE_DEPTH_EXCEEDED,
+                f"Trace depth exceeded maximum ({TRACE_MAX_DEPTH}), possible recursive call chain"))
+            return
+
+        # 保存原始参数（用于事件发布，不包含 _caller_id 和 _trace）
+        original_params = dict(params)
+
+        # Inject caller_id into params for permission checking
+        params["_caller_id"] = caller_id
+
+        # 注入 _trace 到转发的 params（模块可读取也可忽略）
+        params["_trace"] = {
+            "trace_id": trace_id,
+            "parent_rpc_id": internal_id,  # 当前跳的 rpc_id 作为下游的 parent
+            "origin": origin,
+            "depth": depth,
+        }
+
+        # Record pending forward
+        loop = asyncio.get_event_loop()
+        pending = PendingForward(
+            caller_ws=caller_ws,
+            original_id=original_id,
+            caller_id=caller_id,
+            target_id=target,
+            method=method,
+            params=original_params,
+            trace_id=trace_id,
+            parent_rpc_id=parent_rpc_id,
+            depth=depth,
+        )
+        pending.timeout_handle = loop.call_later(
+            timeout, lambda iid=internal_id: asyncio.ensure_future(self._handle_timeout(iid))
+        )
+        self._pending[internal_id] = pending
+
+        # Send to target（从多 slot 中取任意一个可用连接）
+        fwd_msg = json.dumps({
+            "jsonrpc": "2.0",
+            "id": internal_id,
+            "method": actual_method,
+            "params": params or {},
+        })
+        target_slots = self.connections.get(target, {})
+        target_ws = next(iter(target_slots.values()), None) if target_slots else None
+        if target_ws:
+            try:
+                await target_ws.send_text(fwd_msg)
+            except Exception as e:
+                # Target send failed — clean up and error to caller
+                self._pending.pop(internal_id, None)
+                if pending.timeout_handle:
+                    pending.timeout_handle.cancel()
+
+                # 发出 RPC 调用失败事件
+                duration = (time.time() - pending.created_at) * 1000
+                self._publish_rpc_event(internal_id, pending, duration, "error", error=str(e))
+
+                await caller_ws.send_text(_error_msg(
+                    original_id, MODULE_OFFLINE, f"Failed to reach module: {target}"))
+        else:
+            # Target disconnected between check and send
+            self._pending.pop(internal_id, None)
+            if pending.timeout_handle:
+                pending.timeout_handle.cancel()
+
+            # 发出 RPC 调用失败事件
+            duration = (time.time() - pending.created_at) * 1000
+            self._publish_rpc_event(internal_id, pending, duration, "offline", error=f"Module offline: {target}")
+
+            await caller_ws.send_text(_error_msg(
+                original_id, MODULE_OFFLINE, f"Module offline: {target}"))
+
+    async def _handle_timeout(self, internal_id: str):
+        """Called when a forwarded RPC times out."""
+        pending = self._pending.pop(internal_id, None)
+        if not pending:
+            return
+
+        # 发出 RPC 调用超时事件
+        duration = (time.time() - pending.created_at) * 1000
+        self._publish_rpc_event(internal_id, pending, duration, "timeout", error=f"RPC timeout waiting for {pending.target_id}")
+
+        try:
+            await pending.caller_ws.send_text(_error_msg(
+                pending.original_id, RPC_TIMEOUT,
+                f"RPC timeout waiting for {pending.target_id}"))
+        except Exception:
+            pass
+
+    # ── Builtin handlers: registry.* ──
+
+    async def _registry_register(self, caller_id: str, params: dict) -> dict:
+        mid = params.get("module_id")
+        if not mid:
+            raise ValueError("module_id required")
+        # Permission: only Launcher or the module itself (including any instance of the module)
+        from .event_hub import parse_instance_key
+        caller_base = parse_instance_key(caller_id)[0]
+        if caller_id != "launcher" and caller_base != mid:
+            raise PermissionError(f"Module '{caller_id}' cannot register as '{mid}'")
+
+        # 幂等：同一 module_id 已注册（多实例的 #2~#N 会走到这里），直接返回
+        existing = self.registry.modules.get(mid)
+        if existing and existing.get("status") in ("registered", "ready"):
+            return {"module_id": mid, "changed": False}
+
+        print(f"[kernel] registry.register called by {caller_id} for module {mid}")
+
+        # Validate RPC method names: "." is reserved for cross-module routing
+        rpc_tool = params.get("tools", {}).get("rpc", {})
+        if isinstance(rpc_tool, dict):
+            bad_methods = []
+            for _cat, methods in rpc_tool.items():
+                if isinstance(methods, dict):
+                    for key, spec in methods.items():
+                        if "." in key:
+                            bad_methods.append(key)
+                        elif isinstance(spec, dict) and "." in spec.get("method", ""):
+                            bad_methods.append(spec["method"])
+            if bad_methods:
+                raise ValueError(
+                    f"RPC method names must not contain '.': {bad_methods}. "
+                    f"Use snake_case instead (e.g. 'llm_sessions_list' not 'llm_sessions.list'). "
+                    f"The '.' separator is reserved for cross-module routing.")
+
+        result = self.registry.register_module(params)
+        self.event_hub.publish_internal("module.registered", {"module_id": mid}, source=self.kernel_server.module_id)
+
+        # Only publish registry.updated if content actually changed
+        if result.get("changed", True):
+            from datetime import datetime, timezone
+            self.event_hub.publish_internal("registry.updated", {
+                "module_id": mid,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "action": "register",
+            }, source=self.kernel_server.module_id)
+            print(f"[kernel] registry.updated published for {mid}")
+        else:
+            print(f"[kernel] {mid} re-registered with no changes, skipping registry.updated")
+
+        # When Launcher registers, Kernel publishes its own module.ready
+        if mid == "launcher" and self.kernel_server:
+            self.kernel_server.publish_ready()
+            print(f"[kernel] launcher registered → kernel module.ready published")
+
+        return result
+
+    async def _registry_deregister(self, caller_id: str, params: dict) -> dict:
+        mid = params.get("module_id")
+        if not mid:
+            raise ValueError("module_id required")
+        if caller_id != "launcher" and caller_id != mid:
+            raise PermissionError(f"Module '{caller_id}' cannot deregister '{mid}'")
+
+        self.registry.deregister_module(mid)
+        self.event_hub.publish_internal("module.unregistered", {"module_id": mid}, source=self.kernel_server.module_id)
+
+        # Publish registry.updated event for cache invalidation
+        from datetime import datetime, timezone
+        self.event_hub.publish_internal("registry.updated", {
+            "module_id": mid,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "action": "deregister",
+        }, source=self.kernel_server.module_id)
+
+        return {}
+
+    async def _registry_lookup(self, caller_id: str, params: dict) -> dict:
+        field = params.get("field")
+        module = params.get("module")
+        value = params.get("value")
+
+        print(f"[kernel] registry.lookup called by {caller_id}: field={field}, module={module}, value={value}")
+
+        results = self.registry.lookup(
+            field=field,
+            module=module,
+            value=value,
+        )
+
+        print(f"[kernel] registry.lookup results: {len(results)} matches")
+        for r in results:
+            print(f"[kernel]   - {r['module']}.{r['field']} = {r['value']}")
+
+        return {
+            "results": results,
+            "last_update_time": self.registry.last_update_time
+        }
+
+    async def _registry_get(self, caller_id: str, params: dict) -> dict:
+        path = params.get("path", "")
+        if not path:
+            raise ValueError("path required")
+        val, found = self.registry.get_by_path(path)
+        if not found:
+            raise KeyError(f"Path not found: {path}")
+        return {"value": val}
+
+    async def _registry_verify(self, caller_id: str, params: dict) -> dict:
+        token = params.get("token", "")
+        module_id = self.registry.verify_token(token)
+        if not module_id:
+            raise PermissionError("Invalid token")
+        return {"module_id": module_id}
+
+    # ── Builtin handlers: event.* ──
+
+    async def _event_publish(self, caller_id: str, params: dict) -> dict:
+        event_id = params.get("event_id", "")
+        event_type = params.get("event", "")
+        data = params.get("data")
+        echo = params.get("echo", False)
+        droppable = params.get("droppable", False)
+
+        # 提取并校验 priority
+        priority = params.get("priority", "normal")
+        if priority not in ("normal", "high", "critical"):
+            raise ValueError(f"Invalid priority: {priority}, must be 'normal', 'high', or 'critical'")
+
+        # 提取节流参数
+        throttle_key = params.get("throttle_key", "")
+        throttle_ms = params.get("throttle_ms", 0)
+        if not isinstance(throttle_ms, (int, float)):
+            throttle_ms = 0
+
+        # When a module publishes module.ready, update its status in registry
+        # and sync queue_elastic capability to EventHub
+        if event_type == "module.ready":
+            # caller_id is instance_key (e.g. 'assistant' or 'assistant#2')
+            mid = (data or {}).get("module_id", caller_id)
+            self.registry.set_ready(mid, data)
+            # Sync elastic capability to EventHub (per instance_key for queue management)
+            is_elastic = (data or {}).get("queue_elastic", False)
+            self.event_hub.set_module_elastic(caller_id, bool(is_elastic))
+
+            # 多连接/多实例：从配置文件读取并统一应用
+            asyncio.ensure_future(self._apply_module_config(mid, caller_id))
+
+        # When a module reports queue pressure, store it per instance
+        elif event_type == "module.queue_pressure":
+            self.event_hub.report_instance_pressure(caller_id, data or {})
+
+        return self.event_hub.publish_event(caller_id, event_id, event_type, data, echo,
+                                            droppable=droppable, priority=priority,
+                                            throttle_key=throttle_key,
+                                            throttle_ms=int(throttle_ms))
+
+    async def _event_subscribe(self, caller_id: str, params: dict) -> dict:
+        events = params.get("events", [])
+        if not isinstance(events, list) or not events:
+            raise ValueError("events must be a non-empty list")
+        self.event_hub.handle_subscribe(caller_id, events)
+        # Update init time for this instance (caller_id may be instance_key like 'assistant#2')
+        self.registry.update_init_time(caller_id)
+        return {}
+
+    async def _event_unsubscribe(self, caller_id: str, params: dict) -> dict:
+        events = params.get("events", [])
+        if not isinstance(events, list) or not events:
+            raise ValueError("events must be a non-empty list")
+        return self.event_hub.handle_unsubscribe(caller_id, events)
+
+    # ── Builtin handlers: kernel.* ──
+
+    async def _kernel_ping(self, caller_id: str, params: dict) -> dict:
+        return {"pong": True, "timestamp": datetime.now(timezone.utc).isoformat()}
+
+    async def _kernel_stats(self, caller_id: str, params: dict) -> dict:
+        event_stats = self.event_hub.get_stats()
+        return {
+            **event_stats,
+            "rpc": {
+                "total": self._rpc_total,
+                "builtin": self._rpc_builtin,
+                "forwarded": self._rpc_forwarded,
+                "errors": self._rpc_errors
+            }
+        }
+
+    async def _kernel_health(self, caller_id: str, params: dict) -> dict:
+        eh_health = self.event_hub.get_health()
+        return {
+            "status": "healthy",
+            "module_count": len(self.registry.modules),
+            "online_count": sum(
+                1 for m in self.registry.modules.values()
+                if m.get("status") in ("registered", "ready", "degraded")
+            ),
+            "degraded_count": sum(
+                1 for m in self.registry.modules.values()
+                if m.get("status") == "degraded"
+            ),
+            "event_stats": eh_health.get("details", {}),
+        }
+
+    async def _kernel_latencies(self, caller_id: str, params: dict) -> dict:
+        """Get ping/pong latencies for all connected modules.
+
+        Returns:
+            {
+                "latencies": {
+                    "module1": {
+                        "outbound": 12.34,  # ms, Kernel → module
+                        "inbound": 23.45,   # ms, module → Kernel
+                        "status": "ok" | "timeout" | "never"
+                    },
+                    ...
+                }
+            }
+        """
+        import time
+
+        # 更新最后调用时间，触发 ping 间隔调整
+        current_time = time.time()
+        old_time = self.kernel_server._last_latencies_call
+        self.kernel_server._last_latencies_call = current_time
+
+        # 如果是首次调用或距离上次调用超过 30 秒，触发间隔变化
+        if old_time == 0 or (current_time - old_time) > 30:
+            # 通知 ping 循环间隔可能需要调整
+            self.kernel_server._ping_interval_event.set()
+
+        result = {}
+        for module_id in [mid for mid, slots in self.kernel_server.connections.items() if slots]:
+            latency_data = self.kernel_server._pong_latencies.get(module_id, {})
+            status = self.kernel_server._pong_status.get(module_id, "never")
+
+            result[module_id] = {
+                "outbound": latency_data.get("outbound"),
+                "inbound": latency_data.get("inbound"),
+                "status": status,
+                "last_update": latency_data.get("last_update"),
+            }
+
+        # 添加 kernel 自己的数据
+        if "kernel" in self.kernel_server._pong_latencies:
+            kernel_data = self.kernel_server._pong_latencies["kernel"]
+            result["kernel"] = {
+                "outbound": kernel_data.get("outbound"),
+                "inbound": kernel_data.get("inbound"),
+                "status": self.kernel_server._pong_status.get("kernel", "never"),
+                "last_update": kernel_data.get("last_update"),
+            }
+
+        return {"latencies": result}
+
+    async def _kernel_generate_tokens(self, caller_id: str, params: dict) -> dict:
+        """Generate tokens for a list of module names.
+
+        Args:
+            params: {"modules": ["mod1", "mod2", ...]}
+
+        Returns:
+            {"tokens": {"mod1": "token1", "mod2": "token2", ...}}
+        """
+        # Only Launcher may request token generation
+        if caller_id != "launcher":
+            raise PermissionError("Only Launcher may generate tokens")
+
+        modules = params.get("modules", [])
+        if not isinstance(modules, list):
+            raise ValueError("modules must be a list")
+
+        import secrets
+        tokens = {}
+        for module_name in modules:
+            tokens[module_name] = secrets.token_hex(32)
+
+        # Register tokens in registry
+        self.registry.register_tokens(tokens)
+
+        return {"tokens": tokens}
+
+    async def _kernel_register_tokens(self, caller_id: str, params: dict) -> dict:
+        """注册 token 映射（运行时动态注册，如启动新模块、浏览器连接）。"""
+        if caller_id != "launcher":
+            raise PermissionError("Only Launcher may register tokens")
+        remote = bool(params.pop("__remote__", False))
+        self.registry.register_tokens(params, remote=remote)
+        return {}
+
+    async def _kernel_get_module_events(self, caller_id: str, params: dict) -> dict:
+        """获取模块的事件订阅和发布信息（含详细信息和动态统计）"""
+        module_name = params.get("module_name")
+        if not module_name:
+            raise ValueError("module_name is required")
+
+        # 从订阅表查询实际订阅（subscriptions: module_id -> set of (pattern_str, pattern_tuple)）
+        active_subs = []
+        module_patterns = self.event_hub.subscriptions.get(module_name, set())
+        for pattern_str, pattern_tuple in module_patterns:
+            active_subs.append(pattern_str)
+
+        # 从注册数据读取声明（events_subscribe: list, events_publish: dict）
+        module_data = self.registry.modules.get(module_name, {})
+        declared_subs = module_data.get("events_subscribe", [])
+        pub_map = self._flatten_event_publish(module_data.get("events_publish", {}))
+        declared_pubs = sorted(pub_map.keys())
+
+        # 构建每个事件的详细信息
+        event_details = {}
+
+        # 订阅事件的详情：找出谁发布了这个事件
+        all_events = set(active_subs) | set(declared_subs)
+        for event in all_events:
+            info = self.event_hub.get_event_info(event)
+            # 找出哪些模块声明发布此事件
+            publishers = self._find_publishers(event)
+            event_details[event] = {
+                "description": None,
+                "publishers": publishers,
+                "stats": info["stats"],
+            }
+
+        # 发布事件的详情：找出谁订阅了这个事件
+        for event in declared_pubs:
+            info = self.event_hub.get_event_info(event)
+            event_details[event] = {
+                "description": pub_map.get(event),
+                "subscribers": info["subscribers"],
+                "stats": info["stats"],
+            }
+
+        return {
+            "module_name": module_name,
+            "subscriptions": {
+                "active": sorted(active_subs),
+                "declared": sorted(declared_subs)
+            },
+            "publications": {
+                "declared": declared_pubs
+            },
+            "event_details": event_details,
+        }
+
+    def _find_publishers(self, event_type: str) -> list[str]:
+        """查找声明发布某事件的所有模块"""
+        result = []
+        for mid, data in self.registry.modules.items():
+            if data.get("status") not in ("registered", "ready", "degraded"):
+                continue
+            pub_map = self._flatten_event_publish(data.get("events_publish", {}))
+            if event_type in pub_map:
+                result.append(mid)
+        return sorted(result)
+
+    @staticmethod
+    def _flatten_event_publish(d: dict, prefix: str = "") -> dict[str, str | None]:
+        """将嵌套 events_publish dict 展平为 {事件名: description}。
+        含 description 键的 dict 视为叶子节点（事件定义）。"""
+        result = {}
+        for key, value in d.items():
+            path = f"{prefix}.{key}" if prefix else key
+            if isinstance(value, dict) and "description" not in value:
+                result.update(RpcRouter._flatten_event_publish(value, path))
+            else:
+                desc = value.get("description") if isinstance(value, dict) else None
+                result[path] = desc
+        return result
+
+    async def _kernel_get_registry_stats(self, caller_id: str, params: dict) -> dict:
+        """获取注册中心统计数据（用于前端统计面板）
+
+        Returns:
+            {
+                "total_records": int,  # 总注册记录数
+                "by_category": {
+                    "rpc": int,    # tools.rpc.* 数量
+                    "hook": int,   # tools.hook.* 数量
+                    "api": int     # tools.api.* 数量
+                }
+            }
+        """
+        total_records = 0
+        by_category = {
+            "rpc": 0,
+            "hook": 0,
+            "api": 0
+        }
+
+        # 遍历所有已注册模块
+        for module_id, module_data in self.registry.modules.items():
+            if module_data.get("status") not in ("registered", "ready", "degraded"):
+                continue
+
+            # 分别查询 RPC、Hook、API（需要查询到叶子节点）
+            rpc_results = self.registry.lookup(field="tools.rpc.*", module=module_id)
+            hook_results = self.registry.lookup(field="tools.hook.*", module=module_id)
+            api_results = self.registry.lookup(field="tools.api.*", module=module_id)
+
+            by_category["rpc"] += len(rpc_results)
+            by_category["hook"] += len(hook_results)
+            by_category["api"] += len(api_results)
+            total_records += len(rpc_results) + len(hook_results) + len(api_results)
+
+        return {
+            "total_records": total_records,
+            "by_category": by_category
+        }
+
+    # ── Builtin handlers: kernel.report_degraded / kernel.report_recovered ──
+
+    async def _kernel_report_degraded(self, caller_id: str, params: dict) -> dict:
+        """模块报告自身降级。Kernel 验证后更新 Registry 并广播 module.degraded 系统事件。
+        Watchdog 可代发（通过 module_id 参数指定目标模块）。
+
+        Args:
+            params: {
+                module_id: str (可选，默认为 caller_id；Watchdog 代发时指定目标模块),
+                level: "full" | "partial" | "slow",
+                reason: str,
+                affected: list[str] (partial 时必填),
+                available: list[str] (可选),
+                estimated_recovery_ms: int (可选),
+                detail: str (可选),
+            }
+        """
+        # 确定目标模块：默认是调用者自身，Watchdog 可代发
+        target_module = params.get("module_id", caller_id)
+        if target_module != caller_id and caller_id != "watchdog":
+            raise PermissionError(f"Module '{caller_id}' cannot report degradation for '{target_module}'")
+
+        level = params.get("level")
+        if level not in ("full", "partial", "slow"):
+            raise ValueError(f"Invalid degradation level: {level}, must be 'full', 'partial', or 'slow'")
+
+        reason = params.get("reason")
+        if not reason:
+            raise ValueError("reason is required")
+
+        if level == "partial" and not params.get("affected"):
+            raise ValueError("affected list is required when level is 'partial'")
+
+        # 更新 Registry
+        self.registry.set_degraded(target_module, params)
+
+        # 广播 module.degraded 系统事件
+        event_data = {
+            "module_id": target_module,
+            "level": level,
+            "reason": reason,
+            "affected": params.get("affected", []),
+            "available": params.get("available", []),
+            "detail": params.get("detail"),
+        }
+        if params.get("estimated_recovery_ms") is not None:
+            event_data["estimated_recovery_ms"] = params["estimated_recovery_ms"]
+
+        self.event_hub.publish_internal(
+            "module.degraded", event_data,
+            source=caller_id,
+            priority="high"
+        )
+
+        print(f"[kernel] module.degraded: {target_module} level={level} reason={reason} (reported by {caller_id})")
+        return {}
+
+    async def _kernel_report_recovered(self, caller_id: str, params: dict) -> dict:
+        """模块报告自身从降级恢复。Kernel 验证后更新 Registry 并广播 module.recovered 系统事件。
+        Watchdog 可代发（通过 module_id 参数指定目标模块）。
+
+        Args:
+            params: {
+                module_id: str (可选，默认为 caller_id),
+                previous_level: str (可选),
+                duration_ms: int (可选),
+            }
+        """
+        target_module = params.get("module_id", caller_id)
+        if target_module != caller_id and caller_id != "watchdog":
+            raise PermissionError(f"Module '{caller_id}' cannot report recovery for '{target_module}'")
+
+        # 获取降级信息用于事件数据
+        degraded_info = self.registry.get_degraded_info(target_module)
+        previous_level = params.get("previous_level")
+        duration_ms = params.get("duration_ms")
+
+        if degraded_info and not previous_level:
+            previous_level = degraded_info.get("level")
+        if degraded_info and not duration_ms:
+            degraded_at = degraded_info.get("degraded_at")
+            if degraded_at:
+                import time as _time
+                duration_ms = int((_time.time() - degraded_at) * 1000)
+
+        # 更新 Registry
+        self.registry.clear_degraded(target_module)
+
+        # 广播 module.recovered 系统事件
+        event_data = {"module_id": target_module}
+        if previous_level:
+            event_data["previous_level"] = previous_level
+        if duration_ms is not None:
+            event_data["duration_ms"] = duration_ms
+
+        self.event_hub.publish_internal(
+            "module.recovered", event_data,
+            source=caller_id,
+        )
+
+        print(f"[kernel] module.recovered: {target_module} (reported by {caller_id})")
+        return {}
+
+    # ── Builtin handlers: kernel.set_ordering_groups ──
+
+    async def _kernel_set_ordering_groups(self, caller_id: str, params: dict) -> dict:
+        """模块声明保序集合，同组事件 key 保证通过同一 slot 发送。
+
+        Args:
+            params: {
+                "groups": [
+                    {"name": "session", "keys": ["session.started", "session.ended"]}
+                ]
+            }
+        """
+        groups = params.get("groups", [])
+        if not isinstance(groups, list):
+            raise ValueError("groups must be a list")
+        self.event_hub.set_ordering_groups(caller_id, groups)
+        return {}
+
+    async def _kernel_connection_status(self, caller_id: str, params: dict) -> dict:
+        """返回所有模块的连接槽位状态。"""
+        modules = {}
+        for module_id, slots in self.kernel_server.connections.items():
+            if not slots:
+                continue
+            modules[module_id] = {
+                "elastic": self.kernel_server._module_conn_elastic.get(module_id, False),
+                "slots": {str(slot): {"connected": True} for slot in slots}
+            }
+        return {"modules": modules}
+
+    async def _kernel_add_connection(self, caller_id: str, params: dict) -> dict:
+        """手动为指定模块增加一个连接 slot。
+
+        Params: module_id (str)
+        Returns: {slot: int} — 新增的 slot 编号
+        """
+        module_id = params.get("module_id", "")
+        if not module_id:
+            raise ValueError("module_id required")
+
+        current_slots = self.kernel_server.connections.get(module_id, {})
+        if not current_slots:
+            raise RuntimeError(f"模块 {module_id} 未连接")
+
+        max_conns = self.kernel_server._get_max_connections(module_id)
+
+        # 找到下一个空闲 slot
+        next_slot = None
+        for s in range(1, max_conns):
+            if s not in current_slots:
+                next_slot = s
+                break
+
+        if next_slot is None:
+            raise RuntimeError(f"模块 {module_id} 连接已满 ({len(current_slots)}/{max_conns})")
+
+        self.kernel_server._send_connection_offer(module_id, [next_slot])
+        return {"slot": next_slot}
+
+    async def _kernel_instance_pressure(self, caller_id: str, params: dict) -> dict:
+        """Query instance pressure reports.
+
+        Params:
+            module_id (optional): filter by module name. If omitted, return all.
+
+        Returns: {instances: {instance_key: {level, depth, capacity, updated_at}, ...}}
+        """
+        module_id = params.get("module_id", "")
+        if module_id:
+            result = self.event_hub.get_module_pressure(module_id)
+        else:
+            result = dict(self.event_hub._instance_pressure)
+        return {"instances": result}
+
+    async def _kernel_update_module_config(self, caller_id: str, params: dict) -> dict:
+        """热更新模块的多连接/多实例配置。
+
+        从配置文件（通过 launcher.get_module_config）读取最新配置并应用。
+        前端在调用此方法前应先调用 launcher.update_module_config 写入 module.md。
+
+        Params:
+            module_id (str): 目标模块 ID
+        """
+        module_id = params.get("module_id", "")
+        if not module_id:
+            raise RuntimeError("module_id is required")
+
+        if not self.registry.modules.get(module_id):
+            raise RuntimeError(f"模块 {module_id} 未注册")
+
+        if self.registry.is_remote_module(module_id):
+            raise RuntimeError(f"模块 {module_id} 是远程模块，不支持热更新配置")
+
+        result = await self._apply_module_config(module_id, module_id)
+        return result
+
+    async def _kernel_set_scaling(self, caller_id: str, params: dict) -> dict:
+        """全局冻结/解冻扩缩容控制面。仅 Launcher 可调用。
+
+        冻结时：_scaler_tick 跳过所有扩缩容操作，多连接 recovery offer 不发送。
+        解冻时：立即触发一次 scaler tick 以补齐目标状态。
+
+        Params:
+            enabled (bool): True=解冻（正常运行），False=冻结
+        """
+        if caller_id != "launcher":
+            raise PermissionError("Only launcher can control scaling")
+
+        enabled = bool(params.get("enabled", True))
+        server = self.kernel_server
+        server._scaling_frozen = not enabled
+
+        if enabled:
+            # 解冻后立即触发一次检查，补齐实例/连接数
+            asyncio.ensure_future(server._scaler_tick())
+            print(f"[kernel] 扩缩容已解冻，立即触发一次检查")
+        else:
+            print(f"[kernel] 扩缩容已冻结")
+
+        return {"enabled": enabled, "frozen": not enabled}
+
+    async def _apply_module_config(self, module_id: str, instance_key: str) -> dict:
+        """统一的配置应用入口：从配置文件读取多连接/多实例配置并更新 Kernel 状态。
+
+        扩缩容执行由 _scaler_tick 统一负责，本方法只更新配置并唤醒 scaler。
+
+        Args:
+            module_id: 模块名（如 'acp_channel'），用于读取配置文件和管理实例
+            instance_key: 实例键（如 'acp_channel' 或 'acp_channel#2'），用于管理连接
+        """
+        server = self.kernel_server
+
+        # 远程模块（通过 Relay 连入）无配置文件，Launcher 不管理其生命周期。
+        # 多连接配置由模块自身在 registry.register 时声明，从注册表读取后应用。
+        # 多实例扩缩容不适用（Launcher 不管理远程模块生命周期）。
+        if self.registry.is_remote_module(module_id):
+            mod = self.registry.modules.get(module_id, {})
+            max_conns = max(1, min(10, int(mod.get("max_connections", 1))))
+            conn_elastic = bool(mod.get("connection_elastic", False))
+            # 取消旧收缩任务（如从弹性改为非弹性）
+            if not conn_elastic and instance_key in server._shrink_tasks:
+                server._shrink_tasks[instance_key].cancel()
+                del server._shrink_tasks[instance_key]
+            server._module_conn_elastic[instance_key] = conn_elastic
+            # max_connections 已在 registry 中，_get_max_connections() 可直接读取
+            if max_conns > 1:
+                print(f"[kernel] remote module {module_id}: max_connections={max_conns}, connection_elastic={conn_elastic}")
+                server._initiate_multi_connection(instance_key, max_conns)
+            return {
+                "module_id": module_id,
+                "max_connections": max_conns,
+                "connection_elastic": conn_elastic,
+            }
+
+        # 从 Launcher 读取配置文件（唯一权威来源）
+        try:
+            config = await server._call_launcher_rpc("get_module_config", {"module_name": module_id})
+        except Exception as e:
+            print(f"[kernel] _apply_module_config: 获取 {module_id} 配置失败: {e}")
+            return {"module_id": module_id, "error": str(e)}
+
+        max_conns = max(1, min(10, int(config.get("max_connections", 1))))
+        conn_elastic = bool(config.get("connection_elastic", False))
+        max_inst = max(1, min(10, int(config.get("max_instances", 1))))
+        inst_elastic = bool(config.get("instance_elastic", False))
+
+        # ── 更新多连接配置 ──
+        # 如果从弹性改为非弹性，取消旧的收缩任务
+        if not conn_elastic and instance_key in server._shrink_tasks:
+            server._shrink_tasks[instance_key].cancel()
+            del server._shrink_tasks[instance_key]
+        server._module_conn_elastic[instance_key] = conn_elastic
+        # 更新注册中心的 max_connections（供 _get_max_connections 读取）
+        mod = self.registry.modules.get(module_id)
+        if mod is not None:
+            mod["max_connections"] = max_conns
+
+        # ── 更新多实例配置 ──
+        server._module_instance_config[module_id] = {
+            "max_instances": max_inst,
+            "instance_elastic": inst_elastic,
+        }
+
+        print(f"[kernel] apply_module_config({module_id}): max_connections={max_conns}, "
+              f"connection_elastic={conn_elastic}, max_instances={max_inst}, "
+              f"instance_elastic={inst_elastic}")
+
+        # ── 触发多连接建立 ──
+        if max_conns > 1:
+            server._initiate_multi_connection(instance_key, max_conns)
+
+        # 触发一次 scaler tick 以执行扩缩容（仅在未冻结时生效）
+        if not server._scaling_frozen:
+            asyncio.ensure_future(server._scaler_tick())
+
+        return {
+            "module_id": module_id,
+            "max_connections": max_conns,
+            "connection_elastic": conn_elastic,
+            "max_instances": max_inst,
+            "instance_elastic": inst_elastic,
+        }
+
+    async def _trim_excess_instances(self, module_id: str, max_inst: int):
+        """将超出 max_inst 的多余实例标记为 draining（等队列空再关闭）。"""
+        server = self.kernel_server
+        from .event_hub import parse_instance_key
+        inst_keys = self.event_hub.get_instance_keys(module_id)
+        for k in inst_keys:
+            _, num = parse_instance_key(k)
+            if num > max_inst and k not in self.event_hub._draining_instances:
+                self.event_hub.set_draining(k)
+                print(f"[kernel] Draining excess instance: {module_id}#{num} (max={max_inst})")
+
+    async def _kernel_flush_init(self, caller_id: str, params: dict) -> dict:
+        """打开 require_init 门控，对所有暂存的模块发送 system.require_init。
+        由 Launcher 在恢复模式下注册好 event waiters 后调用。"""
+        server = self.kernel_server
+        server._init_gate_open = True
+        pending = server._init_pending[:]
+        server._init_pending.clear()
+
+        sent = 0
+        for inst_key, ws in pending:
+            # 跳过已断开的连接（stale ws）
+            if inst_key not in server.connections or not server.connections[inst_key]:
+                continue
+            if server.registry.needs_init(inst_key):
+                from .event_hub import parse_instance_key
+                base_mid, inst_num = parse_instance_key(inst_key)
+                try:
+                    await ws.send_text(json.dumps({
+                        "jsonrpc": "2.0",
+                        "method": "event",
+                        "params": {
+                            "event": "system.require_init",
+                            "source": server.module_id,
+                            "data": {"module_id": base_mid, "instance_num": inst_num},
+                        },
+                    }))
+                    sent += 1
+                    print(f"[kernel] Sent deferred system.require_init to {inst_key}")
+                except Exception as e:
+                    print(f"[kernel] Failed to send deferred system.require_init to {inst_key}: {e}")
+
+        print(f"[kernel] Init gate opened, sent {sent} deferred require_init")
+        return {"flushed": sent}
+
+    # ── RPC Event Publishing ──
+
+    def _publish_rpc_event(self, rpc_id: str, pending: PendingForward, duration: float,
+                           status: str, result=None, error: str = None):
+        """发出 RPC 调用完成事件（按需工作：只有订阅者时才发出）
+
+        Args:
+            rpc_id: RPC 调用 ID
+            pending: PendingForward 对象
+            duration: 耗时（毫秒）
+            status: 状态 ("success" | "error" | "timeout" | "offline")
+            result: 成功时的返回值（可选）
+            error: 失败时的错误信息（可选）
+        """
+        # 检查是否有订阅者（优化：没人订阅就不发出事件）
+        has_subscribers = False
+        for patterns in self.event_hub.subscriptions.values():
+            for pattern_str, _ in patterns:
+                if pattern_str == "rpc.call.completed" or pattern_str.startswith("rpc.") or pattern_str == "*" or pattern_str == ">":
+                    has_subscribers = True
+                    break
+            if has_subscribers:
+                break
+
+        if not has_subscribers:
+            return
+
+        # 构造事件数据
+        event_data = {
+            "rpc_id": rpc_id,
+            "trace_id": pending.trace_id,
+            "caller": pending.caller_id,
+            "target": pending.target_id,
+            "method": pending.method,
+            "params": pending.params,
+            "duration": round(duration, 2),
+            "status": status,
+            "parent_rpc_id": pending.parent_rpc_id,
+            "depth": pending.depth,
+        }
+
+        if result is not None:
+            event_data["result"] = result
+        if error:
+            event_data["error"] = error
+
+        # 记录到 trace 缓存
+        self._add_to_trace_cache(pending.trace_id, TraceSpan(
+            rpc_id=rpc_id,
+            trace_id=pending.trace_id,
+            caller=pending.caller_id,
+            target=pending.target_id,
+            method=pending.method,
+            duration=round(duration, 2),
+            status=status,
+            error=error,
+            parent_rpc_id=pending.parent_rpc_id,
+            depth=pending.depth,
+            timestamp=datetime.now(timezone.utc).isoformat(),
+        ))
+
+        # 发出事件
+        self.event_hub.publish_internal("rpc.call.completed", event_data, source="kernel")
+
+    # ── Trace 缓存管理 ──
+
+    def _add_to_trace_cache(self, trace_id: str, span: TraceSpan):
+        """添加 span 到 trace 缓存，自动清理过期条目"""
+        now = time.time()
+
+        # 清理过期条目
+        expired = [tid for tid, data in self._trace_cache.items()
+                   if now - data["created_at"] > TRACE_CACHE_TTL]
+        for tid in expired:
+            self._trace_cache.pop(tid, None)
+
+        # 容量限制：FIFO 淘汰
+        if len(self._trace_cache) >= TRACE_CACHE_MAX:
+            oldest = min(self._trace_cache.items(), key=lambda x: x[1]["created_at"])
+            self._trace_cache.pop(oldest[0], None)
+
+        # 添加 span
+        if trace_id not in self._trace_cache:
+            self._trace_cache[trace_id] = {
+                "spans": [],
+                "created_at": now,
+                "origin": span.caller,
+            }
+        self._trace_cache[trace_id]["spans"].append(span)
+
+    async def _trace_query(self, caller_id: str, params: dict) -> dict:
+        """查询指定 trace_id 的完整调用链"""
+        trace_id = params.get("trace_id")
+        if not trace_id:
+            raise ValueError("trace_id is required")
+
+        trace_data = self._trace_cache.get(trace_id)
+        if not trace_data:
+            return {"trace_id": trace_id, "found": False}
+
+        spans = trace_data["spans"]
+        total_duration = sum(s.duration for s in spans)
+
+        return {
+            "trace_id": trace_id,
+            "found": True,
+            "origin": trace_data["origin"],
+            "span_count": len(spans),
+            "total_duration_ms": round(total_duration, 2),
+            "spans": [
+                {
+                    "rpc_id": s.rpc_id,
+                    "caller": s.caller,
+                    "target": s.target,
+                    "method": s.method,
+                    "duration": s.duration,
+                    "status": s.status,
+                    "error": s.error,
+                    "parent_rpc_id": s.parent_rpc_id,
+                    "depth": s.depth,
+                    "timestamp": s.timestamp,
+                }
+                for s in spans
+            ],
+        }
+
+    async def _trace_list(self, caller_id: str, params: dict) -> dict:
+        """列出最近的 trace 摘要"""
+        limit = params.get("limit", 20)
+
+        traces = []
+        for trace_id, data in sorted(
+            self._trace_cache.items(),
+            key=lambda x: x[1]["created_at"],
+            reverse=True
+        )[:limit]:
+            spans = data["spans"]
+            total_duration = sum(s.duration for s in spans)
+            has_error = any(s.status in ("error", "timeout", "offline") for s in spans)
+            started_at = spans[0].timestamp if spans else ""
+
+            traces.append({
+                "trace_id": trace_id,
+                "origin": data["origin"],
+                "span_count": len(spans),
+                "total_duration_ms": round(total_duration, 2),
+                "has_error": has_error,
+                "started_at": started_at,
+            })
+
+        return {
+            "traces": traces,
+            "total": len(self._trace_cache),
+        }
+