@agentunion/kite 1.4.0 → 1.6.0

package/kernel/server.py

@@ -7,21 +7,26 @@ Merges Registry + Event Hub into a single process.
 import asyncio
 import json
 import os
+import time
 
 from fastapi import FastAPI
 from starlette.websockets import WebSocket, WebSocketDisconnect
 
 from .registry_store import RegistryStore
-from .event_hub import EventHub
+from .event_hub import EventHub, instance_key, parse_instance_key
 from .rpc_router import RpcRouter
 
 try:
     import orjson
     def _loads(raw: str):
         return orjson.loads(raw)
+    def _dumps(obj) -> str:
+        return orjson.dumps(obj).decode('utf-8')
 except ImportError:
     def _loads(raw: str):
         return json.loads(raw)
+    def _dumps(obj) -> str:
+        return json.dumps(obj)
 
 
 class KernelServer:
@@ -33,20 +38,33 @@ class KernelServer:
     - Event notifications (delivered to subscribers)
     """
 
-    def __init__(self, launcher_token: str = None, advertise_ip: str = "127.0.0.1", module_id: str = None):
+    def __init__(self, launcher_token: str = None, advertise_ip: str = "127.0.0.1", module_id: str = None, boot_t0: float = 0, is_recover: bool = False):
         if module_id is None:
             raise ValueError("module_id is required")
         self.module_id = module_id
         self.advertise_ip = advertise_ip
         self.port: int = 0  # set by entry.py before uvicorn.run
+        self.boot_t0 = boot_t0  # Startup time for ready event
+        self.is_recover = is_recover  # 恢复模式标记，用于 module.ready reason 字段
+
+        # TLS configuration
+        # 默认值根据环境变量：开发环境允许 WS，生产环境强制 WSS
+        env = os.environ.get("KITE_ENV", "development").lower()
+        self.require_tls = os.environ.get("KITE_REQUIRE_TLS", "true" if env == "production" else "false").lower() == "true"
 
         # Core components
         self.registry = RegistryStore(launcher_token)  # Can be None
         self.event_hub = EventHub()
 
-        # Shared connection table (module_id -> WebSocket)
+        # Shared connection table (module_id -> {slot -> WebSocket})
         # RpcRouter and EventHub both reference this
-        self.connections: dict[str, WebSocket] = {}
+        self.connections: dict[str, dict[int, WebSocket]] = {}
+
+        # ── 多连接状态 ──
+        self._module_conn_elastic: dict[str, bool] = {}   # mid → 是否弹性连接
+        self._offer_tasks: dict[str, asyncio.Task] = {}   # mid → offer task
+        self._shrink_tasks: dict[str, asyncio.Task] = {}  # mid → shrink task
+        self._release_timers: dict[str, asyncio.Task] = {}  # mid → release timer
 
         # RPC router (pass self reference)
         self.rpc_router = RpcRouter(
@@ -57,7 +75,6 @@ class KernelServer:
         )
 
         # Background tasks
-        self._ttl_task: asyncio.Task | None = None
         self._dedup_task: asyncio.Task | None = None
         self._uvicorn_server = None  # set by entry.py for graceful shutdown
         self._shutting_down = False
@@ -67,19 +84,67 @@ class KernelServer:
         self._launcher_subscribed = False
         self._ready_published = False
 
+        # 恢复模式 require_init 门控：关闭时暂缓发送，等 Launcher 就绪后通过 RPC 打开
+        self._init_gate_open = True  # 正常启动时打开，恢复模式由 entry.py 设为 False
+        self._init_pending: list[tuple] = []  # 门控关闭时暂存的 (module_id, ws) 对
+
         # Subscribe to events that Kernel needs to handle
         # Kernel 通过订阅机制接收事件（与其他模块一致）
-        self.event_hub.handle_subscribe(self.module_id, ["module.shutdown"])
+        self.event_hub.handle_subscribe(self.module_id, ["module.shutdown", "system.pong", "system.ready"])
 
         # Register internal event callback for Kernel
         # Kernel 自身没有 WebSocket 连接，通过回调机制接收订阅的事件
         self.event_hub.register_internal_callback(self.module_id, self._handle_internal_event)
 
+        # 正在退出的模块集合（收到 module.shutdown 但尚未全部断开）
+        self._modules_exiting: set[str] = set()
+
         # Debounce timers for disconnected modules (module_id -> asyncio.Task)
         self._debounce_tasks: dict[str, asyncio.Task] = {}
         # Launcher loss timer (35s after launcher offline)
         self._launcher_loss_task: asyncio.Task | None = None
 
+        # Ping/Pong tracking
+        self._ping_sent_times: dict[str, float] = {}  # module_id -> last ping sent time (t1)
+        self._pong_latencies: dict[str, dict] = {}  # module_id -> {"outbound": ms, "inbound": ms, "last_update": timestamp}
+        self._pong_status: dict[str, str] = {}  # module_id -> "ok" | "timeout" | "never"
+        self._ping_timeout_counts: dict[str, int] = {}  # module_id -> consecutive timeout count
+        self._ping_task: asyncio.Task | None = None  # Global ping broadcast task
+        self._first_ping_sent = False  # 标记是否已发送第一次 ping
+        self._kernel_ping: float | None = None  # kernel 自己的 ping 值（所有模块的最小值）
+
+        # ── Instance scaler (auto-scaling) ──
+        self._scaler_task: asyncio.Task | None = None
+        # 全局扩缩容冻结开关（Launcher 通过 kernel.set_scaling RPC 控制）
+        # 冻结时：_scaler_tick 不执行任何扩缩容操作
+        self._scaling_frozen: bool = False
+        # module_id → {max_instances, instance_elastic} from module.ready
+        self._module_instance_config: dict[str, dict] = {}
+        # module_id → timestamp when all instances first went overload
+        self._scale_overload_since: dict[str, float] = {}
+        # module_id → timestamp when instances first went idle (with count > 1)
+        self._scale_idle_since: dict[str, float] = {}
+        # module_id → no scaling actions until this time
+        self._scale_cooldown_until: dict[str, float] = {}
+        # module_id → number of instances pending startup (requested but not yet ready)
+        self._scale_pending: dict[str, int] = {}
+        # 防止 _scaler_tick 并发执行（多个 apply_module_config 可能同时触发）
+        self._scaler_running: bool = False
+        # rpc_id → asyncio.Future for Kernel→Launcher RPC responses
+        self._rpc_futures: dict[str, asyncio.Future] = {}
+
+        # 动态 ping 间隔控制
+        self._ping_interval = 60.0  # 当前 ping 间隔（秒），默认 60 秒
+        self._last_latencies_call = 0.0  # 最后一次调用 kernel.latencies 的时间
+        self._ping_interval_event = asyncio.Event()  # 用于通知间隔变化
+
+        # Debug API (仅在 KITE_DEBUG=1 时启用)
+        self.debug_app: FastAPI | None = None
+        self.debug_port: int = 0
+        self._debug_server = None
+        if os.environ.get("KITE_DEBUG") == "1":
+            self.debug_app = self._create_debug_app()
+
         # Build FastAPI app
         self.app = self._create_app()
 
@@ -92,67 +157,190 @@ class KernelServer:
         @app.on_event("startup")
         async def _startup():
             server.event_hub.start_internal_senders()
-            server._ttl_task = asyncio.create_task(server._ttl_loop())
             server._dedup_task = asyncio.create_task(server._dedup_loop())
+            server._ping_task = asyncio.create_task(server._ping_broadcast_loop())
+            server._scaler_task = asyncio.create_task(server._instance_scaler_loop())
 
         @app.on_event("shutdown")
         async def _shutdown():
-            if server._ttl_task:
-                server._ttl_task.cancel()
             if server._dedup_task:
                 server._dedup_task.cancel()
+            if server._ping_task:
+                server._ping_task.cancel()
+            if server._scaler_task:
+                server._scaler_task.cancel()
 
         # ── WebSocket endpoint ──
 
         @app.websocket("/ws")
         async def ws_endpoint(ws: WebSocket):
-            token = ws.query_params.get("token", "")
             mid_hint = ws.query_params.get("id", "")
 
-            # Token verification (all modules including Launcher need token)
-            module_id = server.registry.verify_token(token)
-            if module_id is None:
-                # Must accept before close (Starlette requirement)
-                await ws.accept()
-                print(f"[kernel] Auth failed: token={token[:8]}... hint={mid_hint}")
+            # Check TLS requirement
+            if server.require_tls:
+                # Check if connection is secure (WSS)
+                # WebSocket.url.scheme will be "wss" for secure connections
+                if ws.url.scheme != "wss":
+                    await ws.close(code=1008, reason="TLS required")
+                    print(f"[kernel] Rejected non-TLS connection from {ws.client.host if ws.client else 'unknown'}")
+                    return
+
+            # Accept connection first (auth via first message)
+            await ws.accept()
+
+            # 关闭中拒绝新连接
+            if server._shutting_down:
+                await ws.close(code=1001, reason="Server shutting down")
+                return
+
+            # 速率限制检查（本地连接豁免，基于 module_id hint，无 hint 时用客户端 IP）
+            client_host = ws.client.host if ws.client else "unknown"
+            rate_key = mid_hint or client_host
+            is_local = client_host in ("127.0.0.1", "::1", "localhost")
+            if not is_local and not server.registry.check_auth_rate_limit(rate_key):
+                print(f"[kernel] Auth rate limited: key={rate_key}")
+                await ws.send_text(_dumps({
+                    "jsonrpc": "2.0", "id": None,
+                    "error": {"code": -32003, "message": "Rate limit exceeded"}
+                }))
+                await ws.close(code=4020, reason="Rate limit exceeded")
+                return
+
+            # Wait for auth message (5s timeout)
+            module_id = None
+            try:
+                raw = await asyncio.wait_for(ws.receive_text(), timeout=5.0)
+                auth_msg = _loads(raw)
+
+                # Validate auth request format
+                if (auth_msg.get("jsonrpc") != "2.0" or
+                    auth_msg.get("method") != "auth" or
+                    "params" not in auth_msg or
+                    "token" not in auth_msg["params"]):
+                    await ws.send_text(_dumps({
+                        "jsonrpc": "2.0",
+                        "id": auth_msg.get("id"),
+                        "error": {"code": -32600, "message": "Invalid auth request"}
+                    }))
+                    await ws.close(code=4001, reason="Invalid auth request")
+                    return
+
+                # Verify token
+                token = auth_msg["params"]["token"]
+                module_id = server.registry.verify_token(token)
+                slot = 0  # 默认 slot 0（主连接）
+                is_slot_auth = False
+
+                if module_id is None:
+                    # 尝试 slot token 认证
+                    slot_result = server.registry.verify_slot_token(token)
+                    if slot_result:
+                        module_id, slot = slot_result
+                        is_slot_auth = True
+                    else:
+                        print(f"[kernel] Auth failed: token={token[:8]}... hint={mid_hint}")
+                        await ws.send_text(_dumps({
+                            "jsonrpc": "2.0",
+                            "id": auth_msg.get("id"),
+                            "error": {"code": -32003, "message": "Authentication failed"}
+                        }))
+                        await ws.close(code=4001, reason="Authentication failed")
+                        return
+
+                # Use id hint for debug mode
+                if module_id == "debug" and mid_hint:
+                    module_id = mid_hint
+
+                # inst_key is encoded in the token: instance 1 → "acp_channel", instance 2 → "acp_channel#2"
+                # parse_instance_key extracts (base_module_id, instance_num) so module_id stays clean
+                module_id, instance_num = parse_instance_key(module_id)
+                inst_key = instance_key(module_id, instance_num)
+
+                # For slot token auth, verify_slot_token returns the full inst_key (e.g. "acp_channel#2")
+                # parse_instance_key above already reconstructs inst_key correctly — no override needed
+
+                # Register connection in both EventHub and shared connections table
+                server.event_hub.add_connection(inst_key, ws, slot=slot)
+                if inst_key not in server.connections:
+                    server.connections[inst_key] = {}
+                server.connections[inst_key][slot] = ws
+
+                # Get session info for auth response
+                session_id = server.event_hub._session_ids.get(inst_key, "")
+                current_seq = server.event_hub._session_seq.get(inst_key, 0)
+
+                # Send auth success response with session info
+                auth_result = {
+                    "ok": True,
+                    "session_id": session_id,
+                    "seq": current_seq
+                }
+                if is_slot_auth:
+                    auth_result["slot"] = slot
+                await ws.send_text(_dumps({
+                    "jsonrpc": "2.0",
+                    "id": auth_msg.get("id"),
+                    "result": auth_result
+                }))
+
+            except asyncio.TimeoutError:
+                print(f"[kernel] Auth timeout: hint={mid_hint}")
                 try:
-                    await ws.close(code=4001, reason="Authentication failed")
+                    await ws.close(code=4002, reason="Auth timeout")
+                except Exception:
+                    pass
+                return
+            except Exception as e:
+                print(f"[kernel] Auth error: {e}")
+                try:
+                    await ws.close(code=4001, reason="Auth error")
                 except Exception:
                     pass
                 return
 
-            # Use id hint for debug mode
-            if module_id == "debug" and mid_hint:
-                module_id = mid_hint
-
-            await ws.accept()
-
-            # Cancel debounce timer if module is reconnecting within 5s window
-            old_debounce = server._debounce_tasks.pop(module_id, None)
-            if old_debounce:
-                old_debounce.cancel()
-                print(f"[kernel] {module_id} reconnected within debounce window")
-
-            # Register connection in both EventHub and shared connections table
-            server.event_hub.add_connection(module_id, ws)
-            server.connections[module_id] = ws
-
-            # Set connected status in registry (if module exists)
-            server.registry.set_connected(module_id)
-
-            # Track Launcher connection
-            if module_id == "launcher":
-                server._launcher_connected = True
-                # Cancel launcher loss timer if reconnecting
-                if server._launcher_loss_task:
-                    server._launcher_loss_task.cancel()
-                    server._launcher_loss_task = None
-                    print(f"[kernel] launcher reconnected, cancelled loss timer")
-                print(f"[kernel] launcher connected")
-
-            # Renew heartbeat on connect
-            if module_id in server.registry.modules:
-                server.registry.heartbeat(module_id)
+            # 主连接（slot 0）：取消 debounce、更新状态、检查是否需要 require_init
+            if not is_slot_auth:
+                # Cancel debounce timer if instance is reconnecting within 5s window
+                old_debounce = server._debounce_tasks.pop(inst_key, None)
+                if old_debounce:
+                    old_debounce.cancel()
+                    print(f"[kernel] {inst_key} reconnected within debounce window")
+
+                # Set connected status in registry (module-level)
+                server.registry.set_connected(module_id)
+
+                # Track Launcher connection
+                if module_id == "launcher":
+                    server._launcher_connected = True
+                    if server._launcher_loss_task:
+                        server._launcher_loss_task.cancel()
+                        server._launcher_loss_task = None
+                        print(f"[kernel] launcher reconnected, cancelled loss timer")
+
+                # 检查该实例是否需要注册/订阅（system.require_init）
+                if server.registry.needs_init(inst_key):
+                    if server._init_gate_open or module_id == "launcher":
+                        try:
+                            await ws.send_text(_dumps({
+                                "jsonrpc": "2.0",
+                                "method": "event",
+                                "params": {
+                                    "event": "system.require_init",
+                                    "source": server.module_id,
+                                    "data": {"module_id": module_id, "instance_num": instance_num},
+                                },
+                            }))
+                            print(f"[kernel] Sent system.require_init to {inst_key}")
+                        except Exception as e:
+                            print(f"[kernel] Failed to send system.require_init to {inst_key}: {e}")
+                    else:
+                        # 门控关闭（恢复模式），暂存待 Launcher 就绪后统一发送
+                        server._init_pending.append((inst_key, ws))
+                        print(f"[kernel] Deferred system.require_init for {inst_key} (gate closed)")
+
+            # Initialize ping status for new connection (per instance_key)
+            if inst_key not in server._pong_status:
+                server._pong_status[inst_key] = "never"
 
             try:
                 while True:
@@ -173,6 +361,20 @@ class KernelServer:
                     if not isinstance(msg, dict):
                         continue
 
+                    # JSON-RPC 2.0 协议版本校验
+                    if msg.get("jsonrpc") != "2.0":
+                        msg_id = msg.get("id")
+                        if msg_id is not None:
+                            try:
+                                await ws.send_text(json.dumps({
+                                    "jsonrpc": "2.0", "id": msg_id,
+                                    "error": {"code": -32600,
+                                              "message": 'Invalid Request: missing or wrong "jsonrpc" field, must be "2.0"'}
+                                }))
+                            except Exception:
+                                pass
+                        continue
+
                     # Classify message type:
                     has_method = "method" in msg
                     has_id = "id" in msg
@@ -180,11 +382,22 @@ class KernelServer:
                     has_error = "error" in msg
 
                     if has_method and has_id:
-                        # RPC Request → dispatch to handler
-                        await server.rpc_router.dispatch(module_id, ws, msg)
+                        # RPC Request → dispatch to handler (use inst_key as caller)
+                        await server.rpc_router.dispatch(inst_key, ws, msg)
                     elif has_id and (has_result or has_error):
-                        # RPC Response → match to pending forward
-                        await server.rpc_router.handle_response(module_id, msg)
+                        # Check if this is a response to Kernel's own RPC call
+                        msg_id = msg.get("id", "")
+                        if isinstance(msg_id, str) and msg_id.startswith("kernel-") and msg_id in server._rpc_futures:
+                            result = msg.get("result")
+                            error = msg.get("error")
+                            if error:
+                                err_msg = error.get("message", str(error)) if isinstance(error, dict) else str(error)
+                                server._resolve_rpc_future(msg_id, error=err_msg)
+                            else:
+                                server._resolve_rpc_future(msg_id, result=result)
+                        else:
+                            # RPC Response → match to pending forward
+                            await server.rpc_router.handle_response(inst_key, msg)
                     # else: notification or unknown — ignore
 
             except WebSocketDisconnect:
@@ -192,21 +405,39 @@ class KernelServer:
             except Exception as e:
                 err = str(e).lower()
                 if "not connected" not in err and "closed" not in err:
-                    print(f"[kernel] WebSocket error for {module_id}: {e}")
+                    print(f"[kernel] WebSocket error for {inst_key}: {e}")
             finally:
-                # Cleanup connection but DON'T immediately set offline — debounce
-                server.event_hub.remove_connection(module_id)
-                server.connections.pop(module_id, None)
-
-                # Cancel existing debounce for this module (if reconnecting fast)
-                old_task = server._debounce_tasks.pop(module_id, None)
-                if old_task:
-                    old_task.cancel()
-
-                # Start 5s debounce timer
-                server._debounce_tasks[module_id] = asyncio.create_task(
-                    server._debounce_offline(module_id)
-                )
+                # 清理该 slot 的连接（使用 inst_key）
+                server.event_hub.remove_connection(inst_key, slot=slot)
+                slots_dict = server.connections.get(inst_key, {})
+                slots_dict.pop(slot, None)
+                if not slots_dict:
+                    server.connections.pop(inst_key, None)
+
+                # 检查是否所有 slot 都已断开
+                remaining = server.connections.get(inst_key, {})
+                if not remaining:
+                    # 所有 slot 断开 → 清理退出标记、启动 debounce
+                    server._modules_exiting.discard(inst_key)
+                    old_task = server._debounce_tasks.pop(inst_key, None)
+                    if old_task:
+                        old_task.cancel()
+                    server._debounce_tasks[inst_key] = asyncio.create_task(
+                        server._debounce_offline(inst_key)
+                    )
+                else:
+                    # 部分 slot 断开 → 不触发 debounce
+                    # 冻结或模块正在退出则跳过 recovery offer
+                    if server._scaling_frozen or inst_key in server._modules_exiting:
+                        pass
+                    # 非弹性模式：发 recovery offer 补充断掉的 slot
+                    elif not server._module_conn_elastic.get(inst_key, False):
+                        max_conns = server._get_max_connections(inst_key)
+                        if max_conns > 1 and len(remaining) < max_conns:
+                            missing = [s for s in range(1, max_conns)
+                                       if s not in remaining]
+                            if missing:
+                                server._send_connection_offer(inst_key, missing)
 
         # ── HTTP endpoints (debug only) ──
 
@@ -241,22 +472,295 @@ class KernelServer:
 
         return app
 
+    def _create_debug_app(self) -> FastAPI:
+        """创建 Debug API（仅在 KITE_DEBUG=1 时使用）"""
+        app = FastAPI(title="Kite Kernel Debug API", docs_url=None, redoc_url=None)
+        server = self
+
+        @app.get("/health")
+        async def debug_health():
+            """健康检查 — 返回 Kernel 是否 ready"""
+            return {
+                "status": "ready" if server._ready_published else "starting",
+                "module_count": len(server.registry.modules),
+                "online_count": sum(
+                    1 for m in server.registry.modules.values()
+                    if m.get("status") in ("registered", "ready")
+                ),
+            }
+
+        @app.get("/status")
+        async def debug_status():
+            """详细状态 — 返回所有模块状态和事件统计"""
+            import time as _time
+            return {
+                "kernel": {
+                    "ready": server._ready_published,
+                    "port": server.port,
+                    "advertise_ip": server.advertise_ip,
+                    "boot_time": server.event_hub._start_time,
+                    "uptime_seconds": round(_time.time() - server.event_hub._start_time),
+                },
+                "registry": {
+                    "modules": {
+                        mid: {
+                            "status": data.get("status"),
+                            "module_type": data.get("module_type"),
+                            "registered_at": data.get("registered_at"),
+                            "max_connections": data.get("max_connections"),
+                            "port": data.get("port"),
+                        }
+                        for mid, data in server.registry.modules.items()
+                    },
+                },
+                "event_hub": server.event_hub.get_stats(),
+            }
+
+        @app.get("/registry")
+        async def debug_registry():
+            """注册中心详情 — 返回所有模块的完整注册信息"""
+            return {
+                "modules": {
+                    mid: {
+                        "status": data.get("status"),
+                        "module_type": data.get("module_type"),
+                        "registered_at": data.get("registered_at"),
+                        "max_connections": data.get("max_connections"),
+                        "port": data.get("port"),
+                        "base_url": data.get("base_url"),
+                        "tools": list(data.get("tools", {}).keys()),
+                        "events_subscribe": data.get("events_subscribe", []),
+                        "events_publish": list(data.get("events_publish", {}).keys()),
+                        "launcher_id": data.get("launcher_id"),
+                    }
+                    for mid, data in server.registry.modules.items()
+                },
+                "module_count": len(server.registry.modules),
+                "last_update_time": server.registry.last_update_time,
+            }
+
+        @app.get("/ws-url")
+        async def debug_ws_url():
+            """返回 Kernel WebSocket 连接地址"""
+            return {
+                "ws_url": f"ws://{server.advertise_ip}:{server.port}/ws",
+                "port": server.port,
+            }
+
+        @app.post("/shutdown")
+        async def debug_shutdown():
+            """触发 Kernel 优雅关闭"""
+            if server._shutting_down:
+                return {"status": "already_shutting_down"}
+
+            server._shutting_down = True
+            # 发布 module.shutdown 事件通知所有模块
+            server.event_hub.publish_internal(
+                "module.shutdown",
+                {"module_id": "kernel", "reason": "debug_api_shutdown"},
+                source="kernel"
+            )
+
+            # 延迟关闭 uvicorn（给事件传递留时间）
+            async def _delayed_shutdown():
+                await asyncio.sleep(1)
+                if server._uvicorn_server:
+                    server._uvicorn_server.should_exit = True
+                if server._debug_server:
+                    server._debug_server.should_exit = True
+
+            asyncio.create_task(_delayed_shutdown())
+            return {"status": "shutting_down"}
+
+        return app
+
     # ── Background loops ──
 
-    async def _ttl_loop(self):
-        """Check heartbeat TTL every 10s and publish offline events."""
+    async def _send_first_ping(self):
+        """发送第一次 ping（在 system.ready 后立即触发）"""
+        import time
+        if self._first_ping_sent:
+            return
+        self._first_ping_sent = True
+
+        t1 = time.time()
+        connected_modules = [mid for mid, slots in self.connections.items() if slots]
+        print(f"[kernel] _send_first_ping: {len(connected_modules)} modules connected")
+
+        if connected_modules:
+            for module_id in connected_modules:
+                self._ping_sent_times[module_id] = t1
+                self._pong_status[module_id] = "never"
+
+            # Broadcast system.ping event
+            self.event_hub.publish_internal(
+                "system.ping",
+                {"ping_time": t1},
+                source=self.module_id
+            )
+
+    async def _ping_broadcast_loop(self):
+        """Broadcast system.ping event with dynamic interval (5s when active, 60s when idle)."""
+        import time
+        import uuid
+
+        # 等待第一次 ping 发送（由 system.ready 触发）
+        while not self._first_ping_sent:
+            await asyncio.sleep(0.5)
+
+        # 第一次 ping 后等待 5 秒检查响应
+        await asyncio.sleep(5)
+        await self._check_ping_timeouts()
+
+        # 初始间隔：60 秒（空闲模式）
+        remaining_wait = 55.0  # 60 - 5 = 55 秒
+
         while True:
-            await asyncio.sleep(10)
             try:
-                expired = self.registry.check_ttl()
-                for mid in expired:
+                # 动态调整间隔
+                current_time = time.time()
+                time_since_last_call = current_time - self._last_latencies_call
+
+                # 如果最近 30 秒内有调用 kernel.latencies，使用 5 秒间隔
+                if time_since_last_call < 30:
+                    target_interval = 5.0
+                else:
+                    target_interval = 60.0
+
+                # 如果间隔发生变化，调整等待时间
+                if target_interval != self._ping_interval:
+                    old_interval = self._ping_interval
+                    self._ping_interval = target_interval
+                    print(f"[kernel] Ping interval changed: {old_interval:.0f}s → {target_interval:.0f}s")
+
+                # 等待剩余时间（可被中断）
+                try:
+                    await asyncio.wait_for(
+                        self._ping_interval_event.wait(),
+                        timeout=remaining_wait
+                    )
+                    # 如果被唤醒（interval 变化），重新计算等待时间
+                    self._ping_interval_event.clear()
+                    remaining_wait = self._ping_interval - 5  # 减去检查超时的 5 秒
+                    continue
+                except asyncio.TimeoutError:
+                    # 正常超时，发送 ping
+                    pass
+
+                # 发送 ping
+                t1 = time.time()
+                connected_modules = [mid for mid, slots in self.connections.items() if slots]
+
+                if connected_modules:
+                    for module_id in connected_modules:
+                        self._ping_sent_times[module_id] = t1
+
+                    # Broadcast system.ping event
                     self.event_hub.publish_internal(
-                        "module.offline", {"module_id": mid}, source=self.module_id)
+                        "system.ping",
+                        {"ping_time": t1},
+                        source=self.module_id
+                    )
+
+                # Check for timeouts after 5s
+                await asyncio.sleep(5)
+                await self._check_ping_timeouts()
+
+                # 计算下一次等待时间
+                remaining_wait = self._ping_interval - 5
+
+            except asyncio.CancelledError:
+                break
             except Exception as e:
-                print(f"[kernel] TTL loop error: {e}")
+                print(f"[kernel] Ping broadcast loop error: {e}")
+
+    async def _check_ping_timeouts(self):
+        """检查 ping 超时并计算汇总统计，连续 3 次超时则强断连接"""
+        import time
+        t_check = time.time()
+        modules_to_disconnect = []
+
+        for module_id in list(self.connections.keys()):
+            if module_id in self._ping_sent_times:
+                # If no pong received within 5s, mark as timeout
+                if module_id not in self._pong_latencies or \
+                   self._pong_latencies[module_id].get("last_update", 0) < self._ping_sent_times[module_id]:
+                    if self._pong_status.get(module_id) == "never":
+                        # Keep as "never" if never received
+                        pass
+                    else:
+                        self._pong_status[module_id] = "timeout"
+                        # 增加连续超时计数
+                        self._ping_timeout_counts[module_id] = self._ping_timeout_counts.get(module_id, 0) + 1
+
+                        # 连续 3 次超时 → 强断连接
+                        if self._ping_timeout_counts[module_id] >= 3:
+                            modules_to_disconnect.append(module_id)
+                else:
+                    # 收到 pong，重置超时计数
+                    self._ping_timeout_counts[module_id] = 0
+
+        # 强断超时连接（关闭所有 slot）
+        for module_id in modules_to_disconnect:
+            slots = self.connections.get(module_id, {})
+            if slots:
+                print(f"[kernel] {module_id} 连续 3 次 Ping 超时，强制断开连接 ({len(slots)} slots)")
+                for ws in list(slots.values()):
+                    try:
+                        await ws.close(code=4012, reason="Ping timeout")
+                    except Exception:
+                        pass
+
+        # 计算汇总统计
+        total_modules = len(self.connections)
+        ok_pings = []
+        timeout_modules = []
+        all_inbounds = []
+        all_outbounds = []
+
+        for module_id in self.connections.keys():
+            status = self._pong_status.get(module_id, "never")
+            if status == "ok":
+                latency = self._pong_latencies.get(module_id, {})
+                outbound = latency.get("outbound")
+                inbound = latency.get("inbound")
+                if outbound is not None:
+                    ok_pings.append((module_id, outbound))
+                    all_outbounds.append(outbound)
+                if inbound is not None:
+                    all_inbounds.append(inbound)
+            elif status == "timeout":
+                timeout_modules.append(module_id)
+
+        # 计算 kernel 自己的 ping 数据并存储
+        if ok_pings:
+            min_module, min_ping = min(ok_pings, key=lambda x: x[1])
+            max_module, max_ping = max(ok_pings, key=lambda x: x[1])
+
+            # kernel 的延迟数据
+            kernel_outbound = min(all_inbounds) if all_inbounds else None
+            kernel_inbound = min(all_outbounds) if all_outbounds else None
+
+            if kernel_outbound is not None and kernel_inbound is not None:
+                self._pong_latencies["kernel"] = {
+                    "outbound": kernel_outbound,
+                    "inbound": kernel_inbound,
+                    "last_update": list(self._ping_sent_times.values())[0] if self._ping_sent_times else t_check
+                }
+                self._pong_status["kernel"] = "ok"
+
+            # 打印汇总信息
+            timeout_info = f", 超时: {timeout_modules}" if timeout_modules else ""
+            print(f"[kernel] Ping 汇总: 发送 {total_modules} 个模块, 最小 {min_ping:.2f}ms ({min_module}), 最大 {max_ping:.2f}ms ({max_module}){timeout_info}")
+        else:
+            self._pong_latencies.pop("kernel", None)
+            self._pong_status.pop("kernel", None)
+            if total_modules > 0:
+                print(f"\033[91m[kernel] 警告: 所有模块 Ping 超时 ({total_modules} 个模块)\033[0m")
+
 
     async def _dedup_loop(self):
-        """Clean up dedup table every 30s."""
+        """Clean up dedup table and throttle cache every 30s."""
         while True:
             await asyncio.sleep(30)
             try:
@@ -265,23 +769,168 @@ class KernelServer:
             except Exception as e:
                 print(f"[kernel] Dedup cleanup error: {e}")
 
+            # 清理过期 slot token
+            try:
+                self.registry.cleanup_expired_slot_tokens()
+            except Exception as e:
+                print(f"[kernel] Slot token cleanup error: {e}")
+
+            # 清理节流缓存：移除 10s 前的条目，防止内存泄漏
+            try:
+                import time
+                cutoff = time.time() - 10
+                stale_keys = [k for k, t in self.event_hub._throttle_last.items() if t < cutoff]
+                for k in stale_keys:
+                    del self.event_hub._throttle_last[k]
+            except Exception as e:
+                print(f"[kernel] Throttle cache cleanup error: {e}")
+
+    # ── 多连接：offer / release ──
+
+    def _get_max_connections(self, module_id: str) -> int:
+        """从 registry 读取模块的 max_connections 配置。module_id 可以是 instance_key。"""
+        base_mid = parse_instance_key(module_id)[0]
+        mod = self.registry.modules.get(base_mid, {})
+        return mod.get("max_connections", 1)
+
+    def _send_connection_offer(self, module_id: str, slots: list[int]):
+        """生成 slot token 并通过事件通知模块建立附加连接。"""
+        if self._shutting_down or module_id in self._modules_exiting:
+            return
+        tokens = self.registry.generate_slot_tokens(module_id, slots)
+        offer_data = {
+            "module_id": module_id,
+            "slots": {
+                str(s): {"token": t, "expires_in": 30}
+                for s, t in tokens.items()
+            }
+        }
+        self.event_hub.publish_internal(
+            "system.connection.offer", offer_data,
+            source=self.module_id
+        )
+        print(f"[kernel] Sent connection offer to {module_id}: slots={slots}")
+
+    def _send_connection_release(self, module_id: str, slots: list[int], reason: str = "idle_shrink"):
+        """请求模块释放指定 slot，超时强制关闭。"""
+        self.event_hub.publish_internal(
+            "system.connection.release",
+            {"module_id": module_id, "slots": slots, "reason": reason, "grace_period": 2},
+            source=self.module_id
+        )
+
+        # 启动 2 秒可取消定时器，超时强制关闭
+        async def _force_close():
+            try:
+                await asyncio.sleep(2)
+                for s in slots:
+                    ws = self.connections.get(module_id, {}).get(s)
+                    if ws:
+                        try:
+                            await ws.close(code=4014, reason=f"Release timeout: {reason}")
+                        except Exception:
+                            pass
+            except asyncio.CancelledError:
+                pass
+
+        timer_key = f"{module_id}:{','.join(map(str, slots))}"
+        old = self._release_timers.pop(timer_key, None)
+        if old:
+            old.cancel()
+        self._release_timers[timer_key] = asyncio.create_task(_force_close())
+        print(f"[kernel] Sent connection release to {module_id}: slots={slots} reason={reason}")
+
+    def _initiate_multi_connection(self, module_id: str, max_conns: int):
+        """启动多连接建立流程。"""
+        if max_conns <= 1:
+            return
+
+        if self._scaling_frozen:
+            return
+
+        if module_id in self._modules_exiting:
+            return
+
+        current_slots = len(self.connections.get(module_id, {}))
+        if current_slots >= max_conns:
+            return
+
+        is_elastic = self._module_conn_elastic.get(module_id, False)
+        missing_slots = [s for s in range(1, max_conns) if s not in self.connections.get(module_id, {})]
+        if not missing_slots:
+            return
+
+        if not is_elastic:
+            # 非弹性：立即发 offer 建满
+            self._send_connection_offer(module_id, missing_slots)
+        else:
+            # 弹性：先建 1 个附加连接，后续根据压力动态调整
+            self._send_connection_offer(module_id, missing_slots[:1])
+            # 启动弹性收缩监控
+            if module_id not in self._shrink_tasks:
+                self._shrink_tasks[module_id] = asyncio.create_task(
+                    self._elastic_shrink_loop(module_id, max_conns)
+                )
+
+    async def _elastic_shrink_loop(self, module_id: str, max_conns: int):
+        """弹性收缩循环：idle 30s → release 1 个最空闲 slot，最少保留 2 个。"""
+        try:
+            while True:
+                await asyncio.sleep(30)
+                slots = self.connections.get(module_id, {})
+                if len(slots) <= 2:
+                    continue  # 最少保留 2 个
+
+                # 检查队列压力
+                level = self.event_hub._queue_levels.get(module_id, "idle")
+                if level != "idle":
+                    continue  # 非空闲不收缩
+
+                # 找最空闲的 slot（避开保序绑定的 slot 和 slot 0）
+                bound_slots = set()
+                for group in self.event_hub._ordering_groups.get(module_id, []):
+                    if group.bound_slot is not None:
+                        bound_slots.add(group.bound_slot)
+
+                candidates = [s for s in slots if s != 0 and s not in bound_slots]
+                if not candidates:
+                    continue
+
+                # Release 最后一个（编号最大的）
+                to_release = max(candidates)
+                self._send_connection_release(module_id, [to_release])
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._shrink_tasks.pop(module_id, None)
+
     # ── Debounce & Launcher loss ──
 
     async def _debounce_offline(self, module_id: str):
-        """Wait 5s after WS disconnect. If module doesn't reconnect, mark offline."""
+        """Wait 5s after WS disconnect. If instance doesn't reconnect, mark offline.
+        module_id here is actually instance_key (e.g. 'assistant' or 'assistant#2').
+        """
         try:
             await asyncio.sleep(5)
         except asyncio.CancelledError:
-            return  # Module reconnected within 5s — cancelled by ws_endpoint
+            return  # Instance reconnected within 5s — cancelled by ws_endpoint
 
-        # 5s elapsed, module did not reconnect — mark offline
+        # 5s elapsed, instance did not reconnect
         self._debounce_tasks.pop(module_id, None)
-        self.registry.set_offline(module_id)
-        self.event_hub.publish_internal("module.offline", {"module_id": module_id}, source=self.module_id)
-        print(f"[kernel] {module_id} offline (5s debounce expired)")
+        base_mid = parse_instance_key(module_id)[0]
+
+        # Check if other instances of this module are still connected
+        if self.event_hub.has_other_instances(base_mid, module_id):
+            # Other instances alive — don't deregister module, just log
+            print(f"[kernel] {module_id} offline (other instances still active)")
+        else:
+            # Last instance → deregister module and publish offline event
+            self.registry.deregister_module(base_mid)
+            self.event_hub.publish_internal("module.offline", {"module_id": base_mid}, source=self.module_id)
+            print(f"[kernel] {module_id} offline → deregistered (5s debounce expired)")
 
         # If launcher went offline, start 35s launcher loss timer
-        if module_id == "launcher":
+        if base_mid == "launcher":
             self._launcher_connected = False
             if not self._launcher_loss_task:
                 self._launcher_loss_task = asyncio.create_task(
@@ -309,6 +958,251 @@ class KernelServer:
         # Shutdown Kernel itself
         await self.shutdown()
 
+    # ── Instance auto-scaler ──
+
+    # Thresholds (aligned with design doc 4.6)
+    _SCALE_CHECK_INTERVAL = 5     # seconds between scaler checks
+    _SCALE_UP_HOLD = 30           # all instances overload for this long → scale up
+    _SCALE_DOWN_HOLD = 30         # instances idle for this long → scale down
+    _SCALE_UP_COOLDOWN = 30       # seconds after scale-up before another scale-up
+    _SCALE_DOWN_COOLDOWN = 15     # seconds after scale-down before another scale-down
+
+    async def _instance_scaler_loop(self):
+        """Periodically check instance pressure and auto-scale elastic modules."""
+        try:
+            while True:
+                await asyncio.sleep(self._SCALE_CHECK_INTERVAL)
+                if self._shutting_down:
+                    return
+                try:
+                    await self._scaler_tick()
+                except Exception as e:
+                    print(f"[kernel] Scaler error: {e}")
+        except asyncio.CancelledError:
+            pass
+
+    async def _scaler_tick(self):
+        """One iteration of the scaler loop."""
+        if self._scaling_frozen or self._scaler_running:
+            return
+        self._scaler_running = True
+        try:
+            await self._scaler_tick_inner()
+        finally:
+            self._scaler_running = False
+
+    async def _scaler_tick_inner(self):
+        """Actual scaler logic, guarded by _scaler_running flag."""
+        now = time.time()
+
+        for module_id, config in list(self._module_instance_config.items()):
+            max_inst = config.get("max_instances", 1)
+
+            # 远程模块（通过 Relay 连入）不受 Launcher 生命周期管理，跳过实例扩缩容
+            # 多连接扩缩（_multi_conn_tick）不受影响
+            if self.registry.is_remote_module(module_id):
+                continue
+
+            inst_elastic = config.get("instance_elastic", False)
+
+            # Get all active instances and their pressure
+            inst_keys = self.event_hub.get_instance_keys(module_id)
+            # Exclude draining instances from active count
+            active_keys = [k for k in inst_keys if not self.event_hub.is_draining(k)]
+            current_count = len(active_keys) + self._scale_pending.get(module_id, 0)
+
+            # ── Fixed scale: 一次性扩/缩到目标数量 ──
+            if not inst_elastic:
+                if current_count < max_inst:
+                    while current_count < max_inst:
+                        await self._scale_up(module_id, current_count)
+                        current_count += 1
+                    # Fixed scale 一次性扩完，不需要 cooldown
+                    self._scale_cooldown_until.pop(module_id, None)
+                elif current_count > max_inst:
+                    # 缩容：一次性 drain 所有超出的实例
+                    excess = current_count - max_inst
+                    for _ in range(excess):
+                        await self._scale_down(module_id, active_keys)
+                        # 更新 active_keys（排除刚标记为 draining 的）
+                        active_keys = [k for k in inst_keys if not self.event_hub.is_draining(k)]
+                continue
+
+            # Cooldown check（仅弹性模式需要）
+            if now < self._scale_cooldown_until.get(module_id, 0):
+                continue
+
+            # ── Elastic scale ──
+            pressures = {}
+            for k in active_keys:
+                p = self.event_hub.get_instance_pressure(k)
+                if p:
+                    pressures[k] = p
+
+            # ── Scale UP check ──
+            if current_count < max_inst:
+                all_overload = (
+                    len(pressures) > 0 and
+                    all(p.get("level") in ("overload", "critical") for p in pressures.values())
+                )
+                if all_overload:
+                    since = self._scale_overload_since.get(module_id)
+                    if since is None:
+                        self._scale_overload_since[module_id] = now
+                    elif now - since >= self._SCALE_UP_HOLD:
+                        await self._scale_up(module_id, current_count)
+                        self._scale_overload_since.pop(module_id, None)
+                else:
+                    self._scale_overload_since.pop(module_id, None)
+
+            # ── Scale DOWN check ──
+            if len(active_keys) > 1:
+                all_idle = (
+                    len(pressures) > 0 and
+                    all(p.get("level") == "idle" for p in pressures.values())
+                )
+                if all_idle:
+                    since = self._scale_idle_since.get(module_id)
+                    if since is None:
+                        self._scale_idle_since[module_id] = now
+                    elif now - since >= self._SCALE_DOWN_HOLD:
+                        await self._scale_down(module_id, active_keys)
+                        self._scale_idle_since.pop(module_id, None)
+                else:
+                    self._scale_idle_since.pop(module_id, None)
+
+    async def _scale_up(self, module_id: str, current_count: int):
+        """Request Launcher to start a new instance."""
+        new_num = current_count + 1
+        print(f"[kernel] Auto-scaling UP: {module_id} → instance #{new_num}")
+
+        self._scale_pending[module_id] = self._scale_pending.get(module_id, 0) + 1
+        self._scale_cooldown_until[module_id] = time.time() + self._SCALE_UP_COOLDOWN
+
+        try:
+            result = await self._call_launcher_rpc("start_module_instance", {
+                "name": module_id,
+                "instance_num": new_num,
+            })
+            if result:
+                self.event_hub.publish_internal("system.instance.started", {
+                    "module_id": module_id,
+                    "instance_num": new_num,
+                    "reason": "auto_scale",
+                }, source=self.module_id)
+                print(f"[kernel] Scale-up requested: {module_id}#{new_num}")
+        except Exception as e:
+            print(f"[kernel] Scale-up failed for {module_id}: {e}")
+        finally:
+            pending = self._scale_pending.get(module_id, 1)
+            self._scale_pending[module_id] = max(0, pending - 1)
+
+    async def _scale_down(self, module_id: str, inst_keys: list[str]):
+        """Mark one instance as draining; stop it when its queue empties."""
+        # 候选：排除 primary（#1）和已在 draining 的实例
+        candidates = []
+        for k in inst_keys:
+            _, num = parse_instance_key(k)
+            if num <= 1:
+                continue  # Never scale down the primary instance
+            if self.event_hub.is_draining(k):
+                continue  # Already draining
+            pressure = self.event_hub.get_instance_pressure(k) or {}
+            depth = pressure.get("depth", 0)
+            candidates.append((k, depth))
+
+        if not candidates:
+            return
+
+        # 优先选队列最浅的实例（最接近空闲）
+        candidates.sort(key=lambda x: x[1])
+        target_key, _ = candidates[0]
+        _, target_num = parse_instance_key(target_key)
+
+        print(f"[kernel] Scale-down: draining {target_key}")
+        self.event_hub.set_draining(target_key)
+        self._scale_cooldown_until[module_id] = time.time() + self._SCALE_DOWN_COOLDOWN
+
+        # 异步等待队列清空后关闭
+        asyncio.ensure_future(self._drain_and_stop(module_id, target_key, target_num))
+
+    async def _drain_and_stop(self, module_id: str, inst_key: str, instance_num: int):
+        """Wait for instance queue to empty, then stop it via Launcher."""
+        try:
+            # 首先等待至少收到一次压力报告，避免刚启动未上报就被误判为空
+            for _ in range(6):  # 最多等 6 * SCALE_CHECK_INTERVAL
+                await asyncio.sleep(self._SCALE_CHECK_INTERVAL)
+                if self._scaling_frozen:
+                    self.event_hub.clear_draining(inst_key)
+                    return
+                if self.event_hub.get_instance_pressure(inst_key) is not None:
+                    break
+
+            while True:
+                await asyncio.sleep(self._SCALE_CHECK_INTERVAL)
+                if self._scaling_frozen:
+                    # 系统冻结（如关机），直接放弃 drain，由关闭流程处理
+                    self.event_hub.clear_draining(inst_key)
+                    return
+                pressure = self.event_hub.get_instance_pressure(inst_key) or {}
+                depth = pressure.get("depth", 0)
+                if depth == 0:
+                    break
+            # 队列已空，通知 Launcher 停止该实例
+            print(f"[kernel] Scale-down: stopping {inst_key} (queue empty)")
+            await self._call_launcher_rpc("stop_module_instance", {
+                "name": module_id,
+                "instance_num": instance_num,
+            })
+            self.event_hub.publish_internal("system.instance.stopped", {
+                "module_id": module_id,
+                "instance_num": instance_num,
+                "reason": "auto_scale",
+            }, source=self.module_id)
+            print(f"[kernel] Scale-down completed: {inst_key}")
+        except Exception as e:
+            print(f"[kernel] Scale-down failed for {inst_key}: {e}")
+        finally:
+            self.event_hub.clear_draining(inst_key)
+
+    async def _call_launcher_rpc(self, method: str, params: dict, timeout: float = 10) -> dict | None:
+        """Call a Launcher RPC method and wait for response."""
+        import uuid
+        launcher_slots = self.connections.get("launcher", {})
+        if not launcher_slots:
+            raise RuntimeError("Launcher not connected")
+        ws = next(iter(launcher_slots.values()))
+
+        rpc_id = f"kernel-{uuid.uuid4().hex[:8]}"
+        await ws.send_text(json.dumps({
+            "jsonrpc": "2.0",
+            "id": rpc_id,
+            "method": method,
+            "params": params,
+        }))
+
+        # Wait for response via rpc_router pending mechanism
+        # Use a simpler approach: the response will come back through handle_response
+        # which routes back to Kernel. We use an asyncio.Event to wait.
+        future = asyncio.get_event_loop().create_future()
+        self._rpc_futures[rpc_id] = future
+
+        try:
+            result = await asyncio.wait_for(future, timeout=timeout)
+            return result
+        except asyncio.TimeoutError:
+            self._rpc_futures.pop(rpc_id, None)
+            raise RuntimeError(f"Launcher RPC timeout: {method}")
+
+    def _resolve_rpc_future(self, rpc_id: str, result: dict = None, error: str = None):
+        """Resolve a pending Kernel→Launcher RPC future."""
+        future = self._rpc_futures.pop(rpc_id, None)
+        if future and not future.done():
+            if error:
+                future.set_exception(RuntimeError(error))
+            else:
+                future.set_result(result)
+
     # ── Self-registration ──
 
     def self_register(self):
@@ -316,8 +1210,8 @@ class KernelServer:
         self.registry.register_module({
             "module_id": self.module_id,
             "module_type": "infrastructure",
-            "api_endpoint": f"http://{self.advertise_ip}:{self.port}",
-            "health_endpoint": "/health",
+            "base_url": f"http://{self.advertise_ip}:{self.port}",
+            "health_path": "/health",
             "metadata": {
                 "ws_endpoint": f"ws://{self.advertise_ip}:{self.port}/ws",
             },
@@ -325,11 +1219,18 @@ class KernelServer:
 
     def publish_ready(self):
         """Publish module.ready event for Kernel (internal, no WS needed)."""
+        import time
+        startup_time = time.monotonic() - self.boot_t0 if self.boot_t0 else 0
+        reason = "recovery" if self.is_recover else "startup"
         self.event_hub.publish_internal("module.ready", {
             "module_id": self.module_id,
             "ws_endpoint": f"ws://{self.advertise_ip}:{self.port}/ws",
             "graceful_shutdown": True,
+            "queue_elastic": True,
+            "startup_time": startup_time,
+            "reason": reason,
         }, source=self.module_id)
+        self._ready_published = True
 
     async def _handle_internal_event(self, event_type: str, data: dict):
         """内部事件处理器（通过 EventHub 回调机制调用）
@@ -347,10 +1248,21 @@ class KernelServer:
             # 处理 Kernel 订阅的事件
             if event_type == "module.shutdown":
                 await self.handle_shutdown_event(data)
+            elif event_type == "system.pong":
+                await self._handle_pong_event(data)
+            elif event_type == "system.ready":
+                # 系统启动完成，立即发送第一次 ping
+                print(f"[kernel] Received system.ready event, triggering first ping")
+                # 检查 recovered 标记：崩溃恢复时重新启动 Ping/Pong
+                if data.get("recovered"):
+                    self._first_ping_sent = False
+                    print(f"[kernel] system.ready (recovered=true), resetting ping state")
+                await self._send_first_ping()
             # 忽略系统广播事件（Kernel 没有订阅但会收到广播）
             elif event_type in ("module.ready", "module.registered", "module.started",
                                 "module.stopped", "module.crashed", "module.exiting",
-                                "module.offline", "system.ready", "registry.updated"):
+                                "module.offline", "module.degraded", "module.recovered",
+                                "registry.updated"):
                 # 这些是系统事件，会广播给所有模块，Kernel 收到是正常的
                 pass
             else:
@@ -363,6 +1275,44 @@ class KernelServer:
             traceback.print_exc()
             raise  # 重新抛出，让 EventHub 的包装器记录
 
+    async def _handle_pong_event(self, data: dict):
+        """处理 system.pong 事件，计算往返延迟
+
+        Args:
+            data: {
+                "module_id": str,
+                "ping_time": float,  # t1 - Kernel 发送 ping 的时间
+                "pong_time": float,  # t2 - 模块收到 ping 并发送 pong 的时间
+            }
+        """
+        import time
+        module_id = data.get("module_id")
+        t1 = data.get("ping_time")  # Kernel 发送 ping 的时间
+        t2 = data.get("pong_time")  # 模块收到 ping 并发送 pong 的时间
+        t3 = time.time()  # Kernel 收到 pong 的时间
+
+        if not module_id or t1 is None or t2 is None:
+            return
+
+        # 验证 ping_time 是否匹配
+        if module_id not in self._ping_sent_times:
+            return
+
+        expected_t1 = self._ping_sent_times[module_id]
+        if abs(t1 - expected_t1) > 0.1:  # 允许 100ms 误差
+            return
+
+        # 计算两个延迟
+        outbound_ms = (t2 - t1) * 1000  # 去程：Kernel → 模块
+        inbound_ms = (t3 - t2) * 1000   # 回程：模块 → Kernel
+
+        self._pong_latencies[module_id] = {
+            "outbound": round(outbound_ms, 2),
+            "inbound": round(inbound_ms, 2),
+            "last_update": t3,
+        }
+        self._pong_status[module_id] = "ok"
+
     async def handle_shutdown_event(self, event_data):
         """处理 module.shutdown 事件（标准优雅退出流程）
 
@@ -378,10 +1328,12 @@ class KernelServer:
             print(f"[kernel] Warning: Invalid shutdown event data type: {type(event_data)}")
             return
 
-        # 防御性检查：验证 module_id
+        # 判断 shutdown 目标
         target_module = event_data.get("module_id")
         if target_module != self.module_id:
-            print(f"[kernel] Warning: Received shutdown for wrong module: {target_module}")
+            # 其他模块的 shutdown 通知：标记该模块正在退出，不再发 offer
+            if target_module:
+                self._modules_exiting.add(target_module)
             return
 
         # 防御性检查：防止重复处理
@@ -422,13 +1374,14 @@ class KernelServer:
             # Step 5: 等待一小段时间确保事件发送完成
             await asyncio.sleep(0.2)
 
-            # Step 6: 关闭所有 WebSocket 连接（包括 Launcher）
+            # Step 6: 关闭所有 WebSocket 连接（包括 Launcher，所有 slot）
             close_tasks = []
-            for mid, ws in list(self.connections.items()):
-                try:
-                    close_tasks.append(ws.close(code=1000, reason="Graceful shutdown complete"))
-                except Exception as e:
-                    print(f"[kernel] 关闭 {mid} 连接失败: {e}")
+            for mid, slots in list(self.connections.items()):
+                for s, ws in list(slots.items()):
+                    try:
+                        close_tasks.append(ws.close(code=1000, reason="Graceful shutdown complete"))
+                    except Exception as e:
+                        print(f"[kernel] 关闭 {mid} slot {s} 连接失败: {e}")
 
             if close_tasks:
                 await asyncio.gather(*close_tasks, return_exceptions=True)
@@ -460,6 +1413,14 @@ class KernelServer:
         self._shutting_down = True
         print("[kernel] 执行清理工作...")
 
+        # 取消所有队列压力倒计时
+        if self.event_hub._critical_timers:
+            print(f"[kernel] 取消 {len(self.event_hub._critical_timers)} 个压力倒计时")
+            for task in self.event_hub._critical_timers.values():
+                if not task.done():
+                    task.cancel()
+            self.event_hub._critical_timers.clear()
+
         # 取消所有 debounce 任务
         if self._debounce_tasks:
             print(f"[kernel] 取消 {len(self._debounce_tasks)} 个 debounce 任务")
@@ -474,15 +1435,28 @@ class KernelServer:
             self._launcher_loss_task.cancel()
             self._launcher_loss_task = None
 
+        # 取消多连接相关任务
+        for name, task_dict in [("offer", self._offer_tasks),
+                                ("shrink", self._shrink_tasks),
+                                ("release", self._release_timers)]:
+            if task_dict:
+                print(f"[kernel] 取消 {len(task_dict)} 个 {name} 任务")
+                for task in task_dict.values():
+                    if not task.done():
+                        task.cancel()
+                task_dict.clear()
+
         # 关闭其他模块的 WebSocket 连接（保留 Launcher 连接）
-        other_connections = {mid: ws for mid, ws in self.connections.items() if mid != "launcher"}
-        if other_connections:
-            print(f"[kernel] 关闭 {len(other_connections)} 个其他模块的 WebSocket 连接")
-            for module_id, ws in other_connections.items():
-                try:
-                    await ws.close(code=1001, reason="Server shutting down")
-                except Exception as e:
-                    print(f"[kernel] 关闭连接失败 {module_id}: {e}")
+        other_mids = [mid for mid in self.connections if mid != "launcher"]
+        if other_mids:
+            count = sum(len(self.connections.get(mid, {})) for mid in other_mids)
+            print(f"[kernel] 关闭 {count} 个其他模块的 WebSocket 连接")
+            for mid in other_mids:
+                for slot, ws in list(self.connections.get(mid, {}).items()):
+                    try:
+                        await ws.close(code=1001, reason="Server shutting down")
+                    except Exception as e:
+                        print(f"[kernel] 关闭连接失败 {mid} slot {slot}: {e}")
 
         # 清空 RPC 转发队列
         pending_count = len(self.rpc_router._pending)
@@ -517,13 +1491,15 @@ class KernelServer:
             self._launcher_loss_task.cancel()
             self._launcher_loss_task = None
 
-        # Close all WebSocket connections
-        print(f"[kernel] Closing {len(self.connections)} WebSocket connections...")
-        for module_id, ws in list(self.connections.items()):
-            try:
-                await ws.close(code=1001, reason="Server shutting down")
-            except Exception as e:
-                print(f"[kernel] Failed to close connection for {module_id}: {e}")
+        # Close all WebSocket connections (multi-slot)
+        total_slots = sum(len(s) for s in self.connections.values())
+        print(f"[kernel] Closing {total_slots} WebSocket connections...")
+        for module_id, slots in list(self.connections.items()):
+            for slot, ws in list(slots.items()):
+                try:
+                    await ws.close(code=1001, reason="Server shutting down")
+                except Exception as e:
+                    print(f"[kernel] Failed to close connection for {module_id} slot {slot}: {e}")
         self.connections.clear()
 
         # Clear pending RPC forwards