@agentunion/kite 1.3.2 → 1.5.0

package/extensions/services/web/web_config.json5

@@ -0,0 +1,29 @@
+{
+  // Web 服务配置
+  server: {
+    host: "0.0.0.0",
+    port: 18766,
+    ssl: false,
+    // SSL 证书路径（如果启用 SSL）
+    ssl_cert: null,
+    ssl_key: null
+  },
+
+  // 静态文件配置
+  static: {
+    directory: "static",
+    cache_max_age: 3600  // 秒
+  },
+
+  // CORS 配置
+  cors: {
+    enabled: false,
+    origins: ["*"]
+  },
+
+  // 日志配置
+  logging: {
+    level: "INFO",
+    format: "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+  }
+}

package/kernel/entry.py

@@ -26,7 +26,113 @@ import uvicorn
 
 
 # ── Module configuration ──
-MODULE_NAME = "kernel"
+
+def _load_module_config() -> dict:
+    """Load module configuration from module.md frontmatter.
+
+    Returns:
+        Dict with keys: name, preferred_port, advertise_ip
+
+    Raises:
+        SystemExit: If module.md is invalid or name is non-compliant
+    """
+    _this_dir = os.path.dirname(os.path.abspath(__file__))
+    module_md = os.path.join(_this_dir, "module.md")
+
+    # Calculate relative path for error messages
+    project_root = os.environ.get("KITE_PROJECT", "")
+    if project_root and _this_dir.startswith(project_root):
+        rel_path = os.path.relpath(_this_dir, project_root)
+    else:
+        rel_path = _this_dir
+
+    # Default values (will be overridden if valid config exists)
+    result = {
+        "name": "",
+        "preferred_port": 0,
+        "advertise_ip": "127.0.0.1"
+    }
+
+    # Check if module.md exists
+    if not os.path.exists(module_md):
+        print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+        print(f"  Path: {rel_path}/module.md")
+        print(f"  Reason: File not found")
+        sys.exit(1)
+
+    try:
+        with open(module_md, encoding="utf-8") as f:
+            text = f.read()
+
+        # Extract YAML frontmatter (between --- markers)
+        import re
+        m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
+        if not m:
+            print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+            print(f"  Path: {rel_path}/module.md")
+            print(f"  Reason: Missing YAML frontmatter")
+            sys.exit(1)
+
+        # Parse YAML frontmatter
+        try:
+            import yaml
+            fm = yaml.safe_load(m.group(1)) or {}
+        except ImportError:
+            print(f"[{rel_path}] ERROR: PyYAML not installed, cannot parse module.md")
+            sys.exit(1)
+        except Exception as e:
+            print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+            print(f"  Path: {rel_path}/module.md")
+            print(f"  Reason: YAML parse error: {e}")
+            sys.exit(1)
+
+        # Validate 'name' field (required)
+        if "name" not in fm:
+            print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+            print(f"  Path: {rel_path}/module.md")
+            print(f"  Reason: Missing 'name' field")
+            sys.exit(1)
+
+        raw_name = str(fm["name"]).strip()
+
+        if not raw_name:
+            print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+            print(f"  Path: {rel_path}/module.md")
+            print(f"  Reason: Empty module name")
+            sys.exit(1)
+
+        # Validate name characters
+        sanitized = re.sub(r'[^a-zA-Z0-9_\-]', '', raw_name)
+
+        if sanitized != raw_name:
+            invalid_chars = ''.join(sorted(set(c for c in raw_name if c not in sanitized)))
+            print(f"[{rel_path}] ERROR: Invalid module configuration in module.md")
+            print(f"  Path: {rel_path}/module.md")
+            print(f"  Reason: Invalid characters in name '{raw_name}': {repr(invalid_chars)}")
+            sys.exit(1)
+
+        result["name"] = sanitized
+
+        # Extract optional fields
+        if "preferred_port" in fm:
+            try:
+                result["preferred_port"] = int(fm["preferred_port"])
+            except (ValueError, TypeError):
+                pass
+
+        if "advertise_ip" in fm:
+            result["advertise_ip"] = str(fm["advertise_ip"])
+
+    except SystemExit:
+        raise  # Re-raise exit to prevent catching by outer except
+    except Exception as e:
+        print(f"[{rel_path}] ERROR: Failed to read module.md: {e}")
+        sys.exit(1)
+
+    return result
+
+_module_config = _load_module_config()
+MODULE_NAME = _module_config["name"]
 
 
 def _fmt_elapsed(t0: float) -> str:
@@ -256,27 +362,6 @@ if _project_root not in sys.path:
 from kernel.server import KernelServer
 
 
-def _read_module_md() -> dict:
-    """Read preferred_port and advertise_ip from own module.md."""
-    md_path = os.path.join(_this_dir, "module.md")
-    result = {"preferred_port": 0, "advertise_ip": "127.0.0.1"}
-    try:
-        with open(md_path, "r", encoding="utf-8") as f:
-            text = f.read()
-        m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
-        if m:
-            try:
-                import yaml
-                fm = yaml.safe_load(m.group(1)) or {}
-            except ImportError:
-                fm = {}
-            result["preferred_port"] = int(fm.get("preferred_port", 0))
-            result["advertise_ip"] = fm.get("advertise_ip", "127.0.0.1")
-    except Exception:
-        pass
-    return result
-
-
 def _bind_port(preferred: int, host: str) -> int:
     """Try preferred port first, fall back to OS-assigned."""
     if preferred:
@@ -284,11 +369,15 @@ def _bind_port(preferred: int, host: str) -> int:
             with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                 s.bind((host, preferred))
                 return preferred
-        except OSError:
-            pass
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind((host, 0))
-        return s.getsockname()[1]
+        except OSError as e:
+            print(f"\033[31m[kernel] ✖ 指定端口 {preferred} 绑定失败: {e}，将使用系统随机端口\033[0m")
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind((host, 0))
+            return s.getsockname()[1]
+    except OSError as e:
+        print(f"\033[31m[kernel] ✖ 端口绑定完全失败 ({host}): {e}\033[0m")
+        raise
 
 
 def _read_stdin_kite_messages(expected: set[str], timeout: float = 10) -> dict[str, dict]:
@@ -362,17 +451,16 @@ def main():
     if is_debug:
         print("[kernel] 调试模式已启用 (KITE_DEBUG=1)，接受所有令牌")
 
-    # Step 1: Read config from own module.md
-    md_config = _read_module_md()
-    advertise_ip = md_config["advertise_ip"]
-    preferred_port = md_config["preferred_port"]
+    # Step 1: Use cached module config (already loaded at module level)
+    advertise_ip = _module_config["advertise_ip"]
+    preferred_port = _module_config["preferred_port"]
 
     # Step 2: Generate launcher token
     import secrets
     launcher_token = secrets.token_urlsafe(32)
 
     # Step 3: Create KernelServer with launcher_token
-    server = KernelServer(launcher_token=launcher_token, advertise_ip=advertise_ip)
+    server = KernelServer(launcher_token=launcher_token, advertise_ip=advertise_ip, module_id=MODULE_NAME, boot_t0=_t0)
 
     # Step 4: Bind port
     bind_host = advertise_ip

package/kernel/event_hub.py

@@ -43,6 +43,9 @@ QUEUE_MAXSIZE = 10000
 # System events that are auto-broadcast to ALL connected modules (no subscription needed)
 SYSTEM_EVENTS = {"module.offline", "module.ready", "module.shutdown"}
 
+# Callback execution pool size (max concurrent callback tasks)
+CALLBACK_POOL_SIZE = 100
+
 
 class EventHub:
 
@@ -66,6 +69,13 @@ class EventHub:
         self._cnt_errors = 0
         self._start_time = time.time()
 
+        # Internal event callbacks (module_id -> callback)
+        # 用于没有 WebSocket 连接的模块（如 Kernel）通过回调接收事件
+        self._internal_callbacks: dict[str, callable] = {}
+
+        # Queue overflow tracking (per module)
+        self._queue_overflow: dict[str, int] = {}  # module_id -> overflow count
+
     # ── Connection lifecycle ──
 
     def add_connection(self, module_id: str, ws: WebSocket):
@@ -119,13 +129,80 @@ class EventHub:
                         await ws.send_text(raw)
                     self._cnt_routed += 1
                 except Exception:
-                    print(f"[kernel] Send failed to {mid}, closing sender")
+                    # Connection closed (normal during shutdown) — exit silently
                     break
         except asyncio.CancelledError:
             pass
 
+    async def _callback_sender_loop(self, mid: str, callback: callable, queue: asyncio.Queue):
+        """回调专用的发送循环（类似 _sender_loop，但调用回调而不是发送 WS）
+
+        从队列取出 JSON-RPC 2.0 Notification，解析后调用回调函数。
+        """
+        try:
+            while True:
+                raw = await queue.get()
+                try:
+                    msg = _loads(raw)
+                    params = msg.get("params", {})
+                    event_type = params.get("event", "")
+                    data = params.get("data", {})
+                    await callback(event_type, data)
+                    self._cnt_routed += 1
+                except Exception as e:
+                    print(f"[kernel] Internal callback error for {mid} on {event_type}: {e}")
+                    import traceback
+                    traceback.print_exc()
+        except asyncio.CancelledError:
+            pass
+
     # ── Subscribe / Unsubscribe ──
 
+    def register_internal_callback(self, module_id: str, callback: callable):
+        """注册内部事件回调（用于 Kernel 自身接收事件）
+
+        Args:
+            module_id: 模块 ID（通常是 "kernel"）
+            callback: 异步回调函数 async def callback(event_type: str, data: dict)
+
+        Raises:
+            TypeError: 如果 callback 不是 callable
+
+        Note:
+            只注册回调和创建队列，不创建 task。
+            需要在事件循环启动后调用 start_internal_senders() 来创建 sender tasks。
+        """
+        if not callable(callback):
+            raise TypeError(f"callback must be callable, got {type(callback)}")
+
+        # 存储回调函数引用
+        self._internal_callbacks[module_id] = callback
+
+        # 创建队列（如果还没有）
+        if module_id not in self._queues:
+            q = asyncio.Queue(maxsize=QUEUE_MAXSIZE)
+            self._queues[module_id] = q
+
+        print(f"[kernel] Registered internal callback for {module_id}")
+
+    def start_internal_senders(self):
+        """启动所有已注册的内部回调的 sender tasks（必须在事件循环中调用）"""
+        for module_id, callback in self._internal_callbacks.items():
+            if module_id not in self._senders:
+                q = self._queues[module_id]
+                self._senders[module_id] = asyncio.create_task(
+                    self._callback_sender_loop(module_id, callback, q)
+                )
+                print(f"[kernel] Started internal sender for {module_id}")
+
+    def unregister_internal_callback(self, module_id: str):
+        """注销内部事件回调"""
+        self._internal_callbacks.pop(module_id, None)
+        self._queues.pop(module_id, None)
+        task = self._senders.pop(module_id, None)
+        if task:
+            task.cancel()
+
     def handle_subscribe(self, module_id: str, events: list[str]):
         if module_id not in self.subscriptions:
             self.subscriptions[module_id] = set()
@@ -134,28 +211,32 @@ class EventHub:
         )
         print(f"[kernel] {module_id} subscribed: {events}")
 
-    def handle_unsubscribe(self, module_id: str, events: list[str]):
+    def handle_unsubscribe(self, module_id: str, events: list[str]) -> dict:
+        """Unsubscribe from events. Returns dict with unsubscribed events and count."""
         subs = self.subscriptions.get(module_id)
+        unsubscribed = []
         if subs:
             to_remove = {item for item in subs if item[0] in events}
+            unsubscribed = [item[0] for item in to_remove]
             subs.difference_update(to_remove)
         print(f"[kernel] {module_id} unsubscribed: {events}")
+        return {"unsubscribed": unsubscribed, "count": len(unsubscribed)}
 
     # ── Event publishing (called by RpcRouter) ──
 
     def publish_event(self, sender_id: str, event_id: str, event_type: str,
                       data: dict = None, echo: bool = False) -> dict:
         """Publish an event from a module. Called by RpcRouter for event.publish RPC.
-        Returns {ok: True} on success, or error dict if validation fails."""
+        Returns empty dict on success, raises exception on validation failure."""
         self._cnt_received += 1
 
         if not event_id or not event_type:
             self._cnt_errors += 1
-            return {"ok": False, "error": "Missing required field: event_id or event"}
+            raise ValueError("Missing required field: event_id or event")
 
         if self.dedup.is_duplicate(event_id):
             self._cnt_dedup += 1
-            return {"ok": True}  # silently accept duplicates
+            return {}  # silently accept duplicates
 
         msg = {
             "jsonrpc": "2.0",
@@ -170,11 +251,17 @@ class EventHub:
         }
 
         self._route_event(sender_id, msg, event_type, echo)
-        return {"ok": True}
+        return {}
 
-    def publish_internal(self, event_type: str, data: dict):
+    def publish_internal(self, event_type: str, data: dict, source: str):
         """Publish a Kernel-originated event (e.g. module.offline, module.registered).
-        No dedup check — internal events are unique by nature."""
+        No dedup check — internal events are unique by nature.
+
+        Args:
+            event_type: 事件类型（如 "module.ready"）
+            data: 事件数据
+            source: 事件来源模块 ID（必填，通常是调用者的 module_id）
+        """
         import uuid
         event_id = str(uuid.uuid4())
         self._cnt_received += 1
@@ -185,25 +272,53 @@ class EventHub:
             "params": {
                 "event_id": event_id,
                 "event": event_type,
-                "source": "kernel",
+                "source": source,
                 "timestamp": datetime.now(timezone.utc).isoformat(),
                 "data": data,
             },
         }
 
-        self._route_event("kernel", msg, event_type, echo=False)
+        self._route_event(source, msg, event_type, echo=False)
 
     # ── Routing ──
 
     def _route_event(self, sender_id: str, msg: dict, event_type: str, echo: bool):
         """Enqueue event to all matching subscribers' delivery queues.
         System events (module.offline, module.ready, module.shutdown) are auto-broadcast
-        to ALL connected modules, regardless of subscription."""
+        to ALL connected modules, regardless of subscription.
+
+        Exception: module.shutdown with a target module_id is sent ONLY to that module.
+        """
         e_parts = tuple(event_type.split("."))
         raw = None  # lazy serialization
+        params = msg.get("params", {})
+        data = params.get("data", {})
 
-        # System events → broadcast to all connected modules
+        # System events → broadcast to all connected modules (unless targeted)
         if event_type in SYSTEM_EVENTS:
+            # Check if this is a targeted shutdown (has module_id in data)
+            if event_type == "module.shutdown":
+                target_module = data.get("module_id", "")
+
+                # Targeted shutdown → send only to target module
+                if target_module:
+                    queue = self._queues.get(target_module)
+                    if queue:
+                        if raw is None:
+                            raw = _dumps(msg)
+                        try:
+                            queue.put_nowait(raw)
+                            self._cnt_queued += 1
+                        except asyncio.QueueFull:
+                            # Queue 满了，记录溢出
+                            self._queue_overflow[target_module] = self._queue_overflow.get(target_module, 0) + 1
+                            overflow_count = self._queue_overflow[target_module]
+                            # 每 100 次溢出打印一次警告
+                            if overflow_count % 100 == 1:
+                                print(f"[kernel] Warning: Queue full for {target_module}, dropped {overflow_count} events")
+                    return
+
+            # Broadcast system events (no target or non-shutdown events)
             for mid, queue in self._queues.items():
                 if mid == sender_id and not echo:
                     continue
@@ -211,9 +326,14 @@ class EventHub:
                     raw = _dumps(msg)
                 try:
                     queue.put_nowait(raw)
+                    self._cnt_queued += 1
                 except asyncio.QueueFull:
-                    pass
-                self._cnt_queued += 1
+                    # Queue 满了，记录溢出
+                    self._queue_overflow[mid] = self._queue_overflow.get(mid, 0) + 1
+                    overflow_count = self._queue_overflow[mid]
+                    if overflow_count % 100 == 1:
+                        print(f"[kernel] Warning: Queue full for {mid}, dropped {overflow_count} events")
+
             return
 
         # Normal events → subscription-based routing
@@ -228,9 +348,14 @@ class EventHub:
                             raw = _dumps(msg)
                         try:
                             queue.put_nowait(raw)
+                            self._cnt_queued += 1
                         except asyncio.QueueFull:
-                            pass
-                        self._cnt_queued += 1
+                            # Queue 满了，记录溢出
+                            self._queue_overflow[mid] = self._queue_overflow.get(mid, 0) + 1
+                            overflow_count = self._queue_overflow[mid]
+                            # 每 100 次溢出打印一次警告
+                            if overflow_count % 100 == 1:
+                                print(f"[kernel] Warning: Queue full for {mid}, dropped {overflow_count} events")
                     break
 
     # ── Stats ──

package/kernel/module.md

@@ -1,33 +1,60 @@
----
-name: kernel
-display_name: Kernel
-type: infrastructure
-state: enabled
-runtime: python
-entry: entry.py
----
-
-# Kernel
-
-Unified infrastructure module that merges Registry (service discovery) and Event Hub (event routing) into a single process with a WebSocket JSON-RPC 2.0 interface.
-
-## Responsibilities
-
-- **Service Registry**: Module registration, heartbeat TTL, glob lookup, dot-path queries
-- **Event Routing**: NATS-style wildcard subscriptions, per-subscriber queues, 1h dedup
-- **RPC Routing**: JSON-RPC 2.0 dispatch for builtin methods + cross-module forwarding
-- **Token Verification**: In-memory token→module_id resolution (no cross-process HTTP)
-
-## Protocol
-
-Single WebSocket endpoint: `ws://127.0.0.1:{port}/ws?token={TOKEN}&id={MODULE_ID}`
-
-Three frame types on the wire:
-- **RPC Request**: `{jsonrpc:"2.0", id, method, params}` — client→Kernel or Kernel→client (forwarded)
-- **RPC Response**: `{jsonrpc:"2.0", id, result}` or `{jsonrpc:"2.0", id, error}` — response to request
-- **Event Notification**: `{jsonrpc:"2.0", method:"event", params:{event_id, event, source, timestamp, data}}` — no id, no response
-
-## HTTP Endpoints (debug only)
-
-- `GET /health` — combined registry + event hub health
-- `GET /stats` — connections, subscriptions, counters
+---
+name: kernel
+display_name: Kite内核
+type: infrastructure
+state: enabled
+runtime: python
+entry: entry.py
+version: 1.0.1
+advertise_ip: 127.0.0.1
+monitor: true
+preferred_port: 0
+display_order: 100
+---
+# Kernel
+
+Unified infrastructure module that merges Registry (service discovery) and Event Hub (event routing) into a single process with a WebSocket JSON-RPC 2.0 interface.
+
+## Responsibilities
+
+- **Service Registry**: Module registration, glob lookup, dot-path queries
+- **Event Routing**: NATS-style wildcard subscriptions, per-subscriber queues, 1h dedup
+- **RPC Routing**: JSON-RPC 2.0 dispatch for builtin methods + cross-module forwarding
+- **Token Verification**: In-memory token→module_id resolution (no cross-process HTTP)
+- **Ping/Pong Monitoring**: Active ping to all connected modules, track response latencies
+
+## Protocol
+
+Single WebSocket endpoint: `ws://127.0.0.1:{port}/ws?token={TOKEN}&id={MODULE_ID}`
+
+Three frame types on the wire:
+- **RPC Request**: `{jsonrpc:"2.0", id, method, params}` — client→Kernel or Kernel→client (forwarded)
+- **RPC Response**: `{jsonrpc:"2.0", id, result}` or `{jsonrpc:"2.0", id, error}` — response to request
+- **Event Notification**: `{jsonrpc:"2.0", method:"event", params:{event_id, event, source, timestamp, data}}` — no id, no response
+
+## HTTP Endpoints (debug only)
+
+- `GET /health` — combined registry + event hub health
+- `GET /stats` — connections, subscriptions, counters
+
+## RPC Methods
+
+### Registry Methods
+- `registry.register` — Register a module
+- `registry.deregister` — Deregister a module
+- `registry.lookup` — Lookup modules by field/value
+- `registry.get` — Get module info by dot-path
+- `registry.verify` — Verify a token
+
+### Event Methods
+- `event.publish` — Publish an event
+- `event.subscribe` — Subscribe to events
+- `event.unsubscribe` — Unsubscribe from events
+
+### Kernel Methods
+- `kernel.ping` — Ping (used for latency measurement)
+- `kernel.stats` — Get kernel statistics
+- `kernel.health` — Get kernel health status
+- `kernel.latencies` — Get ping/pong latencies for all modules
+- `kernel.generate_tokens` — Generate tokens for modules (Launcher only)
+- `kernel.register_tokens` — Register token mapping (Launcher only)