@agentunion/kite 1.0.0

package/core/event_hub/bench_results/2026-02-28_13-44-45.json

@@ -0,0 +1,51 @@
+{
+  "timestamp": "2026-02-28T13:44:45.723676",
+  "env": {
+    "platform": "win32",
+    "python": "3.13.12"
+  },
+  "throughput": {
+    "events": 10000,
+    "send_rate": 7576,
+    "client_recv": 10000,
+    "hub_queued": 10000,
+    "hub_routed": 10000,
+    "send_time": 1.32,
+    "recv_time": 1.43
+  },
+  "latency": {
+    "samples": 200,
+    "avg_ms": 0.5,
+    "p50_ms": 0.48,
+    "p95_ms": 0.66,
+    "p99_ms": 0.78
+  },
+  "fanout_1": {
+    "subs": 1,
+    "events": 2000,
+    "send_rate": 8811,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "fanout_10": {
+    "subs": 10,
+    "events": 2000,
+    "send_rate": 10057,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "fanout_50": {
+    "subs": 50,
+    "events": 2000,
+    "send_rate": 22422,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "hub_counters": {
+    "events_received": 16200,
+    "events_routed": 132200,
+    "events_queued": 132200,
+    "events_deduplicated": 0,
+    "errors": 0
+  }
+}

package/core/event_hub/bench_results/2026-02-28_13-45-39.json

@@ -0,0 +1,51 @@
+{
+  "timestamp": "2026-02-28T13:45:39.158400",
+  "env": {
+    "platform": "win32",
+    "python": "3.13.12"
+  },
+  "throughput": {
+    "events": 10000,
+    "send_rate": 7464,
+    "client_recv": 10000,
+    "hub_queued": 10000,
+    "hub_routed": 10000,
+    "send_time": 1.34,
+    "recv_time": 1.45
+  },
+  "latency": {
+    "samples": 200,
+    "avg_ms": 0.59,
+    "p50_ms": 0.53,
+    "p95_ms": 0.93,
+    "p99_ms": 1.68
+  },
+  "fanout_1": {
+    "subs": 1,
+    "events": 2000,
+    "send_rate": 8694,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "fanout_10": {
+    "subs": 10,
+    "events": 2000,
+    "send_rate": 7599,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "fanout_50": {
+    "subs": 50,
+    "events": 2000,
+    "send_rate": 6664,
+    "avg_recv": 2000,
+    "min_recv": 2000
+  },
+  "hub_counters": {
+    "events_received": 16200,
+    "events_routed": 132200,
+    "events_queued": 132200,
+    "events_deduplicated": 0,
+    "errors": 0
+  }
+}

package/core/event_hub/dedup.py

@@ -0,0 +1,31 @@
+"""
+Event deduplication — 1-hour sliding window.
+Tracks event_id to prevent duplicate forwarding from outbox replays.
+"""
+
+import time
+
+
+class EventDedup:
+
+    def __init__(self, ttl: float = 3600.0):
+        self.ttl = ttl  # seconds to keep event_id (default 1h)
+        self.seen: dict[str, float] = {}  # event_id -> first-seen timestamp
+
+    def is_duplicate(self, event_id: str) -> bool:
+        """Check if event_id was already seen. If new, record it."""
+        if event_id in self.seen:
+            return True
+        self.seen[event_id] = time.time()
+        return False
+
+    def cleanup(self):
+        """Remove entries older than TTL."""
+        cutoff = time.time() - self.ttl
+        expired = [eid for eid, ts in self.seen.items() if ts < cutoff]
+        for eid in expired:
+            del self.seen[eid]
+
+    @property
+    def size(self) -> int:
+        return len(self.seen)

package/core/event_hub/entry.py

@@ -0,0 +1,113 @@
+"""
+Event Hub entry point.
+Reads boot_info from stdin, starts FastAPI server, registers to Registry.
+"""
+
+import json
+import os
+import socket
+import sys
+
+import httpx
+import uvicorn
+
+# Ensure project root is on sys.path
+_this_dir = os.path.dirname(os.path.abspath(__file__))
+_project_root = os.path.dirname(os.path.dirname(_this_dir))
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from core.event_hub.hub import EventHub
+from core.event_hub.server import EventHubServer
+
+
+def _get_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def _register_to_registry(token: str, registry_url: str, port: int, advertise_ip: str = "127.0.0.1"):
+    """Synchronous registration to Registry at startup."""
+    payload = {
+        "action": "register",
+        "module_id": "event_hub",
+        "module_type": "infrastructure",
+        "name": "Event Hub",
+        "api_endpoint": f"http://{advertise_ip}:{port}",
+        "health_endpoint": "/health",
+        "metadata": {
+            "ws_endpoint": f"ws://{advertise_ip}:{port}/ws",
+        },
+    }
+    headers = {"Authorization": f"Bearer {token}"}
+    resp = httpx.post(
+        f"{registry_url}/modules",
+        json=payload,
+        headers=headers,
+        timeout=5,
+    )
+    if resp.status_code == 200:
+        print(f"[event_hub] Registered to Registry")
+    else:
+        print(f"[event_hub] WARNING: Registry returned {resp.status_code}")
+
+
+def _read_test_mode() -> bool:
+    """Read test_mode from module.md frontmatter."""
+    md_path = os.path.join(_this_dir, "module.md")
+    try:
+        with open(md_path, encoding="utf-8") as f:
+            text = f.read()
+        import re, yaml
+        m = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
+        if m:
+            fm = yaml.safe_load(m.group(1)) or {}
+            return bool(fm.get("test_mode", False))
+    except Exception:
+        pass
+    return False
+
+
+def main():
+    # Read boot_info from stdin
+    token = ""
+    registry_port = 0
+    bind_host = "127.0.0.1"
+    advertise_ip = "127.0.0.1"
+    try:
+        line = sys.stdin.readline().strip()
+        if line:
+            boot_info = json.loads(line)
+            token = boot_info.get("token", "")
+            registry_port = boot_info.get("registry_port", 0)
+            bind_host = boot_info.get("bind", "127.0.0.1")
+            advertise_ip = boot_info.get("advertise_ip", "127.0.0.1")
+    except Exception:
+        pass
+
+    if not token or not registry_port:
+        print("[event_hub] ERROR: Missing token or registry_port in boot_info")
+        sys.exit(1)
+
+    print(f"[event_hub] Token received ({len(token)} chars), registry port: {registry_port}")
+
+    registry_url = f"http://127.0.0.1:{registry_port}"
+    port = _get_free_port()
+
+    # Register to Registry
+    _register_to_registry(token, registry_url, port, advertise_ip)
+
+    # Create hub and server
+    test_mode = _read_test_mode()
+    if test_mode:
+        print("[event_hub] WARNING: test_mode enabled, all tokens accepted")
+    hub = EventHub()
+    server = EventHubServer(hub, own_token=token, registry_url=registry_url, test_mode=test_mode)
+
+    print(f"[event_hub] Starting on {bind_host}:{port}")
+    uvicorn.run(server.app, host=bind_host, port=port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

package/core/event_hub/hub.py

@@ -0,0 +1,263 @@
+"""
+Core Event Hub logic.
+WebSocket handling, event processing (validate → dedup → route → ack),
+connection and subscription management.
+"""
+
+import asyncio
+import json
+import time
+from datetime import datetime, timezone
+
+try:
+    import orjson
+except ImportError:
+    orjson = None
+
+from fastapi import WebSocket, WebSocketDisconnect
+
+from .dedup import EventDedup
+from .router import match_parts
+
+
+def _dumps(obj) -> str:
+    if orjson is not None:
+        return orjson.dumps(obj).decode()
+    return json.dumps(obj)
+
+
+def _loads(raw: str):
+    if orjson is not None:
+        return orjson.loads(raw)
+    return json.loads(raw)
+
+QUEUE_MAXSIZE = 10000
+
+
+class EventHub:
+
+    def __init__(self):
+        # Connection table: module_id -> WebSocket
+        self.connections: dict[str, WebSocket] = {}
+        # Connection metadata: module_id -> {connected_at}
+        self.connection_info: dict[str, dict] = {}
+        # Subscription table: module_id -> set of (pattern_str, pattern_tuple)
+        self.subscriptions: dict[str, set[tuple]] = {}
+        # Per-subscriber delivery queue + sender task
+        self._queues: dict[str, asyncio.Queue] = {}
+        self._senders: dict[str, asyncio.Task] = {}
+        # Dedup
+        self.dedup = EventDedup()
+        # Counters (integer attrs for speed)
+        self._cnt_received = 0
+        self._cnt_routed = 0
+        self._cnt_queued = 0
+        self._cnt_dedup = 0
+        self._cnt_errors = 0
+        self._start_time = time.time()
+
+    # ── Connection lifecycle ──
+
+    def add_connection(self, module_id: str, ws: WebSocket):
+        """Add a new connection. If module_id already connected, close the old one."""
+        old_ws = self.connections.get(module_id)
+        if old_ws is not None:
+            asyncio.ensure_future(self._close_old(module_id, old_ws))
+
+        self.connections[module_id] = ws
+        self.connection_info[module_id] = {
+            "connected_at": datetime.now(timezone.utc).isoformat(),
+        }
+        self.subscriptions.pop(module_id, None)
+        # Create delivery queue + sender task
+        q = asyncio.Queue(maxsize=QUEUE_MAXSIZE)
+        self._queues[module_id] = q
+        self._senders[module_id] = asyncio.create_task(
+            self._sender_loop(module_id, ws, q)
+        )
+        print(f"[event_hub] {module_id} connected")
+
+    async def _close_old(self, module_id: str, ws: WebSocket):
+        try:
+            await ws.close(code=4000, reason="replaced by new connection")
+        except Exception:
+            pass
+        print(f"[event_hub] Closed old connection for {module_id}")
+
+    def remove_connection(self, module_id: str):
+        """Clean up on disconnect."""
+        self.connections.pop(module_id, None)
+        self.connection_info.pop(module_id, None)
+        self.subscriptions.pop(module_id, None)
+        self._queues.pop(module_id, None)
+        task = self._senders.pop(module_id, None)
+        if task:
+            task.cancel()
+        print(f"[event_hub] {module_id} disconnected")
+
+    # ── Sender loop (per-subscriber) ──
+
+    async def _sender_loop(self, mid: str, ws: WebSocket, queue: asyncio.Queue):
+        """Drain queue and send to subscriber. Exits on send failure."""
+        try:
+            while True:
+                raw = await queue.get()
+                try:
+                    if len(raw) > 65536:
+                        await asyncio.wait_for(ws.send_text(raw), timeout=5 + len(raw) / 102400)
+                    else:
+                        await ws.send_text(raw)
+                    self._cnt_routed += 1
+                except Exception:
+                    print(f"[event_hub] Send failed to {mid}, closing sender")
+                    break
+        except asyncio.CancelledError:
+            pass
+
+    # ── Subscribe / Unsubscribe ──
+
+    def handle_subscribe(self, module_id: str, events: list[str]):
+        if module_id not in self.subscriptions:
+            self.subscriptions[module_id] = set()
+        self.subscriptions[module_id].update(
+            (p, tuple(p.split("."))) for p in events
+        )
+        print(f"[event_hub] {module_id} subscribed: {events}")
+
+    def handle_unsubscribe(self, module_id: str, events: list[str]):
+        subs = self.subscriptions.get(module_id)
+        if subs:
+            to_remove = {item for item in subs if item[0] in events}
+            subs.difference_update(to_remove)
+        print(f"[event_hub] {module_id} unsubscribed: {events}")
+
+    # ── Main message handler ──
+
+    async def handle_message(self, module_id: str, ws: WebSocket, raw: str):
+        try:
+            msg = _loads(raw)
+        except Exception:
+            await self._send_error(ws, "Invalid JSON")
+            return
+
+        msg_type = msg.get("type", "")
+
+        if msg_type == "subscribe":
+            events = msg.get("events", [])
+            if isinstance(events, list) and events:
+                self.handle_subscribe(module_id, events)
+            return
+
+        if msg_type == "unsubscribe":
+            events = msg.get("events", [])
+            if isinstance(events, list) and events:
+                self.handle_unsubscribe(module_id, events)
+            return
+
+        if msg_type == "event":
+            await self._handle_event(module_id, ws, msg)
+            return
+
+        await self._send_error(ws, f"Unknown message type: {msg_type}")
+
+    # ── Event processing ──
+
+    async def _handle_event(self, module_id: str, ws: WebSocket, msg: dict):
+        """Validate → dedup → auto-fill → route → ack."""
+        self._cnt_received += 1
+
+        event_id = msg.get("event_id")
+        event_type = msg.get("event")
+        if not event_id or not event_type:
+            self._cnt_errors += 1
+            await self._send_error(ws, "Missing required field: event_id or event")
+            return
+
+        if self.dedup.is_duplicate(event_id):
+            self._cnt_dedup += 1
+            await self._send_ack(ws, event_id)
+            return
+
+        if not msg.get("source"):
+            msg["source"] = module_id
+        if not msg.get("timestamp"):
+            msg["timestamp"] = datetime.now(timezone.utc).isoformat()
+
+        await self._route_event(module_id, msg)
+        await self._send_ack(ws, event_id)
+
+    # ── Routing ──
+
+    async def _route_event(self, sender_id: str, msg: dict):
+        """Enqueue event to all matching subscribers' delivery queues."""
+        event_type = msg["event"]
+        e_parts = tuple(event_type.split("."))
+        echo = msg.get("echo", False)
+        raw = None  # lazy serialization
+
+        for mid, patterns in self.subscriptions.items():
+            if mid == sender_id and not echo:
+                continue
+            for _pat_str, p_parts in patterns:
+                if match_parts(p_parts, e_parts):
+                    queue = self._queues.get(mid)
+                    if queue:
+                        if raw is None:
+                            raw = _dumps(msg)
+                        try:
+                            queue.put_nowait(raw)
+                        except asyncio.QueueFull:
+                            await queue.put(raw)
+                        self._cnt_queued += 1
+                    break
+
+    # ── Helpers ──
+
+    @staticmethod
+    async def _send_ack(ws: WebSocket, event_id: str):
+        try:
+            await ws.send_text('{"type":"ack","event_id":"' + event_id + '"}')
+        except Exception:
+            pass
+
+    @staticmethod
+    async def _send_error(ws: WebSocket, message: str):
+        try:
+            await ws.send_json({"type": "error", "message": message})
+        except Exception:
+            pass
+
+    # ── Stats ──
+
+    def _counters_dict(self) -> dict:
+        return {
+            "events_received": self._cnt_received,
+            "events_routed": self._cnt_routed,
+            "events_queued": self._cnt_queued,
+            "events_deduplicated": self._cnt_dedup,
+            "errors": self._cnt_errors,
+        }
+
+    def get_stats(self) -> dict:
+        return {
+            "connections": dict(self.connection_info),
+            "subscriptions": {
+                mid: sorted(p[0] for p in patterns)
+                for mid, patterns in self.subscriptions.items()
+            },
+            "counters": self._counters_dict(),
+        }
+
+    def get_health(self) -> dict:
+        return {
+            "status": "healthy",
+            "details": {
+                "connections": len(self.connections),
+                "subscriptions": sum(
+                    len(p) for p in self.subscriptions.values()
+                ),
+                "events_routed": self._cnt_routed,
+                "dedup_size": self.dedup.size,
+                "uptime_seconds": round(time.time() - self._start_time),
+            },
+        }

package/core/event_hub/module.md

@@ -0,0 +1,21 @@
+---
+name: event_hub
+display_name: Event Hub
+version: "1.0"
+type: infrastructure
+state: enabled
+runtime: python
+entry: entry.py
+events: []
+subscriptions: []
+test_mode: true
+---
+
+# Event Hub
+
+Kite 系统的实时事件路由器。
+
+- 接收模块通过 WebSocket 发送的事件
+- 根据订阅关系（支持 NATS 风格通配符）转发给匹配的模块
+- 事件去重（1h 滑动窗口）防止 outbox 重放导致重复转发
+- ACK 确认机制，模块据此清理本地 outbox

package/core/event_hub/router.py

@@ -0,0 +1,21 @@
+"""
+Subscription table management + NATS-style wildcard matching.
+* matches exactly one segment, > matches one or more (tail only).
+"""
+
+
+def match_event(pattern: str, event_type: str) -> bool:
+    """NATS-style: * matches one segment, > matches one or more (tail only)."""
+    return match_parts(tuple(pattern.split(".")), tuple(event_type.split(".")))
+
+
+def match_parts(p_parts: tuple, e_parts: tuple) -> bool:
+    """Same logic as match_event but on pre-split tuples."""
+    for i, p in enumerate(p_parts):
+        if p == ">":
+            return i < len(e_parts)
+        if i >= len(e_parts):
+            return False
+        if p != "*" and p != e_parts[i]:
+            return False
+    return len(p_parts) == len(e_parts)

package/core/event_hub/server.py

@@ -0,0 +1,138 @@
+"""
+Event Hub HTTP + WebSocket server.
+FastAPI app: /ws (WebSocket), /health, /stats.
+30s timer for heartbeat renewal + dedup cleanup.
+"""
+
+import asyncio
+
+import httpx
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+
+from .hub import EventHub
+
+
+class EventHubServer:
+
+    def __init__(self, hub: EventHub, own_token: str, registry_url: str, test_mode: bool = False):
+        self.hub = hub
+        self.own_token = own_token
+        self.registry_url = registry_url
+        self.test_mode = test_mode
+        self._timer_task: asyncio.Task | None = None
+        self.app = self._create_app()
+
+    # ── Token verification via Registry ──
+
+    async def _verify_token(self, token: str, module_id_hint: str = "") -> str | None:
+        """Call Registry POST /verify to validate a module token. Returns module_id or None."""
+        if self.test_mode:
+            return module_id_hint or token[:32] or "test_module"
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    f"{self.registry_url}/verify",
+                    json={"token": token},
+                    headers={"Authorization": f"Bearer {self.own_token}"},
+                    timeout=5,
+                )
+                if resp.status_code == 200:
+                    body = resp.json()
+                    if body.get("ok"):
+                        return body.get("module_id")
+        except Exception as e:
+            print(f"[event_hub] Token verification failed: {e}")
+        return None
+
+    # ── App factory ──
+
+    def _create_app(self) -> FastAPI:
+        app = FastAPI(title="Kite Event Hub", docs_url=None, redoc_url=None)
+        server = self
+
+        @app.on_event("startup")
+        async def _startup():
+            server._timer_task = asyncio.create_task(server._timer_loop())
+            server._test_task = asyncio.create_task(server._test_event_loop())
+
+        @app.on_event("shutdown")
+        async def _shutdown():
+            if server._timer_task:
+                server._timer_task.cancel()
+            if hasattr(server, '_test_task') and server._test_task:
+                server._test_task.cancel()
+
+        # ── WebSocket endpoint ──
+
+        @app.websocket("/ws")
+        async def ws_endpoint(ws: WebSocket):
+            token = ws.query_params.get("token", "")
+            mid_hint = ws.query_params.get("id", "")
+            module_id = await server._verify_token(token, mid_hint)
+            if not module_id:
+                await ws.close(code=4001, reason="Authentication failed")
+                return
+
+            await ws.accept()
+            server.hub.add_connection(module_id, ws)
+
+            try:
+                while True:
+                    raw = await ws.receive_text()
+                    await server.hub.handle_message(module_id, ws, raw)
+            except WebSocketDisconnect:
+                pass
+            except Exception as e:
+                print(f"[event_hub] WebSocket error for {module_id}: {e}")
+            finally:
+                server.hub.remove_connection(module_id)
+
+        # ── HTTP endpoints ──
+
+        @app.get("/health")
+        async def health():
+            return server.hub.get_health()
+
+        @app.get("/stats")
+        async def stats():
+            return server.hub.get_stats()
+
+        return app
+
+    # ── 30s timer: heartbeat + dedup cleanup ──
+
+    async def _timer_loop(self):
+        while True:
+            await asyncio.sleep(30)
+            # Dedup cleanup (offload to thread to avoid blocking event loop)
+            await asyncio.get_event_loop().run_in_executor(None, self.hub.dedup.cleanup)
+            # Heartbeat to Registry
+            await self._heartbeat()
+
+    async def _heartbeat(self):
+        try:
+            async with httpx.AsyncClient() as client:
+                await client.post(
+                    f"{self.registry_url}/modules",
+                    json={"action": "heartbeat", "module_id": "event_hub"},
+                    headers={"Authorization": f"Bearer {self.own_token}"},
+                    timeout=5,
+                )
+        except Exception:
+            pass
+
+    async def _test_event_loop(self):
+        """Publish a test event every 5 seconds via internal routing."""
+        import uuid
+        from datetime import datetime, timezone
+        while True:
+            await asyncio.sleep(5)
+            msg = {
+                "type": "event",
+                "event_id": str(uuid.uuid4()),
+                "event": "event_hub.test",
+                "source": "event_hub",
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "data": {"message": "heartbeat from event_hub"},
+            }
+            await self.hub._route_event("event_hub", msg)