@agentunion/kite 1.0.0

package/core/registry/server.py

@@ -0,0 +1,256 @@
+"""
+Registry HTTP server.
+7 endpoints: /modules, /lookup, /get/{path}, /tokens, /verify, /query, /health.
+All endpoints except /health require Bearer token auth.
+Connects to Event Hub to publish module lifecycle events.
+"""
+
+import asyncio
+import json
+import uuid
+from typing import Any
+
+import websockets
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import JSONResponse
+
+from .store import RegistryStore
+
+
+class RegistryServer:
+    """FastAPI-based Registry HTTP server."""
+
+    def __init__(self, store: RegistryStore, launcher_token: str = ""):
+        self.store = store
+        self.launcher_token = launcher_token
+        self.app = self._create_app()
+        self._ttl_task: asyncio.Task | None = None
+        # Event Hub WebSocket
+        self._event_hub_ws_url: str = ""
+        self._ws: object | None = None
+        self._ws_task: asyncio.Task | None = None
+
+    def _extract_token(self, request: Request) -> str:
+        """Extract Bearer token from Authorization header."""
+        auth = request.headers.get("Authorization", "")
+        if auth.startswith("Bearer "):
+            return auth[7:].strip()
+        return ""
+
+    def _require_auth(self, request: Request) -> str:
+        """Verify token, return module_id. Raise 401 on failure."""
+        token = self._extract_token(request)
+        module_id = self.store.verify_token(token)
+        if module_id is None:
+            raise HTTPException(status_code=401, detail="Invalid or missing token")
+        return module_id
+
+    def _require_launcher(self, request: Request):
+        """Verify the caller is Launcher. Raise 403 if not."""
+        token = self._extract_token(request)
+        if not self.store.is_launcher(token):
+            raise HTTPException(status_code=403, detail="Only Launcher may call this endpoint")
+
+    # ── Event Hub connection ──
+
+    async def _try_connect_event_hub(self):
+        """Check if Event Hub is registered and connect if not already connected."""
+        if self._ws:
+            return
+        # Look up Event Hub ws_endpoint from our own store
+        eh = self.store.modules.get("event_hub")
+        if not eh:
+            return
+        ws_url = (eh.get("metadata") or {}).get("ws_endpoint", "")
+        if not ws_url:
+            return
+        self._event_hub_ws_url = ws_url
+        if not self._ws_task:
+            self._ws_task = asyncio.create_task(self._ws_loop())
+
+    async def _ws_loop(self):
+        """Connect to Event Hub, reconnect on failure."""
+        while True:
+            try:
+                await self._ws_connect()
+            except asyncio.CancelledError:
+                return
+            except Exception as e:
+                print(f"[registry] Event Hub connection error: {e}")
+            self._ws = None
+            await asyncio.sleep(5)
+
+    async def _ws_connect(self):
+        """Single WebSocket session."""
+        # Use registry's own per-module token (registered by Launcher via /tokens)
+        # to avoid conflicting with Launcher's connection (same launcher_token → same module_id)
+        token = self.store.token_map.get("registry", "") or self.launcher_token
+        ws_url = f"{self._event_hub_ws_url}?token={token}"
+        async with websockets.connect(ws_url) as ws:
+            self._ws = ws
+            print("[registry] Connected to Event Hub")
+            async for raw in ws:
+                try:
+                    msg = json.loads(raw)
+                except (json.JSONDecodeError, TypeError):
+                    continue
+                msg_type = msg.get("type", "")
+                if msg_type == "error":
+                    print(f"[registry] Event Hub error: {msg.get('message')}")
+
+    async def _publish_event(self, event_type: str, data: dict):
+        """Publish event to Event Hub. Best-effort, no-op if not connected."""
+        if not self._ws:
+            return
+        from datetime import datetime, timezone
+        msg = {
+            "type": "event",
+            "event_id": str(uuid.uuid4()),
+            "event": event_type,
+            "source": "registry",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "data": data,
+        }
+        try:
+            await self._ws.send(json.dumps(msg))
+        except Exception:
+            pass
+
+    # ── App factory ──
+
+    def _create_app(self) -> FastAPI:
+        app = FastAPI(title="Kite Registry", docs_url=None, redoc_url=None)
+        server = self
+
+        @app.on_event("startup")
+        async def _startup():
+            server._ttl_task = asyncio.create_task(server._ttl_loop())
+
+        @app.on_event("shutdown")
+        async def _shutdown():
+            if server._ttl_task:
+                server._ttl_task.cancel()
+            if server._ws_task:
+                server._ws_task.cancel()
+            if server._ws:
+                await server._ws.close()
+
+        # ── 1. POST /modules ──
+
+        @app.post("/modules")
+        async def modules(request: Request):
+            caller = server._require_auth(request)
+            body = await request.json()
+            action = body.get("action", "")
+
+            if action == "register":
+                if "module_id" not in body:
+                    raise HTTPException(400, "module_id required")
+                # Only Launcher or the module itself may register
+                if caller != "launcher" and caller != body["module_id"]:
+                    raise HTTPException(403, f"Module '{caller}' cannot register as '{body['module_id']}'")
+                result = server.store.register_module(body)
+                if result.get("ok"):
+                    mid = body["module_id"]
+                    await server._publish_event("module.registered", {"module_id": mid})
+                    # If Event Hub just registered, try connecting
+                    if mid == "event_hub":
+                        await server._try_connect_event_hub()
+                return result
+
+            elif action == "deregister":
+                mid = body.get("module_id")
+                if not mid:
+                    raise HTTPException(400, "module_id required")
+                if caller != "launcher" and caller != mid:
+                    raise HTTPException(403, f"Module '{caller}' cannot deregister '{mid}'")
+                result = server.store.deregister_module(mid)
+                if result.get("ok"):
+                    await server._publish_event("module.unregistered", {"module_id": mid})
+                return result
+
+            elif action == "heartbeat":
+                mid = body.get("module_id")
+                if not mid:
+                    raise HTTPException(400, "module_id required")
+                if caller != "launcher" and caller != mid:
+                    raise HTTPException(403, f"Module '{caller}' cannot heartbeat for '{mid}'")
+                result = server.store.heartbeat(mid)
+                if result.get("ok"):
+                    await server._publish_event("module.heartbeat", {"module_id": mid})
+                return result
+
+            else:
+                raise HTTPException(400, f"Unknown action: {action}")
+
+        # ── 2. GET /lookup ──
+
+        @app.get("/lookup")
+        async def lookup(request: Request, field: str = None, module: str = None, value: str = None):
+            server._require_auth(request)
+            return server.store.lookup(field=field, module=module, value=value)
+
+        # ── 3. GET /get/{path} ──
+
+        @app.get("/get/{path:path}")
+        async def get_by_path(request: Request, path: str):
+            server._require_auth(request)
+            val, found = server.store.get_by_path(path)
+            if not found:
+                raise HTTPException(404, f"Path not found: {path}")
+            return val
+
+        # ── 4. POST /tokens ──
+
+        @app.post("/tokens")
+        async def register_tokens(request: Request):
+            server._require_launcher(request)
+            body = await request.json()
+            server.store.register_tokens(body)
+            return {"ok": True}
+
+        # ── 5. POST /verify ──
+
+        @app.post("/verify")
+        async def verify_token(request: Request):
+            server._require_auth(request)
+            body = await request.json()
+            target_token = body.get("token", "")
+            module_id = server.store.verify_token(target_token)
+            if module_id:
+                return {"ok": True, "module_id": module_id}
+            return {"ok": False}
+
+        # ── 6. POST /query (stub) ──
+        # TODO: implement LLM semantic query per design §5.1
+        #       accept {"question": "..."}, search registry with LLM, return matched modules/tools
+
+        @app.post("/query")
+        async def query(request: Request):
+            server._require_auth(request)
+            body = await request.json()
+            question = body.get("question", "")
+            return {"ok": False, "error": "LLM query not implemented yet", "question": question}
+
+        # ── 7. GET /health ──
+
+        @app.get("/health")
+        async def health():
+            return {
+                "status": "healthy",
+                "module_count": len(server.store.modules),
+                "online_count": sum(
+                    1 for m in server.store.modules.values()
+                    if m.get("status") == "online"
+                ),
+            }
+
+        return app
+
+    async def _ttl_loop(self):
+        """Periodically check heartbeat TTL and publish offline events."""
+        while True:
+            await asyncio.sleep(10)
+            expired = self.store.check_ttl()
+            for mid in expired:
+                await self._publish_event("module.offline", {"module_id": mid})

package/core/registry/store.py

@@ -0,0 +1,232 @@
+"""
+Registry in-memory store.
+Manages module records, token verification, heartbeat TTL, lookup and get-by-path.
+No persistence — Registry crash triggers Kite full restart, all modules re-register.
+"""
+
+import fnmatch
+import time
+from typing import Any
+
+
+class RegistryStore:
+
+    def __init__(self, launcher_token: str):
+        self.launcher_token = launcher_token
+        self.token_map: dict[str, str] = {}       # module_id -> token
+        self.modules: dict[str, dict] = {}         # module_id -> registration payload
+        self.heartbeats: dict[str, float] = {}     # module_id -> last heartbeat timestamp
+        self.ttl = 60           # seconds before marking offline
+        self.heartbeat_interval = 30
+
+    # ── Token verification ──
+
+    def verify_token(self, token: str) -> str | None:
+        """Return module_id if token is valid, None otherwise."""
+        if token == self.launcher_token:
+            return "launcher"
+        for mid, tok in self.token_map.items():
+            if token == tok:
+                return mid
+        return None
+
+    def is_launcher(self, token: str) -> bool:
+        return token == self.launcher_token
+
+    def register_tokens(self, mapping: dict[str, str]):
+        """Register module_id -> token mapping. Only Launcher may call this."""
+        self.token_map.update(mapping)
+
+    # ── Module lifecycle ──
+
+    _REQUIRED_FIELDS = ("module_id", "module_type", "api_endpoint")
+
+    def register_module(self, data: dict) -> dict:
+        """Register or update a module. Idempotent — same module_id overwrites."""
+        # Validate required fields
+        missing = [f for f in self._REQUIRED_FIELDS if not data.get(f)]
+        if missing:
+            return {"ok": False, "error": f"Missing required fields: {', '.join(missing)}"}
+
+        mid = data["module_id"]
+
+        # Warn on tool name conflicts
+        new_tools = data.get("tools", {})
+        if isinstance(new_tools, dict):
+            for tool_name in new_tools:
+                for other_mid, other_data in self.modules.items():
+                    if other_mid == mid:
+                        continue
+                    if tool_name in other_data.get("tools", {}):
+                        print(f"[registry] WARNING: tool '{tool_name}' registered by both '{other_mid}' and '{mid}'")
+
+        # Strip action field — it's a request verb, not part of the registration payload
+        record = {k: v for k, v in data.items() if k != "action"}
+        record["status"] = "online"
+        record["registered_at"] = time.time()
+        self.modules[mid] = record
+        self.heartbeats[mid] = time.time()
+        return {"ok": True, "ttl": self.ttl, "heartbeat_interval": self.heartbeat_interval}
+
+    def deregister_module(self, module_id: str) -> dict:
+        """Remove a module record immediately."""
+        self.modules.pop(module_id, None)
+        self.heartbeats.pop(module_id, None)
+        return {"ok": True}
+
+    def heartbeat(self, module_id: str) -> dict:
+        """Renew heartbeat for a module."""
+        if module_id not in self.modules:
+            return {"ok": False, "error": "module not registered"}
+        self.heartbeats[module_id] = time.time()
+        self.modules[module_id]["status"] = "online"
+        return {"ok": True}
+
+    def check_ttl(self) -> list[str]:
+        """Mark modules as offline if heartbeat expired. Returns list of newly-offline module_ids."""
+        now = time.time()
+        expired = []
+        for mid, last in list(self.heartbeats.items()):
+            if mid in self.modules and now - last > self.ttl:
+                if self.modules[mid].get("status") != "offline":
+                    self.modules[mid]["status"] = "offline"
+                    expired.append(mid)
+        return expired
+
+    # ── Get by dot-path ──
+
+    def get_by_path(self, path: str) -> Any | None:
+        """
+        Resolve a dot-path like 'kernel.tools.read_file.endpoint'.
+        First segment is module_id, rest navigates into the registration dict.
+        Handles dotted keys (e.g. 'session.ended') by greedy matching.
+        Returns (value, True) on hit, (None, False) on miss.
+        """
+        parts = path.split(".")
+        if not parts:
+            return None, False
+
+        # First segment: module_id
+        module_id = parts[0]
+        data = self.modules.get(module_id)
+        if data is None:
+            return None, False
+        if len(parts) == 1:
+            return data, True
+
+        return self._resolve_path(data, parts[1:])
+
+    @staticmethod
+    def _resolve_path(obj: Any, parts: list[str]) -> tuple[Any, bool]:
+        """
+        Walk into obj using parts. Greedy: try joining multiple parts
+        as a single dotted key before falling back to single-segment traversal.
+        """
+        if not parts:
+            return obj, True
+        if not isinstance(obj, dict):
+            return None, False
+
+        # Try longest key first (greedy match for dotted keys like 'session.ended')
+        for i in range(len(parts), 0, -1):
+            candidate = ".".join(parts[:i])
+            if candidate in obj:
+                return RegistryStore._resolve_path(obj[candidate], parts[i:])
+
+        return None, False
+
+    # ── Lookup (glob search) ──
+
+    def lookup(self, field: str = None, module: str = None, value: str = None) -> list[dict]:
+        """
+        Search across all online modules. All three params support glob patterns.
+        Returns list of {field, module, api_endpoint, value}.
+        """
+        results = []
+        for mid, data in self.modules.items():
+            if data.get("status") != "online":
+                continue
+            if module and not fnmatch.fnmatch(mid, module):
+                continue
+
+            api_ep = data.get("api_endpoint", "")
+
+            if field:
+                matches = self._match_fields(data, field)
+                for fpath, fval in matches:
+                    if value and not self._value_matches(fval, value):
+                        continue
+                    results.append({
+                        "field": fpath,
+                        "module": mid,
+                        "api_endpoint": api_ep,
+                        "value": fval,
+                    })
+            elif value:
+                for k, v in data.items():
+                    if self._value_matches(v, value):
+                        results.append({
+                            "field": k,
+                            "module": mid,
+                            "api_endpoint": api_ep,
+                            "value": v,
+                        })
+            else:
+                results.append({
+                    "field": "module_id",
+                    "module": mid,
+                    "api_endpoint": api_ep,
+                    "value": mid,
+                })
+
+        return results
+
+    # ── Lookup helpers ──
+
+    @staticmethod
+    def _match_fields(data: dict, pattern: str) -> list[tuple[str, Any]]:
+        """
+        Walk a module's registration dict and find all field paths matching a glob pattern.
+        Pattern like 'tools.*' matches top-level keys under 'tools'.
+        Pattern like 'tools.*file*' matches tool names containing 'file'.
+        Pattern like 'events_publish.session.ended' matches dotted keys via greedy join.
+        Returns list of (field_path, value).
+        """
+        parts = pattern.split(".")
+        return RegistryStore._walk_fields(data, parts, "")
+
+    @staticmethod
+    def _walk_fields(obj: Any, parts: list[str], prefix: str) -> list[tuple[str, Any]]:
+        """Recursively walk dict, matching glob pattern segments against keys."""
+        if not parts:
+            return [(prefix, obj)] if prefix else []
+        if not isinstance(obj, dict):
+            return []
+
+        results = []
+        # Try greedy: join multiple pattern segments to match dotted keys
+        for i in range(len(parts), 0, -1):
+            candidate_pattern = ".".join(parts[:i])
+            remaining = parts[i:]
+
+            for key in obj:
+                if fnmatch.fnmatch(key, candidate_pattern):
+                    new_prefix = f"{prefix}.{key}" if prefix else key
+                    if remaining:
+                        results.extend(
+                            RegistryStore._walk_fields(obj[key], remaining, new_prefix)
+                        )
+                    else:
+                        results.append((new_prefix, obj[key]))
+
+            if results:
+                break  # greedy: longest match wins
+
+        return results
+
+    @staticmethod
+    def _value_matches(val: Any, pattern: str) -> bool:
+        """Check if a value matches a glob pattern. Dicts/lists never match."""
+        if isinstance(val, (dict, list)):
+            return False
+        return fnmatch.fnmatch(str(val), pattern)

package/extensions/services/watchdog/entry.py

@@ -0,0 +1,143 @@
+"""
+Watchdog entry point.
+Reads boot_info from stdin, registers to Registry, starts health monitor.
+"""
+
+import json
+import os
+import socket
+import sys
+
+import httpx
+import uvicorn
+
+# Ensure project root is on sys.path
+_this_dir = os.path.dirname(os.path.abspath(__file__))
+_project_root = os.path.dirname(os.path.dirname(os.path.dirname(_this_dir)))
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+from extensions.services.watchdog.monitor import HealthMonitor
+from extensions.services.watchdog.server import WatchdogServer
+
+
+def _get_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+def _register_to_registry(token: str, registry_url: str, port: int):
+    payload = {
+        "action": "register",
+        "module_id": "watchdog",
+        "module_type": "service",
+        "name": "Watchdog",
+        "api_endpoint": f"http://127.0.0.1:{port}",
+        "health_endpoint": "/health",
+        "events_publish": {
+            "watchdog.module.unhealthy": {"description": "Module failed health check"},
+            "watchdog.module.recovered": {"description": "Module recovered from unhealthy"},
+            "watchdog.alert": {"description": "Module restarted too many times"},
+        },
+        "events_subscribe": [
+            "module.started",
+            "module.stopped",
+        ],
+    }
+    headers = {"Authorization": f"Bearer {token}"}
+    resp = httpx.post(
+        f"{registry_url}/modules",
+        json=payload, headers=headers, timeout=5,
+    )
+    if resp.status_code == 200:
+        print("[watchdog] Registered to Registry")
+    else:
+        print(f"[watchdog] WARNING: Registry returned {resp.status_code}")
+
+
+def _get_launcher_url(token: str, registry_url: str) -> str:
+    """Discover Launcher API endpoint from Registry."""
+    headers = {"Authorization": f"Bearer {token}"}
+    try:
+        resp = httpx.get(
+            f"{registry_url}/get/launcher.api_endpoint",
+            headers=headers, timeout=5,
+        )
+        if resp.status_code == 200:
+            return resp.json()
+    except Exception:
+        pass
+    return ""
+
+
+def _get_event_hub_ws(token: str, registry_url: str) -> str:
+    """Discover Event Hub WebSocket endpoint from Registry, with retry."""
+    import time
+    headers = {"Authorization": f"Bearer {token}"}
+    deadline = time.time() + 10
+    while time.time() < deadline:
+        try:
+            resp = httpx.get(
+                f"{registry_url}/get/event_hub.metadata.ws_endpoint",
+                headers=headers, timeout=3,
+            )
+            if resp.status_code == 200:
+                val = resp.json()
+                if val:
+                    return val
+        except Exception:
+            pass
+        time.sleep(1)
+    return ""
+
+
+def main():
+    # Read boot_info from stdin
+    token = ""
+    registry_port = 0
+    try:
+        line = sys.stdin.readline().strip()
+        if line:
+            boot_info = json.loads(line)
+            token = boot_info.get("token", "")
+            registry_port = boot_info.get("registry_port", 0)
+    except Exception:
+        pass
+
+    if not token or not registry_port:
+        print("[watchdog] ERROR: Missing token or registry_port in boot_info")
+        sys.exit(1)
+
+    print(f"[watchdog] Token received ({len(token)} chars), registry port: {registry_port}")
+
+    registry_url = f"http://127.0.0.1:{registry_port}"
+    port = _get_free_port()
+
+    # Register to Registry
+    _register_to_registry(token, registry_url, port)
+
+    # Discover Launcher URL
+    launcher_url = _get_launcher_url(token, registry_url)
+    if not launcher_url:
+        print("[watchdog] WARNING: Could not discover Launcher URL, restart disabled")
+
+    # Discover Event Hub WebSocket URL
+    event_hub_ws = _get_event_hub_ws(token, registry_url)
+    if not event_hub_ws:
+        print("[watchdog] WARNING: Could not discover Event Hub WS, events disabled")
+
+    # Create monitor and server
+    monitor = HealthMonitor(
+        own_token=token,
+        registry_url=registry_url,
+        launcher_url=launcher_url,
+    )
+    server = WatchdogServer(monitor, token=token, event_hub_ws=event_hub_ws)
+
+    print(f"[watchdog] Starting on port {port}")
+    uvicorn.run(server.app, host="127.0.0.1", port=port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

package/extensions/services/watchdog/module.md

@@ -0,0 +1,25 @@
+---
+name: watchdog
+display_name: Watchdog
+version: "1.0"
+type: service
+state: enabled
+runtime: python
+entry: entry.py
+events:
+  - watchdog.module.unhealthy
+  - watchdog.module.recovered
+  - watchdog.alert
+subscriptions:
+  - module.started
+  - module.stopped
+---
+
+# Watchdog（保活模块）
+
+应用层健康监控扩展模块。Launcher 处理进程级故障，Watchdog 处理应用级故障（进程活着但不健康）。
+
+- 心跳检测 — 定期检查各模块的 `/health` 端点
+- 资源监控 — 检测模块健康状态中的异常指标
+- 非 core 模块重启 — 发现异常时通过 Launcher API 重启模块
+- 告警通知 — 模块持续异常时，通过 Event Hub 发布告警事件