@agentunion/kite 1.0.7 → 1.2.0

package/extensions/services/watchdog/monitor.py

@@ -78,6 +78,16 @@ class HealthMonitor:
         self._running = False
         self._psutil = None  # lazy import
 
+        # Restart decision state (module.exiting / module.stopped / module.ready)
+        self._exit_intents: dict[str, str] = {}        # module_id -> action from module.exiting
+        self._graceful_modules: dict[str, bool] = {     # module_id -> supports graceful shutdown
+            "registry": True, "event_hub": True,        # started before Watchdog, default True
+        }
+        self._system_shutting_down = False
+        self._system_ready = False
+        self._system_ready_event = asyncio.Event()
+        self._crash_counts: dict[str, int] = {}         # module_id -> consecutive crash count
+
     # ── Module discovery ──
 
     async def discover_modules(self):
@@ -126,15 +136,21 @@ class HealthMonitor:
         for mid, pid in monitored.items():
             seen.add(mid)
             h = health_map.get(mid, {})
+            ep = h.get("api_endpoint", "")
+            hp = h.get("health_endpoint", "/health")
             if mid not in self.modules:
                 self.modules[mid] = ModuleStatus(
                     module_id=mid,
-                    api_endpoint=h.get("api_endpoint", ""),
-                    health_endpoint=h.get("health_endpoint", "/health"),
+                    api_endpoint=ep,
+                    health_endpoint=hp,
                     pid=pid,
                 )
             else:
                 self.modules[mid].pid = pid
+                # Refresh endpoints once Registry has them
+                if ep:
+                    self.modules[mid].api_endpoint = ep
+                    self.modules[mid].health_endpoint = hp
 
         for mid in list(self.modules):
             if mid not in seen:
@@ -144,6 +160,8 @@ class HealthMonitor:
 
     async def _check_one(self, status: ModuleStatus):
         """Check a single module's /health endpoint."""
+        if not status.api_endpoint:
+            return  # Not yet registered in Registry, will be picked up on next discover
         url = f"{status.api_endpoint}{status.health_endpoint}"
         status.last_check = time.time()
 
@@ -348,38 +366,135 @@ class HealthMonitor:
     # ── Incoming event handler ──
 
     async def handle_event(self, msg: dict):
-        """Handle events from Event Hub (module.started / module.stopped)."""
+        """Handle events from Event Hub — restart decisions + health tracking."""
         event_type = msg.get("event", "")
         data = msg.get("data", {})
         module_id = data.get("module_id", "")
 
+        if event_type == "system.ready":
+            print("[watchdog] Received system.ready")
+            self._system_ready = True
+            self._system_ready_event.set()
+            return
+
         if not module_id or module_id == "watchdog":
             return
 
         if event_type == "module.started":
             print(f"[watchdog] Received module.started: {module_id}")
-            # Trigger immediate discovery to pick up the new module
+            self._crash_counts.pop(module_id, None)
             await self.discover_modules()
 
         elif event_type == "module.stopped":
             print(f"[watchdog] Received module.stopped: {module_id}")
-            # Remove from tracking — it's gone
             self.modules.pop(module_id, None)
+            await self._handle_module_stopped(module_id, data)
+
+        elif event_type == "module.exiting":
+            action = data.get("action", "none")
+            print(f"[watchdog] Received module.exiting: {module_id}, action={action}")
+            self._exit_intents[module_id] = action
+
+        elif event_type == "module.ready":
+            graceful = bool(data.get("graceful_shutdown"))
+            print(f"[watchdog] Received module.ready: {module_id}, graceful_shutdown={graceful}")
+            self._graceful_modules[module_id] = graceful
+
+        elif event_type == "module.shutdown":
+            reason = data.get("reason", "")
+            if reason == "system_shutdown":
+                print(f"[watchdog] Received system_shutdown signal")
+                self._system_shutting_down = True
+
+    async def _handle_module_stopped(self, module_id: str, data: dict):
+        """Restart decision engine — called when module.stopped is received.
+
+        Priority:
+        1. System shutting down → no restart
+        2. Has exit_intent → follow the declared action (none/restart/restart_delay)
+        3. No intent (crash) → increment crash count, restart up to MAX_RESTARTS
+        """
+        # Sync graceful_shutdown from Launcher (covers missed module.ready)
+        if "graceful_shutdown" in data:
+            self._graceful_modules[module_id] = bool(data["graceful_shutdown"])
+
+        if self._system_shutting_down:
+            print(f"[watchdog] {module_id} stopped during shutdown, skipping restart")
+            return
+
+        intent = self._exit_intents.pop(module_id, None)
+        if intent is not None:
+            if intent == "none":
+                print(f"[watchdog] {module_id} exited intentionally (action=none), no restart")
+                return
+            elif intent == "restart":
+                print(f"[watchdog] {module_id} requested restart, restarting now")
+                await self._restart_module_by_id(module_id, reason="module_requested")
+                return
+            elif intent == "restart_delay":
+                delay = 5
+                print(f"[watchdog] {module_id} requested delayed restart, waiting {delay}s")
+                await asyncio.sleep(delay)
+                await self._restart_module_by_id(module_id, reason="module_requested_delay")
+                return
+            else:
+                print(f"[watchdog] {module_id} exited with unknown action '{intent}', no restart")
+                return
+
+        # No exit intent → treat as crash
+        self._crash_counts[module_id] = self._crash_counts.get(module_id, 0) + 1
+        count = self._crash_counts[module_id]
+        exit_code = data.get("exit_code", -1)
+
+        if count <= self.MAX_RESTARTS:
+            print(f"[watchdog] {module_id} crashed (exit_code={exit_code}), restarting ({count}/{self.MAX_RESTARTS})")
+            await self._restart_module_by_id(module_id, reason="crash")
+        else:
+            print(f"[watchdog] {module_id} crashed {count} times, giving up")
+            await self._publish("watchdog.alert", {
+                "module_id": module_id,
+                "message": f"{module_id} exceeded {self.MAX_RESTARTS} crash restarts",
+            })
+
+    async def _restart_module_by_id(self, module_id: str, reason: str = "restart"):
+        """Restart a module via Launcher API by module_id."""
+        print(f"[watchdog] Requesting restart for {module_id} (reason={reason})")
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    f"{self.launcher_url}/launcher/modules/{module_id}/restart",
+                    json={"reason": reason},
+                    timeout=15,
+                )
+                if resp.status_code == 200:
+                    print(f"[watchdog] {module_id} restart requested successfully")
+                else:
+                    print(f"[watchdog] {module_id} restart failed: HTTP {resp.status_code}")
+        except Exception as e:
+            print(f"[watchdog] {module_id} restart error: {e}")
 
     # ── Main loop ──
 
     async def run(self):
-        """Main monitoring loop with dynamic intervals per resource state."""
+        """Main monitoring loop with dynamic intervals per resource state.
+        Waits for system.ready before starting health checks."""
         self._running = True
-        print("[watchdog] Monitor started")
-        last_discover = 0
+        print("[watchdog] Monitor started, waiting for system.ready...")
+
+        # Wait for system.ready (or manual start via API later)
+        while self._running and not self._system_ready:
+            try:
+                await asyncio.wait_for(self._system_ready_event.wait(), timeout=1.0)
+            except asyncio.TimeoutError:
+                continue
+
+        if not self._running:
+            return
+        print("[watchdog] system.ready received, starting health checks")
 
         while self._running:
-            now = time.time()
-            # Rediscover every 30s
-            if now - last_discover >= 30:
-                await self.discover_modules()
-                last_discover = now
+            # Re-discover every cycle to pick up newly started/stopped modules
+            await self.discover_modules()
 
             if self.modules:
                 tasks = []

package/extensions/services/watchdog/server.py

@@ -27,7 +27,10 @@ class WatchdogServer:
         self._ws_task: asyncio.Task | None = None
         self._heartbeat_task: asyncio.Task | None = None
         self._ws: object | None = None
+        self._ready_sent = False
         self._start_time = time.time()
+        self._uvicorn_server = None  # set by entry.py for graceful shutdown
+        self._shutting_down = False
         self.app = self._create_app()
 
         # Wire up publish callback on monitor
@@ -80,29 +83,53 @@ class WatchdogServer:
 
     async def _ws_loop(self):
         """Connect to Event Hub, subscribe, and listen. Reconnect on failure."""
-        while True:
+        retry_delay = 0.5  # start with 0.5s
+        max_delay = 30  # cap at 30s
+        while not self._shutting_down:
             try:
                 await self._ws_connect()
+                retry_delay = 0.5  # reset on successful connection
             except asyncio.CancelledError:
                 return
             except Exception as e:
-                print(f"[watchdog] Event Hub connection error: {e}")
+                print(f"[watchdog] Event Hub connection error: {e}, retrying in {retry_delay:.1f}s")
             self._ws = None
-            await asyncio.sleep(5)  # reconnect delay
+            if self._shutting_down:
+                return
+            await asyncio.sleep(retry_delay)
+            retry_delay = min(retry_delay * 2, max_delay)  # exponential backoff
 
     async def _ws_connect(self):
         """Single WebSocket session: connect, subscribe, receive loop."""
-        url = f"{self.event_hub_ws}?token={self.token}"
-        async with websockets.connect(url, ping_interval=None, ping_timeout=None, close_timeout=10) as ws:
+        url = f"{self.event_hub_ws}?token={self.token}&id=watchdog"
+        print(f"[watchdog] WS connecting to {self.event_hub_ws}")
+        async with websockets.connect(url, open_timeout=3, ping_interval=None, ping_timeout=None, close_timeout=10) as ws:
             self._ws = ws
             print("[watchdog] Connected to Event Hub")
 
             # Subscribe to module lifecycle events
             await ws.send(json.dumps({
                 "type": "subscribe",
-                "events": ["module.started", "module.stopped"],
+                "events": ["system.ready", "module.started", "module.stopped", "module.exiting", "module.ready", "module.shutdown"],
             }))
 
+            # Send module.ready (once) so Launcher knows we're up
+            if not self._ready_sent:
+                from datetime import datetime, timezone
+                ready_msg = {
+                    "type": "event",
+                    "event_id": str(uuid.uuid4()),
+                    "event": "module.ready",
+                    "source": "watchdog",
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "data": {
+                        "module_id": "watchdog",
+                        "graceful_shutdown": True,
+                    },
+                }
+                await ws.send(json.dumps(ready_msg))
+                self._ready_sent = True
+
             # Receive loop
             async for raw in ws:
                 try:
@@ -110,13 +137,48 @@ class WatchdogServer:
                 except (json.JSONDecodeError, TypeError):
                     continue
 
-                msg_type = msg.get("type", "")
-                if msg_type == "event":
-                    await self.monitor.handle_event(msg)
-                elif msg_type == "ack":
-                    pass  # publish confirmed
-                elif msg_type == "error":
-                    print(f"[watchdog] Event Hub error: {msg.get('message')}")
+                try:
+                    msg_type = msg.get("type", "")
+                    if msg_type == "event":
+                        event = msg.get("event", "")
+                        data = msg.get("data") if isinstance(msg.get("data"), dict) else {}
+                        if event == "module.shutdown" and data.get("module_id") == "watchdog":
+                            await self._handle_shutdown(data)
+                            return
+                        await self.monitor.handle_event(msg)
+                    elif msg_type == "ack":
+                        pass  # publish confirmed
+                    elif msg_type == "error":
+                        print(f"[watchdog] Event Hub error: {msg.get('message')}")
+                except Exception as e:
+                    print(f"[watchdog] 事件处理异常（已忽略）: {e}")
+
+    async def _handle_shutdown(self, data: dict):
+        """Handle module.shutdown event — ack, cleanup, ready, exit."""
+        print("[watchdog] Received shutdown request")
+        self._shutting_down = True
+        # Step 1: Send ack
+        await self._publish_event({
+            "event": "module.shutdown.ack",
+            "data": {"module_id": "watchdog", "estimated_cleanup": 2},
+        })
+        # Step 2: Cleanup
+        self.monitor.stop()
+        if self._monitor_task:
+            self._monitor_task.cancel()
+        if self._heartbeat_task:
+            self._heartbeat_task.cancel()
+        if hasattr(self, '_test_task') and self._test_task:
+            self._test_task.cancel()
+        # Step 3: Send ready (before closing WS!)
+        await self._publish_event({
+            "event": "module.shutdown.ready",
+            "data": {"module_id": "watchdog"},
+        })
+        print("[watchdog] Shutdown ready, exiting")
+        # Step 4: Trigger uvicorn exit (WS will close when uvicorn shuts down)
+        if self._uvicorn_server:
+            self._uvicorn_server.should_exit = True
 
     async def _publish_event(self, event: dict):
         """Publish an event to Event Hub via WebSocket."""

package/extensions/services/web/__init__.py

@@ -0,0 +1 @@
+

package/extensions/services/web/config.yaml

@@ -0,0 +1,149 @@
+user:
+  phone_number: ""       # 当前 AI 使用的号码
+  phone_numbers: []      # AI 号码列表，可在配置页面管理
+  owners: []             # 主人列表，每项包含 phone/name 等基础信息
+  # 示例:
+  # phone_numbers:
+  #   - "13800138000"
+  #   - "13800138001"
+  # owners:
+  #   - phone: "13900139000"
+  #     name: "张三"
+  #     note: "公司CEO"
+  #   - phone: "13700137000"
+  #     name: "李四"
+
+server:
+  host: "0.0.0.0"
+  port: 18766
+  ssl: true
+
+bluetooth:
+  auto_connect: true
+  reconnect_interval: 5
+  reconnect_max_interval: 60
+  device_address: ""
+
+audio:
+  sample_rate: 16000
+  channels: 1
+  format: "s16le"
+  record_calls: true
+
+llm:
+  active_provider: "openai"
+  providers:
+    openai:
+      base_url: "https://api.openai.com/v1"
+      api_key: ""
+      model: "gpt-4o"
+      temperature: 0.7
+      max_tokens: 1024
+    claude:
+      base_url: "https://api.anthropic.com/v1"
+      api_key: ""
+      model: "claude-sonnet-4-20250514"
+      temperature: 0.7
+      max_tokens: 1024
+    gemini:
+      base_url: "https://generativelanguage.googleapis.com/v1beta"
+      api_key: ""
+      model: "gemini-2.0-flash"
+      temperature: 0.7
+      max_tokens: 1024
+
+asr:
+  provider: "whisper"
+  whisper:
+    base_url: "https://api.openai.com/v1"
+    api_key: ""
+    model: "whisper-1"
+    language: "zh"
+  xunfei:
+    app_id: ""
+    api_key: ""
+    api_secret: ""
+  volcengine:
+    app_id: ""
+    access_token: ""
+    resource_id: "volc.bigasr.sauc.duration"
+    ws_url: "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_async"
+  tencent:
+    app_id: ""
+    secret_id: ""
+    secret_key: ""
+    engine_model_type: "16k_zh_large"
+
+tts:
+  provider: "edge-tts"
+  edge_tts:
+    voice: "zh-CN-XiaoxiaoNeural"
+    rate: "+0%"
+    volume: "+0%"
+  volcengine:
+    app_id: ""
+    access_token: ""
+    cluster: "volcano_tts"
+    voice_type: "BV001_streaming"
+    speed_ratio: 1.0
+    volume_ratio: 1.0
+    pitch_ratio: 1.0
+    encoding: "mp3"
+    sample_rate: 24000
+  tencent:
+    app_id: ""
+    secret_id: ""
+    secret_key: ""
+    voice_type: 101001
+    codec: "pcm"
+    sample_rate: 16000
+    speed: 0
+    volume: 0
+  azure:
+    api_key: ""
+    region: "eastasia"
+    voice: "zh-CN-XiaoxiaoNeural"
+    rate: "1.0"
+    volume: "100"
+  openai:
+    api_key: ""
+    base_url: "https://api.openai.com/v1"
+    model: "tts-1"
+    voice: "alloy"
+    speed: 1.0
+  aliyun:
+    access_key_id: ""
+    access_key_secret: ""
+    app_key: ""
+    voice: "zhixiaobai"
+    sample_rate: 16000
+    speech_rate: 0
+    volume: 50
+  local:
+    engine: "sherpa-onnx"
+    model_path: ""
+    voice: ""
+    sample_rate: 16000
+
+vad:
+  mode: 3
+  energy_threshold: 300
+  silence_threshold_ms: 800
+  min_speech_ms: 250
+
+call:
+  max_duration_seconds: 300
+  no_response_timeout: 15
+  no_response_max_retries: 3
+  incoming_confirm_timeout: 15
+  incoming_default_action: "reject"
+  opening_message: ""
+
+webhook:
+  default_url: ""
+  retry_count: 3
+  retry_delays: [1, 5, 30]
+  timeout: 10
+
+data:
+  base_dir: "./data"