@agentunion/kite 1.4.0 → 1.6.0

package/extensions/services/watchdog/monitor.py

@@ -18,7 +18,9 @@ SYSTEM_BROADCAST_EVENTS = {
     "module.ready", "module.registered", "module.started", "module.stopped",
     "module.crashed", "module.exiting", "module.offline",
     "module.shutdown.ack", "module.shutdown.ready",
+    "module.degraded", "module.recovered",
     "system.ready", "registry.updated",
+    "system.instance.started", "system.instance.stopped",
 }
 
 # Module health states
@@ -35,10 +37,10 @@ CRITICAL = "critical"
 class ModuleStatus:
     """Track health and resource state for a single module."""
 
-    def __init__(self, module_id: str, api_endpoint: str, health_endpoint: str, pid: int = None):
+    def __init__(self, module_id: str, base_url: str, health_path: str, pid: int = None):
         self.module_id = module_id
-        self.api_endpoint = api_endpoint
-        self.health_endpoint = health_endpoint
+        self.base_url = base_url
+        self.health_path = health_path
         self.pid = pid
         # Health check state
         self.state: str = UNKNOWN
@@ -55,6 +57,8 @@ class ModuleStatus:
         self.memory_samples: list[float] = []  # last 5 memory_rss samples
         self.recovery_since: float = 0  # when recovery observation started
         self.last_metrics: dict = {}
+        # Startup metrics
+        self.startup_time: float = 0  # module startup time in seconds (from module.ready event)
 
 
 class HealthMonitor:
@@ -63,9 +67,13 @@ class HealthMonitor:
     # Thresholds
     HEALTH_TIMEOUT = 5           # HTTP timeout per health check
     FAIL_THRESHOLD = 3           # consecutive failures before restart
-    MAX_RESTARTS = 3             # max restarts before giving up on a module
+    MAX_STARTUP_RETRIES = 5      # 启动失败（未到 ready）最多重试 5 次
     ALERT_AFTER_RESTARTS = 2     # publish alert after this many restarts
 
+    # Crash restart policy
+    MAX_RESTART_DELAY = 600      # 最大重启延迟：10分钟
+    STABLE_RUN_TIME = 600        # 稳定运行时间：10分钟（超过此时间才重置崩溃计数）
+
     # Resource thresholds
     MEMORY_WARNING = 70          # memory_percent > 70% → warning
     MEMORY_CRITICAL = 85         # memory_percent > 85% → critical
@@ -82,9 +90,12 @@ class HealthMonitor:
         self.kernel_port = kernel_port
         self.publish_event = publish_event  # async callable(event_dict)
         self.rpc_call = None  # set by entry.py: async callable(method, params)
+        self.request_exit = None  # set by entry.py: callable(exit_code) — 请求进程退出
         self.modules: dict[str, ModuleStatus] = {}
         self._running = False
         self._psutil = None  # lazy import
+        self._dirty = False  # 模块上下线标记，定时器到期时检查
+        self._first_check_done = False  # 首次检查是否完成
 
         # Restart decision state (module.exiting / module.stopped / module.ready)
         self._exit_intents: dict[str, str] = {}        # module_id -> action from module.exiting
@@ -94,7 +105,11 @@ class HealthMonitor:
         self._system_shutting_down = False
         self._system_ready = False
         self._system_ready_event = asyncio.Event()
+        self._monitor_enabled = False  # 监控开关，由 Launcher 通过 RPC 控制
         self._crash_counts: dict[str, int] = {}         # module_id -> consecutive crash count
+        self._restart_tasks: dict[str, asyncio.Task] = {}  # module_id -> pending restart task
+        self._module_ready_times: dict[str, float] = {}    # module_id -> last ready timestamp
+        self._module_ever_ready: dict[str, bool] = {}      # module_id -> has ever been ready
 
         # Launcher loss tracking
         self._launcher_offline = False
@@ -102,6 +117,23 @@ class HealthMonitor:
         self._launcher_restart_requested = False  # True if launcher requested restart
         self._launcher_startup_info = None  # Startup info from launcher (python, argv, cwd, env)
 
+        # 降级状态跟踪：module_id -> {level, reason, degraded_at, ...}
+        self._degraded_modules: dict[str, dict] = {}
+        # 超长降级检测间隔（秒）
+        self._degraded_timeout = 300  # 5 分钟
+
+        # 健康事件持久化
+        self._health_events_dir = ""
+        self._health_events_path = ""
+        self._health_events_max = 10000  # 单文件最大条数
+        instance_dir = os.environ.get("KITE_INSTANCE_DIR", "")
+        if instance_dir:
+            self._health_events_dir = os.path.join(instance_dir, "health")
+            os.makedirs(self._health_events_dir, exist_ok=True)
+            self._health_events_path = os.path.join(self._health_events_dir, "events.jsonl")
+        # RPC 调用 Kernel 的回调（由 entry.py 设置）
+        self.rpc_call_kernel = None  # async callable(method, params) → response
+
     # ── Module discovery ──
 
     async def discover_modules(self):
@@ -121,41 +153,53 @@ class HealthMonitor:
             print(f"[watchdog] Launcher RPC failed: {e}")
             return
 
-        # Step 2: Get health endpoints from Registry via RPC
-        health_map = {}  # name -> {api_endpoint, health_endpoint}
+        # Step 2: Get health info from Registry via RPC
+        health_map = {}  # name -> {base_url, health_path}
         try:
-            resp = await self.rpc_call("registry.lookup", {"field": "health_endpoint"})
+            # 查 base_url：有 base_url 的模块才有 HTTP 服务
+            resp = await self.rpc_call("registry.lookup", {"field": "base_url"})
             result = resp.get("result", {})
             for entry in result.get("results", []):
                 mid = entry.get("module", "")
                 if mid in monitored:
                     health_map[mid] = {
-                        "api_endpoint": entry.get("api_endpoint", ""),
-                        "health_endpoint": entry.get("value", "/health"),
+                        "base_url": entry.get("value", ""),
+                        "health_path": "/health",
                     }
         except Exception:
             pass
 
+        try:
+            # 查 health_path：覆盖默认值
+            resp = await self.rpc_call("registry.lookup", {"field": "health_path"})
+            result = resp.get("result", {})
+            for entry in result.get("results", []):
+                mid = entry.get("module", "")
+                if mid in health_map:
+                    health_map[mid]["health_path"] = entry.get("value", "/health")
+        except Exception:
+            pass
+
         # Step 3: Sync module list
         seen = set()
         for mid, pid in monitored.items():
             seen.add(mid)
             h = health_map.get(mid, {})
-            ep = h.get("api_endpoint", "")
-            hp = h.get("health_endpoint", "/health")
+            ep = h.get("base_url", "")
+            hp = h.get("health_path", "/health")
             if mid not in self.modules:
                 self.modules[mid] = ModuleStatus(
                     module_id=mid,
-                    api_endpoint=ep,
-                    health_endpoint=hp,
+                    base_url=ep,
+                    health_path=hp,
                     pid=pid,
                 )
             else:
                 self.modules[mid].pid = pid
                 # Refresh endpoints once Registry has them
                 if ep:
-                    self.modules[mid].api_endpoint = ep
-                    self.modules[mid].health_endpoint = hp
+                    self.modules[mid].base_url = ep
+                    self.modules[mid].health_path = hp
 
         for mid in list(self.modules):
             if mid not in seen:
@@ -165,7 +209,7 @@ class HealthMonitor:
 
     async def _check_one(self, status: ModuleStatus):
         """Check a single module's health via RPC."""
-        if not status.api_endpoint:
+        if not status.base_url:
             return  # Not yet registered in Registry, will be picked up on next discover
 
         status.last_check = time.time()
@@ -217,9 +261,10 @@ class HealthMonitor:
             "error": status.last_error,
         })
 
-        # Restart if threshold reached
+        # Restart if threshold reached (but NOT if module is degraded — degraded ≠ crashed)
         if (status.fail_count >= self.FAIL_THRESHOLD
-                and status.restarted_count < self.MAX_RESTARTS):
+                and status.restarted_count < self.MAX_RESTARTS
+                and not self.is_degraded(status.module_id)):
             await self._restart_module(status)
 
     # ── Restart via Launcher API ──
@@ -238,6 +283,10 @@ class HealthMonitor:
                 status.restarted_count += 1
                 status.fail_count = 0
                 print(f"[watchdog] {mid} restart requested")
+                await self._persist_health_event("watchdog.module.restarted", {
+                    "module": mid, "restarted_count": status.restarted_count,
+                    "reason": "resource_critical" if status.resource_state == CRITICAL else "restart",
+                })
                 if status.restarted_count >= self.ALERT_AFTER_RESTARTS:
                     await self._publish("watchdog.alert", {
                         "module_id": mid,
@@ -251,10 +300,8 @@ class HealthMonitor:
 
     # ── Resource monitoring ──
 
-    def _collect_metrics(self, status: ModuleStatus) -> dict | None:
-        """Collect CPU/memory metrics for a module via psutil."""
-        if not status.pid:
-            return None
+    def _collect_metrics_sync(self, pid: int) -> dict | None:
+        """Synchronous metrics collection — runs in thread pool to avoid blocking event loop."""
         if not self._psutil:
             try:
                 import psutil
@@ -262,11 +309,11 @@ class HealthMonitor:
             except ImportError:
                 return None
         try:
-            proc = self._psutil.Process(status.pid)
+            proc = self._psutil.Process(pid)
             return {
                 "memory_rss": round(proc.memory_info().rss / 1024 / 1024, 1),
                 "memory_percent": round(proc.memory_percent(), 1),
-                "cpu_percent": round(proc.cpu_percent(interval=0.1), 1),
+                "cpu_percent": round(proc.cpu_percent(interval=0), 1),
                 "thread_count": proc.num_threads(),
             }
         except Exception:
@@ -274,7 +321,10 @@ class HealthMonitor:
 
     async def _check_resources(self, status: ModuleStatus):
         """Collect metrics and evaluate resource state."""
-        metrics = self._collect_metrics(status)
+        if not status.pid:
+            return
+        loop = asyncio.get_event_loop()
+        metrics = await loop.run_in_executor(None, self._collect_metrics_sync, status.pid)
         if not metrics:
             return
         status.last_metrics = metrics
@@ -331,6 +381,11 @@ class HealthMonitor:
                 await self._publish("watchdog.module.resource_warning", {
                     "module_id": status.module_id, "metrics": metrics,
                 })
+                # 资源告警 → 代发降级（level: slow）
+                await self._report_degraded_to_kernel(
+                    status.module_id, "slow", "resource_pressure",
+                    detail=f"CPU {metrics.get('cpu_percent', 0)}%, MEM {metrics.get('memory_percent', 0)}%"
+                )
         else:  # NORMAL
             status.resource_warning_count = 0
             status.resource_critical_count = 0
@@ -343,6 +398,11 @@ class HealthMonitor:
                     await self._publish("watchdog.module.resource_recovered", {
                         "module_id": status.module_id,
                     })
+                    # 资源恢复 → 代发恢复（仅当是 Watchdog 代发的降级时）
+                    if self.is_degraded(status.module_id):
+                        deg_info = self._degraded_modules.get(status.module_id, {})
+                        if deg_info.get("reason") == "resource_pressure":
+                            await self._report_recovered_to_kernel(status.module_id)
 
     async def _resource_restart(self, status: ModuleStatus):
         """Restart module due to resource critical timeout."""
@@ -383,7 +443,8 @@ class HealthMonitor:
         if event_type == "system.ready":
             print("[watchdog] Received system.ready")
             self._system_ready = True
-            self._system_ready_event.set()
+            await self._persist_health_event("system.ready", {})
+            # 不再自动启动监控，等待 Launcher 通过 RPC 调用 start_monitor
             return
 
         # module.offline — track launcher state
@@ -409,18 +470,27 @@ class HealthMonitor:
             return
 
         if not module_id or module_id == "watchdog":
+            # Handle registry.updated (no module_id)
+            if event_type == "registry.updated":
+                self._dirty = True  # 标记，等定时器到期时重新获取
             return
 
         if event_type == "module.started":
             print(f"[watchdog] Received module.started: {module_id}")
             self._crash_counts.pop(module_id, None)
-            # Don't await discover_modules() here - it blocks event processing!
-            # Schedule it as a background task instead
-            asyncio.create_task(self.discover_modules())
+            await self._persist_health_event("module.started", {"module": module_id})
+            self._dirty = True  # 标记，等定时器到期时获取新模块信息
 
         elif event_type == "module.stopped":
             print(f"[watchdog] Received module.stopped: {module_id}")
-            self.modules.pop(module_id, None)
+            self.modules.pop(module_id, None)  # 立即从监控列表移除
+            self._degraded_modules.pop(module_id, None)  # 清除降级状态
+            await self._persist_health_event("module.stopped", {
+                "module": module_id,
+                "exit_code": data.get("exit_code"),
+                "stop_type": data.get("stop_type"),
+            })
+            self._dirty = True  # 标记，等定时器到期时同步清单
             await self._handle_module_stopped(module_id, data)
 
         elif event_type == "module.exiting":
@@ -441,13 +511,40 @@ class HealthMonitor:
 
         elif event_type == "module.ready":
             graceful = bool(data.get("graceful_shutdown"))
-            print(f"[watchdog] Received module.ready: {module_id}, graceful_shutdown={graceful}")
+            startup_time = data.get("startup_time", 0)
+            print(f"[watchdog] Received module.ready: {module_id}, graceful_shutdown={graceful}, startup_time={startup_time:.3f}s")
             self._graceful_modules[module_id] = graceful
+            # Save startup time to module status
+            if module_id in self.modules:
+                self.modules[module_id].startup_time = startup_time
             # Reset launcher loss tracking when launcher reconnects
             if module_id == "launcher":
                 self._launcher_offline = False
                 self._launcher_had_exiting = False
 
+            # 记录模块 ready 时间和状态
+            self._module_ready_times[module_id] = time.time()
+            self._module_ever_ready[module_id] = True
+
+            # 取消 pending 的重启任务（如果有）
+            if module_id in self._restart_tasks:
+                task = self._restart_tasks.pop(module_id)
+                if not task.done():
+                    task.cancel()
+
+            # 检查是否稳定运行超过 STABLE_RUN_TIME，如果是则重置崩溃计数
+            # 注意：这里不立即重置，而是在下次崩溃时检查运行时长
+
+        elif event_type == "module.degraded":
+            await self._handle_module_degraded(module_id, data)
+
+        elif event_type == "module.recovered":
+            await self._handle_module_recovered(module_id, data)
+
+        elif event_type in ("system.instance.started", "system.instance.stopped"):
+            # 实例扩缩容事件：标记需要重新发现，下次检查周期自动更新监控列表
+            self.mark_dirty()
+
         # Layer 2: 忽略系统广播事件
         elif event_type in SYSTEM_BROADCAST_EVENTS:
             pass
@@ -460,15 +557,23 @@ class HealthMonitor:
         """Restart decision engine — called when module.stopped is received.
 
         Priority:
+        0. Monitor disabled → no restart (Launcher controls module lifecycle)
         1. System shutting down → no restart
         2. stop_type == "graceful_stop" → Launcher主动停止，不重启
-        3. stop_type == "process_exit" + has exit_intent → 按 intent 处理
-        4. stop_type == "process_exit" + no intent → 崩溃，重启
+        3. Launcher 在 stopped 事件中标记 restart=False + exit_type=graceful → 不重启（兜底）
+        4. stop_type == "process_exit" + has exit_intent → 按 intent 处理
+        5. stop_type == "process_exit" + no intent → 崩溃，重启
         """
         # Sync graceful_shutdown from Launcher (covers missed module.ready)
         if "graceful_shutdown" in data:
             self._graceful_modules[module_id] = bool(data["graceful_shutdown"])
 
+        # 监控未开启时，不做任何重启决策（启动阶段由 Launcher 管理）
+        if not self._monitor_enabled:
+            print(f"[watchdog] {module_id} stopped, but monitoring is disabled — skipping restart")
+            self._exit_intents.pop(module_id, None)
+            return
+
         if self._system_shutting_down:
             print(f"[watchdog] {module_id} stopped during shutdown, skipping restart")
             return
@@ -484,6 +589,13 @@ class HealthMonitor:
             self._exit_intents.pop(module_id, None)  # 清理可能残留的intent
             return
 
+        # 兜底：检查 Launcher 在 module.stopped 事件中的决策（防止 module.exiting 因竞态丢失）
+        if data.get("restart") is False and data.get("exit_type") == "graceful":
+            reason = data.get("reason", "unknown")
+            print(f"[watchdog] {module_id} exited gracefully (reason={reason}), no restart (Launcher decision)")
+            self._exit_intents.pop(module_id, None)
+            return
+
         # stop_type == "process_exit": 进程自行退出，需要判断是否崩溃
         intent = self._exit_intents.pop(module_id, None)
         if intent is not None:
@@ -505,19 +617,77 @@ class HealthMonitor:
                 return
 
         # No exit intent → treat as crash
-        self._crash_counts[module_id] = self._crash_counts.get(module_id, 0) + 1
-        count = self._crash_counts[module_id]
-        exit_code = data.get("exit_code", -1)
+        # 如果已有 pending 的重启任务，取消它（避免重复重启）
+        if module_id in self._restart_tasks:
+            existing_task = self._restart_tasks[module_id]
+            if not existing_task.done():
+                print(f"[watchdog] {module_id} already has a pending restart, skipping duplicate")
+                return
+
+        # 检查模块是否曾经 ready 过
+        ever_ready = self._module_ever_ready.get(module_id, False)
 
-        if count <= self.MAX_RESTARTS:
-            print(f"[watchdog] {module_id} crashed (exit_code={exit_code}), restarting ({count}/{self.MAX_RESTARTS})")
-            await self._restart_module_by_id(module_id, reason="crash")
+        if ever_ready:
+            # 运行时崩溃：检查是否稳定运行超过 STABLE_RUN_TIME
+            last_ready_time = self._module_ready_times.get(module_id, 0)
+            run_time = time.time() - last_ready_time
+
+            if run_time >= self.STABLE_RUN_TIME:
+                # 稳定运行超过 10 分钟，重置崩溃计数
+                print(f"[watchdog] {module_id} ran stably for {run_time:.0f}s, resetting crash count")
+                self._crash_counts[module_id] = 0
+
+            # 运行时崩溃：无限重试，使用指数退避
+            self._crash_counts[module_id] = self._crash_counts.get(module_id, 0) + 1
+            count = self._crash_counts[module_id]
+            exit_code = data.get("exit_code", -1)
+
+            # 指数退避：1秒、2秒、4秒、8秒...最大10分钟
+            delay = min(2 ** (count - 1), self.MAX_RESTART_DELAY)
+            print(f"\033[94m[watchdog] {module_id} crashed (exit_code={exit_code}), will restart in {delay}s (crash #{count}, runtime crash)\033[0m")
+
+            # 创建延迟重启任务
+            restart_task = asyncio.create_task(self._delayed_restart(module_id, delay, "runtime_crash", count))
+            self._restart_tasks[module_id] = restart_task
         else:
-            print(f"[watchdog] {module_id} crashed {count} times, giving up")
-            await self._publish("watchdog.alert", {
-                "module_id": module_id,
-                "message": f"{module_id} exceeded {self.MAX_RESTARTS} crash restarts",
-            })
+            # 启动失败（未到 ready）：最多重试 5 次
+            self._crash_counts[module_id] = self._crash_counts.get(module_id, 0) + 1
+            count = self._crash_counts[module_id]
+            exit_code = data.get("exit_code", -1)
+
+            if count <= self.MAX_STARTUP_RETRIES:
+                # 指数退避：1秒、2秒、4秒、8秒、16秒
+                delay = 2 ** (count - 1)
+                print(f"\033[94m[watchdog] {module_id} failed to start (exit_code={exit_code}), will retry in {delay}s ({count}/{self.MAX_STARTUP_RETRIES})\033[0m")
+
+                # 创建延迟重启任务
+                restart_task = asyncio.create_task(self._delayed_restart(module_id, delay, "startup_failure", count))
+                self._restart_tasks[module_id] = restart_task
+            else:
+                print(f"\033[94m[watchdog] {module_id} failed to start {count} times, giving up\033[0m")
+                await self._publish("watchdog.alert", {
+                    "module_id": module_id,
+                    "message": f"{module_id} failed to start after {self.MAX_STARTUP_RETRIES} retries",
+                })
+
+    async def _delayed_restart(self, module_id: str, delay: float, crash_type: str, attempt: int):
+        """延迟重启模块（指数退避）
+
+        Args:
+            module_id: 模块名
+            delay: 延迟时间（秒）
+            crash_type: 崩溃类型 (startup_failure, runtime_crash)
+            attempt: 重启尝试次数
+        """
+        try:
+            await asyncio.sleep(delay)
+            # 构造详细的 reason，包含崩溃类型和尝试次数
+            reason = f"watchdog_{crash_type}_attempt_{attempt}"
+            await self._restart_module_by_id(module_id, reason=reason)
+        except asyncio.CancelledError:
+            print(f"[watchdog] {module_id} restart cancelled")
+        finally:
+            self._restart_tasks.pop(module_id, None)
 
 
     async def _quick_check_launcher_exit(self):
@@ -532,13 +702,16 @@ class HealthMonitor:
                 print(f"[watchdog] Launcher 已退出（检测 {(i+1)*0.2:.1f}s），启动新实例")
                 self._start_new_instance()
                 print("[watchdog] 新实例已启动，watchdog 退出")
-                sys.exit(0)
+                if self.request_exit:
+                    self.request_exit(0)
+                return
 
         # 5s 后仍未退出，强制重启
         print("[watchdog] Launcher 5s 内未退出，强制重启")
         self._start_new_instance()
         print("[watchdog] 新实例已启动，watchdog 退出")
-        sys.exit(0)
+        if self.request_exit:
+            self.request_exit(0)
 
     async def _handle_launcher_lost(self):
         """Handle launcher_lost: decide whether to start a new Kite instance.
@@ -548,12 +721,12 @@ class HealthMonitor:
         """
         if self._launcher_had_exiting:
             print("[watchdog] Launcher had sent module.exiting before loss → normal exit, following suit")
-            sys.exit(0)
         else:
             print("[watchdog] Launcher lost without module.exiting → crash detected, starting new instance")
             self._start_new_instance()
             print("[watchdog] New instance started, exiting")
-            sys.exit(0)
+        if self.request_exit:
+            self.request_exit(0)
 
     def _start_new_instance(self):
         """Start a new Kite instance using saved startup info from launcher."""
@@ -659,6 +832,150 @@ class HealthMonitor:
         except Exception as e:
             print(f"[watchdog] Failed to start new instance: {e}")
 
+    # ── 降级状态管理 ──
+
+    async def _handle_module_degraded(self, module_id: str, data: dict):
+        """处理 module.degraded 事件：记录降级状态，降级模块不触发重启。"""
+        level = data.get("level", "unknown")
+        reason = data.get("reason", "unknown")
+        print(f"[watchdog] Module degraded: {module_id} level={level} reason={reason}")
+        self._degraded_modules[module_id] = {
+            "level": level,
+            "reason": reason,
+            "affected": data.get("affected", []),
+            "degraded_at": time.time(),
+        }
+        # 持久化健康事件
+        await self._persist_health_event("module.degraded", {
+            "module": module_id, "level": level, "reason": reason,
+            "detail": data.get("detail"),
+        })
+
+    async def _handle_module_recovered(self, module_id: str, data: dict):
+        """处理 module.recovered 事件：清除降级状态。"""
+        prev = self._degraded_modules.pop(module_id, None)
+        duration_ms = data.get("duration_ms")
+        print(f"[watchdog] Module recovered: {module_id} (duration={duration_ms}ms)")
+        await self._persist_health_event("module.recovered", {
+            "module": module_id,
+            "previous_level": data.get("previous_level"),
+            "duration_ms": duration_ms,
+        })
+
+    def is_degraded(self, module_id: str) -> bool:
+        """检查模块是否处于降级状态。"""
+        return module_id in self._degraded_modules
+
+    async def _report_degraded_to_kernel(self, module_id: str, level: str, reason: str, detail: str = None):
+        """通过 Kernel RPC 代发降级事件（资源类降级）。"""
+        if not self.rpc_call_kernel:
+            return
+        try:
+            params = {
+                "module_id": module_id,
+                "level": level,
+                "reason": reason,
+                "affected": [],
+            }
+            if detail:
+                params["detail"] = detail
+            await self.rpc_call_kernel("kernel.report_degraded", params)
+        except Exception as e:
+            print(f"[watchdog] Failed to report degraded for {module_id}: {e}")
+
+    async def _report_recovered_to_kernel(self, module_id: str):
+        """通过 Kernel RPC 代发恢复事件（资源类恢复）。"""
+        if not self.rpc_call_kernel:
+            return
+        try:
+            await self.rpc_call_kernel("kernel.report_recovered", {"module_id": module_id})
+        except Exception as e:
+            print(f"[watchdog] Failed to report recovered for {module_id}: {e}")
+
+    async def _check_degraded_timeout(self):
+        """检查超长降级（超过 5 分钟），发布告警事件。"""
+        now = time.time()
+        for mid, info in list(self._degraded_modules.items()):
+            degraded_at = info.get("degraded_at", 0)
+            if degraded_at and (now - degraded_at) >= self._degraded_timeout:
+                duration = int((now - degraded_at) * 1000)
+                # 只告警一次：标记已告警
+                if not info.get("_timeout_alerted"):
+                    info["_timeout_alerted"] = True
+                    print(f"[watchdog] Module {mid} degraded for {duration}ms (>{self._degraded_timeout}s), alerting")
+                    await self._publish("watchdog.module.degraded_timeout", {
+                        "module_id": mid,
+                        "level": info.get("level"),
+                        "reason": info.get("reason"),
+                        "duration_ms": duration,
+                    })
+                    await self._persist_health_event("watchdog.module.degraded_timeout", {
+                        "module": mid, "duration_ms": duration,
+                    })
+
+    # ── 健康事件持久化 ──
+
+    async def _persist_health_event(self, event: str, data: dict):
+        """将健康事件追加到 events.jsonl，按天归档。"""
+        if not self._health_events_dir:
+            return
+        record = {
+            "ts": datetime.now(timezone.utc).isoformat(),
+            "event": event,
+            **data,
+        }
+        line = json.dumps(record, ensure_ascii=False) + "\n"
+        try:
+            # 写入当前文件
+            with open(self._health_events_path, "a", encoding="utf-8") as f:
+                f.write(line)
+            # 写入按天归档
+            today = datetime.now().strftime("%Y-%m-%d")
+            month_dir = os.path.join(self._health_events_dir, today[:7])
+            os.makedirs(month_dir, exist_ok=True)
+            daily_path = os.path.join(month_dir, f"{today}.jsonl")
+            with open(daily_path, "a", encoding="utf-8") as f:
+                f.write(line)
+        except Exception as e:
+            print(f"[watchdog] Failed to persist health event: {e}")
+
+    def get_health_events(self, limit: int = 50, since: str = None) -> dict:
+        """查询健康事件记录。
+
+        Args:
+            limit: 最多返回条数（默认 50，最大 500）
+            since: ISO 8601 时间戳，只返回此时间之后的事件
+
+        Returns:
+            {"events": [...], "total": int, "has_more": bool}
+        """
+        limit = min(max(1, limit), 500)
+        if not self._health_events_path or not os.path.exists(self._health_events_path):
+            return {"events": [], "total": 0, "has_more": False}
+
+        events = []
+        try:
+            with open(self._health_events_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        record = json.loads(line)
+                        if since and record.get("ts", "") <= since:
+                            continue
+                        events.append(record)
+                    except json.JSONDecodeError:
+                        continue
+        except Exception:
+            pass
+
+        total = len(events)
+        # 返回最新的 limit 条（倒序）
+        events = events[-limit:]
+        events.reverse()
+        return {"events": events, "total": total, "has_more": total > limit}
+
     async def _restart_module_by_id(self, module_id: str, reason: str = "restart"):
         """Restart a module via Launcher RPC by module_id."""
         print(f"[watchdog] Requesting restart for {module_id} (reason={reason})")
@@ -675,16 +992,51 @@ class HealthMonitor:
         except Exception as e:
             print(f"[watchdog] {module_id} restart error: {e}")
 
+    # ── Monitor control (called by Launcher via RPC) ──
+
+    def start_monitor(self) -> dict:
+        """由 Launcher 调用，开启监控（模块重启职责移交给 Watchdog）"""
+        if self._monitor_enabled:
+            return {"status": "already_enabled"}
+        self._monitor_enabled = True
+        # 同时设置 system.ready，唤醒健康检查循环
+        self._system_ready = True
+        self._system_ready_event.set()
+        print("[watchdog] 监控已开启（由 Launcher 控制）")
+        return {"status": "enabled"}
+
+    def stop_monitor(self) -> dict:
+        """由 Launcher 调用，关闭监控（模块重启职责收回 Launcher）"""
+        if not self._monitor_enabled:
+            return {"status": "already_disabled"}
+        self._monitor_enabled = False
+        # 取消所有 pending 的重启任务
+        for module_id, task in list(self._restart_tasks.items()):
+            if not task.done():
+                task.cancel()
+                print(f"[watchdog] 取消 {module_id} 的 pending 重启任务")
+        self._restart_tasks.clear()
+        print("[watchdog] 监控已关闭（由 Launcher 控制）")
+        return {"status": "disabled"}
+
     # ── Main loop ──
 
     async def run(self):
-        """Main monitoring loop with dynamic intervals per resource state.
-        Waits for system.ready before starting health checks."""
+        """Main monitoring loop with grouped timers per resource state.
+
+        机制：
+        - system.ready + start_monitor 后等 15 秒再开始首次检查
+        - 首次检查时查注册中心获取健康接口清单
+        - 3 个状态组（NORMAL/WARNING/CRITICAL）各自独立周期
+        - 单循环维护 3 个 next_check 时间戳，每次 sleep 到最近到期的组
+        - 模块状态变化时自动迁移到对应组
+        - dirty 标记：事件驱动标记 + 定时器驱动获取，防止抖动
+        """
         self._running = True
-        print("[watchdog] Monitor started, waiting for system.ready...")
+        print("[watchdog] Monitor started, waiting for Launcher to enable monitoring...")
 
-        # Wait for system.ready (or manual start via API later)
-        while self._running and not self._system_ready:
+        # Wait for Launcher to call start_monitor() via RPC
+        while self._running and not self._monitor_enabled:
             try:
                 await asyncio.wait_for(self._system_ready_event.wait(), timeout=1.0)
             except asyncio.TimeoutError:
@@ -692,28 +1044,82 @@ class HealthMonitor:
 
         if not self._running:
             return
-        print("[watchdog] system.ready received, starting health checks")
+        print("[watchdog] Monitoring enabled, first check in 15s...")
+
+        # 等 15 秒，给模块充分时间完成注册
+        for _ in range(15):
+            if not self._running:
+                return
+            await asyncio.sleep(1)
+
+        # 首次发现
+        await self.discover_modules()
+        self._first_check_done = True
+        self._dirty = False
+        print(f"[watchdog] Initial discovery done, monitoring {len(self.modules)} modules")
+
+        # 分组定时器：3 个 next_check 时间戳
+        now = asyncio.get_event_loop().time()
+        next_check = {
+            NORMAL: now,     # 立即开始首次检查
+            WARNING: now,
+            CRITICAL: now,
+        }
+        # 上次 discover 时间（定期兜底重发现，5 分钟）
+        last_discovery = now
+        discovery_interval = 300.0
 
         while self._running:
-            # Re-discover every cycle to pick up newly started/stopped modules
-            await self.discover_modules()
-
-            if self.modules:
-                tasks = []
-                for s in self.modules.values():
-                    tasks.append(self._check_one(s))
-                    tasks.append(self._check_resources(s))
-                await asyncio.gather(*tasks, return_exceptions=True)
-
-            # Sleep = shortest interval needed by any module
-            interval = self._min_interval()
-            await asyncio.sleep(interval)
-
-    def _min_interval(self) -> float:
-        """Return the shortest check interval needed across all modules."""
-        if not self.modules:
-            return self.INTERVALS[NORMAL]
-        return min(self.INTERVALS.get(s.resource_state, 15) for s in self.modules.values())
+            now = asyncio.get_event_loop().time()
+
+            # 找到最近到期的时间戳
+            nearest = min(next_check.values())
+            wait = max(0, nearest - now)
+            if wait > 0:
+                await asyncio.sleep(wait)
+                if not self._running:
+                    return
+                now = asyncio.get_event_loop().time()
+
+            # dirty 检查：任意定时器到期时，如果有 dirty 标记就重新获取清单
+            if self._dirty:
+                await self.discover_modules()
+                self._dirty = False
+                last_discovery = now
+
+            # 定期兜底重发现（5 分钟）
+            if now - last_discovery >= discovery_interval:
+                await self.discover_modules()
+                self._dirty = False
+                last_discovery = now
+
+            # 按模块当前 resource_state 分组
+            groups: dict[str, list[ModuleStatus]] = {NORMAL: [], WARNING: [], CRITICAL: []}
+            for s in self.modules.values():
+                groups.get(s.resource_state, groups[NORMAL]).append(s)
+
+            # 检查到期的组
+            for level in (CRITICAL, WARNING, NORMAL):
+                if now >= next_check[level] and groups[level]:
+                    tasks = []
+                    for s in groups[level]:
+                        tasks.append(self._check_one(s))
+                        tasks.append(self._check_resources(s))
+                    await asyncio.gather(*tasks, return_exceptions=True)
+                    # 更新该组的下次检查时间
+                    next_check[level] = now + self.INTERVALS[level]
+
+            # 没有模块的组也要推进时间戳，防止空组一直触发
+            for level in (CRITICAL, WARNING, NORMAL):
+                if not groups[level] and now >= next_check[level]:
+                    next_check[level] = now + self.INTERVALS[level]
+
+            # 超长降级检测
+            await self._check_degraded_timeout()
+
+    def mark_dirty(self):
+        """外部标记需要重新获取模块清单（如 module.offline 事件）。"""
+        self._dirty = True
 
     def stop(self):
         self._running = False
@@ -723,7 +1129,8 @@ class HealthMonitor:
     def get_status(self) -> dict:
         """Return current health and resource status of all monitored modules."""
         return {
-            mid: {
+            "_monitor_enabled": self._monitor_enabled,
+            **{mid: {
                 "state": s.state,
                 "fail_count": s.fail_count,
                 "restarted_count": s.restarted_count,
@@ -732,6 +1139,7 @@ class HealthMonitor:
                 "last_error": s.last_error,
                 "resource_state": s.resource_state,
                 "metrics": s.last_metrics,
+                "startup_time": s.startup_time,
             }
             for mid, s in self.modules.items()
-        }
+        }}