@agentunion/kite 1.5.0 → 1.6.1

package/launcher/process_manager.py

@@ -8,6 +8,7 @@ stdio lifecycle:
 - Structured messages: JSON lines with "kite" field, dispatched via callback
 """
 
+import collections
 import glob
 import json
 import os
@@ -16,6 +17,7 @@ import subprocess
 import sys
 import threading
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, asdict
 from typing import Callable
 
@@ -34,21 +36,48 @@ class ProcessRecord:
     cmd: list
     module_dir: str
     started_at: float
+    instance_num: int = 1
 
 
 class ProcessManager:
     """Manage child processes for all Kite modules."""
 
+    @staticmethod
+    def _instance_key(name: str, instance_num: int = 1) -> str:
+        """Build internal dict key: 'name' for instance 1, 'name#N' for N>1."""
+        return name if instance_num <= 1 else f"{name}#{instance_num}"
+
+    @staticmethod
+    def parse_instance_key(key: str) -> tuple[str, int]:
+        """Parse internal key back to (module_name, instance_num)."""
+        if "#" in key:
+            name, num_str = key.rsplit("#", 1)
+            try:
+                return name, int(num_str)
+            except ValueError:
+                pass
+        return key, 1
+
+    LOG_BUFFER_SIZE = 1000  # lines per module
+
     def __init__(self, kite_token: str, instance_id: str = "",
-                 on_kite_message: Callable[[str, dict], None] | None = None):
+                 on_kite_message: Callable[[str, dict], None] | None = None,
+                 on_log_line: Callable[[str, float, str], None] | None = None):
         """
         Args:
             on_kite_message: callback(module_name, msg_dict) for structured stdout messages.
                              Called from stdout reader thread — must be thread-safe.
+            on_log_line: callback(module_name, timestamp, line) for every log line.
+                         Called from stdout reader thread — must be thread-safe.
         """
         self.kite_token = kite_token
         self.instance_id = instance_id or str(os.getpid())
         self._on_kite_message = on_kite_message
+        self._on_log_line = on_log_line
+        # Per-module ring buffers: name -> deque of (seq, ts, line)
+        self._log_buffers: dict[str, collections.deque] = {}
+        self._log_seq: dict[str, int] = {}  # name -> next seq number
+        self._log_lock = threading.Lock()
         # Use KITE_MODULE_DATA (set by Launcher for itself)
         launcher_dir = os.environ.get("KITE_MODULE_DATA") or os.path.join(os.environ["KITE_INSTANCE_DIR"], "launcher")
         self.data_dir = os.path.join(launcher_dir, "state")
@@ -249,10 +278,24 @@ class ProcessManager:
         """
         my_pid = os.getpid()
         inst_name = os.path.basename(os.environ.get("KITE_INSTANCE_DIR", "")) or "unknown"
-        killed = 0
+
+        # Collect all processes*.json files
         pattern = os.path.join(self.data_dir, "processes*.json")
-        for filepath in glob.glob(pattern):
-            killed += self._cleanup_one_leftover_file(filepath, my_pid)
+        files_to_cleanup = list(glob.glob(pattern))
+
+        if not files_to_cleanup:
+            return {}
+
+        # Parallel cleanup: each file runs its own 3-phase cleanup independently
+        killed = 0
+        with ThreadPoolExecutor(max_workers=len(files_to_cleanup)) as executor:
+            futures = [executor.submit(self._cleanup_one_leftover_file, filepath, my_pid) for filepath in files_to_cleanup]
+            for future in as_completed(futures):
+                try:
+                    killed += future.result()
+                except Exception as e:
+                    print(f"[launcher] 清理残留文件时出错: {e}")
+
         return {inst_name: killed} if killed else {}
 
     def cleanup_global_leftovers(self) -> dict[str, int]:
@@ -274,15 +317,35 @@ class ProcessManager:
         except OSError:
             return {}
 
+        # Collect all processes*.json files
+        files_to_cleanup = []
         for entry in entries:
             state_dir = os.path.join(workspace, entry, "launcher", "state")
             if not os.path.isdir(state_dir):
                 continue
             pattern = os.path.join(state_dir, "processes*.json")
             for filepath in glob.glob(pattern):
-                killed = self._cleanup_one_leftover_file(filepath, my_pid)
-                if killed:
-                    result[entry] = result.get(entry, 0) + killed
+                files_to_cleanup.append((filepath, entry))
+
+        # Parallel cleanup: each file runs its own 3-phase cleanup independently
+        if not files_to_cleanup:
+            return result
+
+        with ThreadPoolExecutor(max_workers=len(files_to_cleanup)) as executor:
+            futures = {}
+            for filepath, entry in files_to_cleanup:
+                future = executor.submit(self._cleanup_one_leftover_file, filepath, my_pid)
+                futures[future] = entry
+
+            for future in as_completed(futures):
+                entry = futures[future]
+                try:
+                    killed = future.result()
+                    if killed:
+                        result[entry] = result.get(entry, 0) + killed
+                except Exception as e:
+                    print(f"[launcher] 清理实例 {entry} 时出错: {e}")
+
         return result
 
     def _cleanup_one_leftover_file(self, filepath: str, my_pid: int) -> int:
@@ -290,7 +353,10 @@ class ProcessManager:
 
         - Skip if launcher_pid matches my_pid (our own records).
         - Skip if launcher_pid is still alive (parallel instance).
-        - Otherwise clean up child processes and delete the file.
+        - Otherwise clean up child processes in 3 phases:
+          1. Kill all watchdog processes (serial)
+          2. Kill all middle-tier processes (parallel)
+          3. Kill all kernel processes (serial)
         Returns number of processes killed.
         """
         try:
@@ -327,30 +393,87 @@ class ProcessManager:
             return 0
 
         # Dead launcher (or old format) — clean up its child processes
-        # Sort: watchdog first, kernel last, others in middle (prevents cascading issues)
-        def _cleanup_sort_key(entry):
-            name = entry.get("name", "")
-            if name == "watchdog":
-                return (0, name)
-            if name == "kernel":
-                return (2, name)
-            return (1, name)
+        # New approach: All processes start cleanup in parallel with dependency checks
+        all_procs = records
+        if not all_procs:
+            try:
+                os.remove(filepath)
+            except OSError:
+                pass
+            return 0
 
-        records_sorted = sorted(records, key=_cleanup_sort_key)
+        # Shared state for dependency tracking (instance-local)
+        killed_names = set()  # Track which process names have been killed
+        killed_lock = threading.Lock()
+        killed_condition = threading.Condition(killed_lock)  # Notify when any process is killed
 
-        killed = 0
-        for entry in records_sorted:
+        def cleanup_with_deps(entry: dict) -> bool:
+            """Cleanup a single process with dependency checking."""
+            name = entry.get("name", "")
             pid = entry.get("pid", 0)
             cmd = entry.get("cmd", [])
-            name = entry.get("name", "?")
+
             if not pid:
-                continue
-            if self._is_pid_alive(pid) and self._cmd_matches(pid, cmd):
-                print(f"[launcher] 正在清理残留进程: {name} (PID {pid}) [{os.path.basename(filepath)}]")
-                self._force_kill(pid)
-                killed += 1
-            else:
+                return False
+
+            # Step 1: Verify process is alive
+            if not self._is_pid_alive(pid):
                 print(f"[launcher] 残留进程 {name} (PID {pid}) 已不存在")
+                with killed_condition:
+                    killed_names.add(name)
+                    killed_condition.notify_all()  # Notify all waiters
+                return False
+
+            # Step 2: Verify command line (防止 PID 复用误杀)
+            if not self._cmd_matches(pid, cmd):
+                print(f"[launcher] 残留进程 {name} (PID {pid}) 命令行不匹配，跳过")
+                with killed_condition:
+                    killed_names.add(name)  # Mark as "handled" to unblock dependents
+                    killed_condition.notify_all()
+                return False
+
+            # Step 3: Wait for dependencies (instance-local, event-driven)
+            dependencies = self._get_process_dependencies(name, all_procs)
+            if dependencies:
+                with killed_condition:
+                    while True:
+                        # Check if all dependencies are satisfied
+                        remaining = dependencies - killed_names
+                        if not remaining:
+                            break
+                        # Wait for notification (releases lock, reacquires on wake)
+                        killed_condition.wait()
+
+            # Step 4: Kill the process
+            print(f"[launcher] 正在清理残留进程: {name} (PID {pid}) [{os.path.basename(filepath)}]")
+            self._force_kill(pid)
+
+            # Step 5: Verify process is dead (wait up to 100ms)
+            for i in range(10):
+                if not self._is_pid_alive(pid):
+                    break
+                import time
+                time.sleep(0.01)
+            else:
+                print(f"[launcher] 警告: 进程 {name} (PID {pid}) 在 100ms 后仍未退出")
+
+            # Step 6: Mark as killed and notify waiters
+            with killed_condition:
+                killed_names.add(name)
+                killed_condition.notify_all()  # Wake up all waiting threads
+
+            return True
+
+        # Launch all cleanup tasks in parallel
+        killed = 0
+        with ThreadPoolExecutor(max_workers=len(all_procs)) as executor:
+            futures = [executor.submit(cleanup_with_deps, entry) for entry in all_procs]
+            for future in as_completed(futures):
+                try:
+                    if future.result():
+                        killed += 1
+                except Exception as e:
+                    print(f"[launcher] 并行清理进程时出错: {e}")
 
         try:
             os.remove(filepath)
@@ -358,6 +481,61 @@ class ProcessManager:
             pass
         return killed
 
+    def _get_process_dependencies(self, name: str, all_procs: list) -> set:
+        """Get the set of process names that must be killed before this one.
+
+        Dependency rules (instance-local):
+        - watchdog: no dependencies
+        - kernel: depends on all other processes in this instance
+        - others: depend on watchdog (if exists)
+        """
+        if name == "watchdog":
+            return set()
+        elif name == "kernel":
+            # Kernel depends on all other processes in this instance
+            return {p.get("name", "") for p in all_procs if p.get("name") != "kernel"}
+        else:
+            # Middle-tier processes depend on watchdog (if exists in this instance)
+            watchdog_exists = any(p.get("name") == "watchdog" for p in all_procs)
+            return {"watchdog"} if watchdog_exists else set()
+
+    def _kill_one_process(self, entry: dict, filepath: str, skip_cmd_check: bool = False) -> bool:
+        """Kill a single process. Returns True if killed, False otherwise.
+
+        Args:
+            entry: Process record dict with pid, cmd, name
+            filepath: Path to processes*.json (for logging)
+            skip_cmd_check: If True, skip command line verification (faster, for leftover cleanup)
+        """
+        import time
+        pid = entry.get("pid", 0)
+        cmd = entry.get("cmd", [])
+        name = entry.get("name", "?")
+        if not pid:
+            return False
+
+        # Check if process is alive
+        if not self._is_pid_alive(pid):
+            print(f"[launcher] 残留进程 {name} (PID {pid}) 已不存在")
+            return False
+
+        # Verify command line (unless skipped for performance)
+        if not skip_cmd_check and not self._cmd_matches(pid, cmd):
+            print(f"[launcher] 残留进程 {name} (PID {pid}) 命令行不匹配，跳过")
+            return False
+
+        print(f"[launcher] 正在清理残留进程: {name} (PID {pid}) [{os.path.basename(filepath)}]")
+        self._force_kill(pid)
+
+        # Verify the process is actually dead (wait up to 100ms)
+        for i in range(10):
+            if not self._is_pid_alive(pid):
+                return True
+            time.sleep(0.01)
+
+        print(f"[launcher] 警告: 进程 {name} (PID {pid}) 在 100ms 后仍未退出")
+        return True
+
     def _is_pid_alive(self, pid: int) -> bool:
         """Check if a process with given PID exists.
 
@@ -604,8 +782,18 @@ class ProcessManager:
         """Force kill a process by PID."""
         try:
             if IS_WINDOWS:
-                subprocess.run(["taskkill", "/F", "/PID", str(pid)],
-                               capture_output=True, timeout=5)
+                # Use TerminateProcess API directly (faster than taskkill)
+                PROCESS_TERMINATE = 0x0001
+                handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, pid)
+                if handle:
+                    try:
+                        ctypes.windll.kernel32.TerminateProcess(handle, 1)
+                    finally:
+                        ctypes.windll.kernel32.CloseHandle(handle)
+                else:
+                    # Fallback to taskkill if OpenProcess fails
+                    subprocess.run(["taskkill", "/F", "/PID", str(pid)],
+                                   capture_output=True, timeout=5)
             else:
                 import signal
                 os.kill(pid, signal.SIGKILL)
@@ -614,29 +802,37 @@ class ProcessManager:
 
     # ── Start module ──
 
-    def start_module(self, info, boot_info: dict = None) -> bool:
+    def start_module(self, info, boot_info: dict = None, instance_num: int = 1) -> bool:
         """Start a module as a subprocess. Returns True on success.
         If boot_info is provided, it is written as JSON to the child's stdin pipe.
         stdin is kept open after boot_info for Launcher to send additional messages.
+
+        Args:
+            instance_num: Instance number (1=primary, >1=extra instance).
         """
         from .module_scanner import ModuleInfo
         info: ModuleInfo
 
-        if info.name in self._processes:
-            proc = self._processes[info.name]
+        key = self._instance_key(info.name, instance_num)
+        label = info.name if instance_num <= 1 else f"{info.name}#{instance_num}"
+
+        if key in self._processes:
+            proc = self._processes[key]
             if proc.poll() is None:
-                print(f"[launcher] {info.name} 已在运行 (PID {proc.pid})")
+                print(f"[launcher] {label} 已在运行 (PID {proc.pid})")
                 return True
 
         cmd = self._build_cmd(info)
         if not cmd:
-            print(f"[launcher] 错误: 无法构建 {info.name} 的启动命令")
+            print(f"[launcher] 错误: 无法构建 {label} 的启动命令")
             return False
 
         env = os.environ.copy()
         env["PYTHONUTF8"] = "1"           # Force UTF-8 in child Python processes
         env["PYTHONUNBUFFERED"] = "1"     # Disable output buffering for real-time logs
         env["KITE_MODULE_DATA"] = os.path.join(os.environ["KITE_INSTANCE_DIR"], info.name)
+        if instance_num > 1:
+            env["KITE_INSTANCE_NUM"] = str(instance_num)
         if info.launch.env:
             env.update(info.launch.env)
 
@@ -644,8 +840,6 @@ class ProcessManager:
         if IS_WINDOWS:
             creation_flags = subprocess.CREATE_NO_WINDOW
 
-        use_stdin = boot_info is not None and info.launch.boot_stdin
-
         cwd = info.module_dir
         if info.launch.cwd:
             cwd = os.path.normpath(os.path.join(info.module_dir, info.launch.cwd))
@@ -655,48 +849,50 @@ class ProcessManager:
                 cmd,
                 cwd=cwd,
                 env=env,
-                stdin=subprocess.PIPE if use_stdin else None,
+                stdin=subprocess.PIPE,  # stdin 管道始终创建，解耦 boot_info
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 creationflags=creation_flags,
             )
         except Exception as e:
-            print(f"[launcher] 错误: 启动 {info.name} 失败: {e}")
+            print(f"[launcher] 错误: 启动 {label} 失败: {e}")
             return False
 
         # Write boot_info to stdin — keep stdin open for additional messages
-        if use_stdin and proc.stdin:
+        if boot_info is not None and info.launch.boot_stdin and proc.stdin:
             try:
                 proc.stdin.write(json.dumps(boot_info).encode() + b"\n")
                 proc.stdin.flush()
             except Exception as e:
-                print(f"[launcher] 警告: 向 {info.name} 写入 boot_info 失败: {e}")
+                print(f"[launcher] 警告: 向 {label} 写入 boot_info 失败: {e}")
 
-        self._processes[info.name] = proc
-        self._records[info.name] = ProcessRecord(
+        self._processes[key] = proc
+        self._records[key] = ProcessRecord(
             name=info.name,
             pid=proc.pid,
             cmd=cmd,
             module_dir=info.module_dir,
             started_at=time.time(),
+            instance_num=instance_num,
         )
 
         # Daemon thread to read stdout: structured messages + log lines
         t = threading.Thread(
-            target=self._read_stdout, args=(info.name, proc),
+            target=self._read_stdout, args=(label, proc),
             daemon=True,
         )
         t.start()
 
-        print(f"[launcher] 已启动 {info.name} (PID {proc.pid})")
+        print(f"[launcher] 已启动 {label} (PID {proc.pid})")
         return True
 
-    def write_stdin(self, name: str, data: dict) -> bool:
+    def write_stdin(self, name: str, data: dict, instance_num: int = 1) -> bool:
         """Write a structured JSON message to a module's stdin.
         Used by Launcher to send additional messages after boot_info (e.g. launcher_ws_token).
         Returns True on success.
         """
-        proc = self._processes.get(name)
+        key = self._instance_key(name, instance_num)
+        proc = self._processes.get(key)
         if not proc or not proc.stdin or proc.stdin.closed:
             return False
         try:
@@ -707,13 +903,14 @@ class ProcessManager:
             print(f"[launcher] 警告: 向 {name} 写入 stdin 失败: {e}")
             return False
 
-    def close_stdio(self, name: str):
+    def close_stdio(self, name: str, instance_num: int = 1):
         """Close stdin for a module. Called by Launcher after module.ready → module.started.
         Only closes stdin (no more messages to send). stdout is left open so the reader
         thread can continue relaying logs — it exits naturally when the process dies.
         On Windows, closing stdout while a reader thread is blocked on readline() can deadlock.
         """
-        proc = self._processes.get(name)
+        key = self._instance_key(name, instance_num)
+        proc = self._processes.get(key)
         if not proc:
             return
         if proc.stdin and not proc.stdin.closed:
@@ -769,37 +966,94 @@ class ProcessManager:
                     print(text)
                 else:
                     print(f"[{name}] {text}")
+                # Note: terminal buffering is handled by _tprint hook in logging_setup,
+                # so the buffer contains the full timestamped+colored output line.
         except Exception:
             pass
 
+    def get_log_history(self, name: str, offset: int = 0, count: int = 200) -> dict:
+        """Return buffered log lines for a module.
+
+        Args:
+            name: module name (label)
+            offset: return lines with seq >= offset; -1 means last `count` lines
+            count: max lines to return
+        Returns:
+            {"lines": [{"seq", "ts", "text"}], "total": int, "module": name}
+        """
+        with self._log_lock:
+            buf = self._log_buffers.get(name)
+            if buf is None:
+                return {"lines": [], "total": 0, "module": name}
+            all_lines = list(buf)
+        if offset < 0:
+            selected = all_lines[-count:]
+        else:
+            selected = [e for e in all_lines if e[0] >= offset][:count]
+        return {
+            "lines": [{"seq": e[0], "ts": e[1], "text": e[2]} for e in selected],
+            "total": len(all_lines),
+            "module": name,
+        }
+
+    def list_log_modules(self) -> list[str]:
+        """Return list of modules that have log buffer entries."""
+        with self._log_lock:
+            return list(self._log_buffers.keys())
+
     # ── Stop module ──
 
-    def stop_module(self, name: str, timeout: float = 5) -> bool:
+    def stop_module(self, name: str, timeout: float = 5, instance_num: int = 1) -> bool:
         """Stop a module by name. Returns True if stopped successfully."""
-        proc = self._processes.get(name)
+        key = self._instance_key(name, instance_num)
+        label = name if instance_num <= 1 else f"{name}#{instance_num}"
+        proc = self._processes.get(key)
         if not proc or proc.poll() is not None:
-            self._processes.pop(name, None)
-            self._records.pop(name, None)
+            self._processes.pop(key, None)
+            self._records.pop(key, None)
             return True
 
         # Close stdio before terminating
-        self.close_stdio(name)
+        self.close_stdio(name, instance_num)
 
-        print(f"[launcher] 正在停止 {name} (PID {proc.pid})...")
+        print(f"[launcher] 正在停止 {label} (PID {proc.pid})...")
         proc.terminate()
         try:
             proc.wait(timeout=timeout)
         except subprocess.TimeoutExpired:
-            print(f"[launcher] {name} 在 {timeout}s 内未退出，强制终止")
+            print(f"[launcher] {label} 在 {timeout}s 内未退出，强制终止")
             proc.kill()
             try:
                 proc.wait(timeout=3)
             except subprocess.TimeoutExpired:
                 self._force_kill(proc.pid)
 
-        self._processes.pop(name, None)
-        self._records.pop(name, None)
-        print(f"[launcher] 已停止 {name}")
+        self._processes.pop(key, None)
+        self._records.pop(key, None)
+        print(f"[launcher] 已停止 {label}")
+        return True
+
+    def ensure_exited(self, name: str, timeout: float = 3, instance_num: int = 1) -> bool:
+        """确保进程已完全退出（用于崩溃后确认端口释放）。"""
+        key = self._instance_key(name, instance_num)
+        proc = self._processes.get(key)
+        if not proc:
+            return True
+        if proc.poll() is not None:
+            self._processes.pop(key, None)
+            self._records.pop(key, None)
+            return True
+        # 还活着，等待或强杀
+        try:
+            proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            try:
+                proc.wait(timeout=2)
+            except subprocess.TimeoutExpired:
+                self._force_kill(proc.pid)
+        self._processes.pop(key, None)
+        self._records.pop(key, None)
         return True
 
     def stop_all(self, timeout: float = 10):
@@ -808,7 +1062,7 @@ class ProcessManager:
             return
 
         # Filter to only still-running processes
-        alive = [n for n, p in self._processes.items() if p.poll() is None]
+        alive = [k for k, p in self._processes.items() if p.poll() is None]
         if not alive:
             # All already exited — just clean up bookkeeping
             self._processes.clear()
@@ -818,16 +1072,17 @@ class ProcessManager:
         print(f"[launcher] 正在停止所有模块: {', '.join(alive)}")
 
         # Phase 1: close stdio and send terminate to all
-        for name in alive:
-            self.close_stdio(name)
-            proc = self._processes.get(name)
+        for key in alive:
+            mod_name, inst_num = self.parse_instance_key(key)
+            self.close_stdio(mod_name, inst_num)
+            proc = self._processes.get(key)
             if proc and proc.poll() is None:
                 proc.terminate()
 
         # Phase 2: wait for all to exit
         deadline = time.time() + timeout
-        for name in alive:
-            proc = self._processes.get(name)
+        for key in alive:
+            proc = self._processes.get(key)
             if not proc:
                 continue
             remaining = max(0.1, deadline - time.time())
@@ -837,10 +1092,10 @@ class ProcessManager:
                 pass
 
         # Phase 3: force kill survivors
-        for name in alive:
-            proc = self._processes.get(name)
+        for key in alive:
+            proc = self._processes.get(key)
             if proc and proc.poll() is None:
-                print(f"[launcher] 强制终止 {name} (PID {proc.pid})")
+                print(f"[launcher] 强制终止 {key} (PID {proc.pid})")
                 proc.kill()
                 try:
                     proc.wait(timeout=3)
@@ -855,33 +1110,57 @@ class ProcessManager:
 
     def persist_records(self):
         """Write current process records to processes.json (with launcher_pid for multi-instance)."""
+        import time as _time
+        t0 = _time.monotonic()
         data = {
             "launcher_pid": os.getpid(),
             "debug": os.environ.get("KITE_DEBUG") == "1",
             "cwd": os.environ.get("KITE_CWD", ""),
             "records": [asdict(r) for r in self._records.values()],
         }
+        t1 = _time.monotonic()
         self._write_records_file(data)
+        t2 = _time.monotonic()
+        serialize_ms = (t1 - t0) * 1000
+        write_ms = (t2 - t1) * 1000
+        if serialize_ms + write_ms > 50:
+            print(f"[launcher] persist_records 耗时: serialize={serialize_ms:.0f}ms, write={write_ms:.0f}ms")
 
     def check_exited(self) -> list[tuple[str, int]]:
-        """Non-blocking check for exited child processes. Returns [(name, returncode)]."""
+        """Non-blocking check for exited child processes. Returns [(key, returncode)].
+        Key is 'name' for primary instances, 'name#N' for extra instances.
+        Use parse_instance_key() to extract (module_name, instance_num).
+        """
         exited = []
-        for name in list(self._processes.keys()):
-            proc = self._processes[name]
+        for key in list(self._processes.keys()):
+            proc = self._processes[key]
             rc = proc.poll()
             if rc is not None:
-                exited.append((name, rc))
-                self._processes.pop(name, None)
-                self._records.pop(name, None)
+                exited.append((key, rc))
+                self._processes.pop(key, None)
+                self._records.pop(key, None)
         return exited
 
-    def get_record(self, name: str) -> ProcessRecord | None:
-        return self._records.get(name)
+    def get_record(self, name: str, instance_num: int = 1) -> ProcessRecord | None:
+        key = self._instance_key(name, instance_num)
+        return self._records.get(key)
 
-    def is_running(self, name: str) -> bool:
-        proc = self._processes.get(name)
+    def is_running(self, name: str, instance_num: int = 1) -> bool:
+        key = self._instance_key(name, instance_num)
+        proc = self._processes.get(key)
         return proc is not None and proc.poll() is None
 
+    def get_module_instance_keys(self, name: str) -> list[str]:
+        """Return all active instance keys for a given module name."""
+        keys = []
+        for key in self._processes:
+            mod_name, _ = self.parse_instance_key(key)
+            if mod_name == name:
+                proc = self._processes[key]
+                if proc.poll() is None:
+                    keys.append(key)
+        return keys
+
     def _write_records_file(self, data):
         """Write data to the instance's processes JSON file."""
         try: