@team-agent/installer 0.2.2 → 0.2.4

package/src/team_agent/diagnose/orphan_cleanup.py

@@ -16,7 +16,6 @@ import signal
 import subprocess
 import time
 from datetime import datetime, timezone
-from pathlib import Path
 from typing import Any
 
 # Pattern: argv contains "team_agent.coordinator --workspace <path>" anywhere.
@@ -39,6 +38,7 @@ _EPHEMERAL_PATH_HINTS = (
     "team-agent-test-",
 )
 _SIGTERM_WAIT_SECONDS = 3.0
+_SIGKILL_WAIT_SECONDS = 2.0
 
 
 def find_coordinator_processes(*, runner=subprocess.run) -> list[dict[str, Any]]:
@@ -91,7 +91,7 @@ def classify_orphan(entry: dict[str, Any]) -> tuple[bool, str]:
     workspace = entry.get("workspace")
     if not workspace:
         return False, "cmdline_unparsed"
-    if not Path(workspace).exists():
+    if not os.path.exists(workspace):
         return True, "workspace_path_missing"
     for hint in _EPHEMERAL_PATH_HINTS:
         if hint in workspace:
@@ -104,12 +104,35 @@ def cleanup_orphan_coordinators(
     confirm: bool = False,
     runner=subprocess.run,
     killer=os.kill,
+    pg_killer=None,
+    pgid_getter=None,
     sleeper=time.sleep,
+    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
+    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
 ) -> dict[str, Any]:
     """Scan for orphan coordinators. Without confirm: dry-run (just classify and report).
-    With confirm: SIGTERM each orphan and wait up to _SIGTERM_WAIT_SECONDS for the
-    process to exit; report success/failure per pid."""
+    With confirm: SIGTERM each orphan, wait up to _SIGTERM_WAIT_SECONDS for graceful
+    exit; if still alive, escalate to SIGKILL and wait _SIGKILL_WAIT_SECONDS. Only
+    report status='failed' (with error='alive_after_sigkill') when the process
+    survives BOTH signals — that's extremely rare and almost always indicates a
+    zombie/uninterruptible-sleep kernel state.
+
+    Mac mini 2026-05-26 evidence: real orphan coordinators have been observed alive
+    40+ hours; many of them never exit on SIGTERM (signal handler suppressed during
+    long sqlite reads, or the python interpreter is hosting an async loop that
+    swallows the term signal). SIGKILL escalation is required for production.
+
+    pg_killer / pgid_getter default to os.killpg / os.getpgid; mock them in tests.
+    If pgid_getter succeeds AND returns a pgid > 1 AND the pgid != pid (i.e. the
+    process leads its own process group with children), we signal the WHOLE group;
+    otherwise we signal the pid directly. This catches orphan coordinators that
+    spawned subprocess.Popen children which would otherwise survive a pid-only
+    SIGTERM."""
     now = datetime.now(timezone.utc).isoformat()
+    if pg_killer is None:
+        pg_killer = getattr(os, "killpg", None)
+    if pgid_getter is None:
+        pgid_getter = getattr(os, "getpgid", None)
     entries = find_coordinator_processes(runner=runner)
     classified: list[dict[str, Any]] = []
     orphans: list[dict[str, Any]] = []
@@ -131,30 +154,19 @@ def cleanup_orphan_coordinators(
     killed: list[dict[str, Any]] = []
     failed: list[dict[str, Any]] = []
     for entry in orphans:
-        pid = entry["pid"]
-        try:
-            killer(pid, signal.SIGTERM)
-        except (ProcessLookupError, PermissionError, OSError) as exc:
-            failed.append({**entry, "error": str(exc)})
-            continue
-        # Wait briefly; if the process is still alive after _SIGTERM_WAIT_SECONDS,
-        # mark as failed (caller may want to SIGKILL).
-        deadline = time.monotonic() + _SIGTERM_WAIT_SECONDS
-        gone = False
-        while time.monotonic() < deadline:
-            try:
-                killer(pid, 0)
-            except ProcessLookupError:
-                gone = True
-                break
-            except (PermissionError, OSError):
-                gone = True
-                break
-            sleeper(0.1)
-        if gone:
-            killed.append(entry)
+        outcome = _terminate_orphan(
+            entry["pid"], killer=killer, pg_killer=pg_killer,
+            pgid_getter=pgid_getter, sleeper=sleeper,
+            sigterm_wait_seconds=sigterm_wait_seconds,
+            sigkill_wait_seconds=sigkill_wait_seconds,
+        )
+        annotated = {**entry, **outcome}
+        if outcome.get("status") == "killed":
+            killed.append(annotated)
+        elif outcome.get("status") == "missing":
+            killed.append(annotated)
         else:
-            failed.append({**entry, "error": "still_alive_after_sigterm"})
+            failed.append(annotated)
     return {
         "ok": True,
         "scanned": len(classified),
@@ -166,6 +178,162 @@ def cleanup_orphan_coordinators(
     }
 
 
+def _terminate_orphan(
+    pid: int,
+    *,
+    killer,
+    pg_killer,
+    pgid_getter,
+    sleeper,
+    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
+    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
+) -> dict[str, Any]:
+    """SIGTERM → wait 3s → SIGKILL → wait 2s escalation. Returns one of:
+      {status: 'killed', sigkill_required: False, signaled: 'pid'|'pgid'}
+      {status: 'killed', sigkill_required: True,  signaled: 'pid'|'pgid'}
+      {status: 'missing', error: '<exc>'} — process gone before SIGTERM
+      {status: 'failed',  error: 'alive_after_sigkill'} — process survived both
+      {status: 'failed',  error: '<exc>'} — permission denied / OS error
+    """
+    pgid, pgid_error = _safe_getpgid(pid, pgid_getter)
+    use_group = bool(pg_killer and pgid is not None and pgid > 1 and pgid != pid)
+    signaled = "pgid" if use_group else "pid"
+
+    def send(sig: int) -> tuple[bool, str | None]:
+        try:
+            if use_group:
+                pg_killer(pgid, sig)
+            else:
+                killer(pid, sig)
+        except ProcessLookupError:
+            return False, "process_lookup_error"
+        except (PermissionError, OSError) as exc:
+            return False, str(exc)
+        return True, None
+
+    ok, err = send(signal.SIGTERM)
+    if not ok:
+        if err == "process_lookup_error":
+            return {"status": "missing", "signaled": signaled, "pgid": pgid}
+        return {"status": "failed", "error": err, "signaled": signaled, "pgid": pgid}
+    if _wait_for_exit(pid, sigterm_wait_seconds, killer=killer, sleeper=sleeper):
+        return {
+            "status": "killed",
+            "sigkill_required": False,
+            "signaled": signaled,
+            "pgid": pgid,
+            "pgid_error": pgid_error,
+        }
+    # SIGTERM did not work — escalate.
+    ok, err = send(signal.SIGKILL)
+    if not ok:
+        if err == "process_lookup_error":
+            # Race: died between checks.
+            return {
+                "status": "killed",
+                "sigkill_required": False,
+                "signaled": signaled,
+                "pgid": pgid,
+                "pgid_error": pgid_error,
+            }
+        return {
+            "status": "failed",
+            "error": err,
+            "signaled": signaled,
+            "pgid": pgid,
+            "sigkill_attempted": True,
+        }
+    if _wait_for_exit(pid, sigkill_wait_seconds, killer=killer, sleeper=sleeper):
+        return {
+            "status": "killed",
+            "sigkill_required": True,
+            "signaled": signaled,
+            "pgid": pgid,
+            "pgid_error": pgid_error,
+        }
+    return {
+        "status": "failed",
+        "error": "alive_after_sigkill",
+        "signaled": signaled,
+        "pgid": pgid,
+        "sigkill_required": True,
+    }
+
+
+def _safe_getpgid(pid: int, pgid_getter) -> tuple[int | None, str | None]:
+    if pgid_getter is None:
+        return None, "getpgid_unavailable"
+    try:
+        return pgid_getter(pid), None
+    except (ProcessLookupError, PermissionError, OSError) as exc:
+        return None, str(exc)
+
+
+def _wait_for_exit(pid: int, timeout: float, *, killer, sleeper) -> bool:
+    deadline = time.monotonic() + max(timeout, 0.0)
+    while time.monotonic() < deadline:
+        try:
+            killer(pid, 0)
+        except ProcessLookupError:
+            return True
+        except (PermissionError, OSError):
+            return True
+        sleeper(0.1)
+    # Final check after the deadline elapses.
+    try:
+        killer(pid, 0)
+    except ProcessLookupError:
+        return True
+    except (PermissionError, OSError):
+        return True
+    return False
+
+
+def orphan_gate(
+    *,
+    fix: bool = False,
+    confirm: bool = False,
+    runner=subprocess.run,
+    killer=os.kill,
+    pg_killer=None,
+    pgid_getter=None,
+    sleeper=time.sleep,
+    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
+    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
+) -> dict[str, Any]:
+    if fix and not confirm:
+        return {
+            "ok": False,
+            "gate": "orphans",
+            "status": "refused",
+            "reason": "fix_requires_confirm",
+            "action": "re-run with --gate orphans --fix --confirm",
+        }
+    result = cleanup_orphan_coordinators(
+        confirm=fix and confirm,
+        runner=runner,
+        killer=killer,
+        pg_killer=pg_killer,
+        pgid_getter=pgid_getter,
+        sleeper=sleeper,
+        sigterm_wait_seconds=sigterm_wait_seconds,
+        sigkill_wait_seconds=sigkill_wait_seconds,
+    )
+    orphans = result.get("orphans") or []
+    failed = result.get("failed") or []
+    passed = not orphans if not fix else not failed
+    envelope = {
+        **result,
+        "ok": passed,
+        "gate": "orphans",
+        "status": "passed" if passed else "failed",
+        "fix": bool(fix),
+    }
+    if not fix and orphans:
+        envelope["action_required"] = "re-run with --gate orphans --fix --confirm"
+    return envelope
+
+
 def format_cleanup_orphans(result: dict[str, Any]) -> str:
     lines = [
         f"Coordinator orphan scan @ {result.get('scanned_at')}",
@@ -175,7 +343,9 @@ def format_cleanup_orphans(result: dict[str, Any]) -> str:
     if result.get("dry_run"):
         lines.append("  mode: DRY-RUN (no SIGTERM sent; re-run with --confirm)")
     else:
-        lines.append(f"  killed: {len(result.get('killed') or [])}")
+        killed_entries = result.get("killed") or []
+        escalated = sum(1 for k in killed_entries if k.get("sigkill_required"))
+        lines.append(f"  killed: {len(killed_entries)}  (sigkill_required: {escalated})")
         lines.append(f"  failed: {len(result.get('failed') or [])}")
     for orphan in result.get("orphans") or []:
         lines.append(
@@ -190,4 +360,5 @@ __all__ = [
     "classify_orphan",
     "find_coordinator_processes",
     "format_cleanup_orphans",
+    "orphan_gate",
 ]

package/src/team_agent/display/__init__.py

@@ -1,9 +1,27 @@
 from __future__ import annotations
 
+from team_agent.display.adaptive import (
+    ADAPTIVE_BLOCK_REASONS,
+    adaptive_blocked,
+    close_adaptive_display,
+    close_adaptive_windows,
+    open_adaptive_display,
+    prepare_adaptive_windows,
+    probe_display_capabilities,
+)
+from team_agent.display.backend import (
+    ADAPTIVE_DISPLAY_BACKEND,
+    DISPLAY_BACKENDS_WITH_WORKER_VIEWS,
+    GHOSTTY_DISPLAY_BACKENDS,
+    VALID_DISPLAY_BACKENDS,
+    resolve_display_backend,
+)
 from team_agent.display.close import (
+    close_adaptive_display,
     close_ghostty_display,
     close_ghostty_workspace,
     close_ghostty_workspace_slot,
+    close_team_display_backends,
 )
 from team_agent.display.ghostty import (
     ghostty_app_exists,
@@ -34,10 +52,19 @@ from team_agent.display.workspace import (
 )
 
 __all__ = [
+    "ADAPTIVE_BLOCK_REASONS",
+    "ADAPTIVE_DISPLAY_BACKEND",
+    "DISPLAY_BACKENDS_WITH_WORKER_VIEWS",
     "GHOSTTY_WORKSPACE_PANES_PER_WINDOW",
+    "GHOSTTY_DISPLAY_BACKENDS",
+    "VALID_DISPLAY_BACKENDS",
+    "adaptive_blocked",
+    "close_adaptive_display",
+    "close_adaptive_windows",
     "close_ghostty_display",
     "close_ghostty_workspace",
     "close_ghostty_workspace_slot",
+    "close_team_display_backends",
     "ghostty_app_exists",
     "ghostty_attach_args",
     "ghostty_command",
@@ -51,11 +78,15 @@ __all__ = [
     "ghostty_workspace_window_name",
     "kill_ghostty_workspace_linked_sessions",
     "open_ghostty_worker_window",
+    "open_adaptive_display",
     "open_ghostty_workspace",
     "open_ghostty_workspace_agent_display",
     "open_worker_displays",
+    "prepare_adaptive_windows",
     "prepare_ghostty_display_session",
     "prepare_ghostty_workspace_aggregator",
     "prepare_ghostty_workspace_linked_sessions",
+    "probe_display_capabilities",
+    "resolve_display_backend",
     "set_ghostty_workspace_pane_title",
 ]