@team-agent/installer 0.2.10 → 0.3.0

package/src/team_agent/diagnose/orphan_cleanup.py

@@ -1,364 +0,0 @@
-"""Stage 14 (Gap 37a) — `team-agent doctor --cleanup-orphans` implementation.
-
-Scans `ps` for processes matching `team_agent.coordinator --workspace <path>` and
-classifies any whose workspace path no longer exists (or matches the test-tempdir
-pattern) as an orphan. Dry-run by default; --confirm sends SIGTERM.
-
-Mac mini 2026-05-26 evidence: 35 orphan coordinator processes alive simultaneously
-pointing at /var/folders/.../T/team-agent-watcher-dedupe-* paths that had been removed
-hours earlier. Each holds a long-lived Python interpreter + SQLite connection.
-"""
-from __future__ import annotations
-
-import os
-import re
-import signal
-import subprocess
-import time
-from datetime import datetime, timezone
-from typing import Any
-
-# Pattern: argv contains "team_agent.coordinator --workspace <path>" anywhere.
-_COORDINATOR_ARGV_RE = re.compile(
-    r"team_agent\.coordinator(?:\.__main__)?(?:\s+|.*?)\s--workspace\s+(\S+)"
-)
-# Test-tempdir patterns that indicate the workspace is ephemeral and almost certainly orphan.
-_EPHEMERAL_PATH_HINTS = (
-    "team-agent-watcher-dedupe-",
-    "team-agent-gap",
-    "team-agent-stage",
-    "team-agent-orchestrator-",
-    "team-agent-rm-",
-    "team-agent-claim-",
-    "team-agent-hotfix",
-    "team-agent-multi",
-    "team-agent-progress-",
-    "team-agent-fanout-",
-    "team-agent-in-flight-",
-    "team-agent-test-",
-)
-_SIGTERM_WAIT_SECONDS = 3.0
-_SIGKILL_WAIT_SECONDS = 2.0
-
-
-def find_coordinator_processes(*, runner=subprocess.run) -> list[dict[str, Any]]:
-    """Return list of {pid, etime, cmdline, workspace} dicts for every running
-    team_agent.coordinator process visible to ps. workspace is None when the cmdline
-    doesn't parse — those are noted but not auto-classified as orphan."""
-    try:
-        proc = runner(
-            ["ps", "-Awwo", "pid=,etime=,command="],
-            text=True,
-            capture_output=True,
-            timeout=5,
-            check=False,
-        )
-    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
-        return []
-    if proc.returncode != 0 or not proc.stdout:
-        return []
-    rows: list[dict[str, Any]] = []
-    for line in proc.stdout.splitlines():
-        parts = line.strip().split(None, 2)
-        if len(parts) < 3:
-            continue
-        pid_s, etime, cmdline = parts[0], parts[1], parts[2]
-        if "team_agent.coordinator" not in cmdline:
-            continue
-        if "ps -Awwo" in cmdline:
-            continue
-        try:
-            pid = int(pid_s)
-        except ValueError:
-            continue
-        if pid == os.getpid():
-            continue
-        match = _COORDINATOR_ARGV_RE.search(cmdline)
-        workspace = match.group(1) if match else None
-        rows.append({
-            "pid": pid,
-            "etime": etime,
-            "cmdline": cmdline,
-            "workspace": workspace,
-        })
-    return rows
-
-
-def classify_orphan(entry: dict[str, Any]) -> tuple[bool, str]:
-    """Return (is_orphan, reason). An entry is orphan when its workspace path no longer
-    exists on disk OR matches a known ephemeral-tempdir pattern (test workspaces should
-    NEVER spawn long-lived coordinators)."""
-    workspace = entry.get("workspace")
-    if not workspace:
-        return False, "cmdline_unparsed"
-    if not os.path.exists(workspace):
-        return True, "workspace_path_missing"
-    for hint in _EPHEMERAL_PATH_HINTS:
-        if hint in workspace:
-            return True, f"ephemeral_tempdir_pattern:{hint}"
-    return False, "workspace_alive"
-
-
-def cleanup_orphan_coordinators(
-    *,
-    confirm: bool = False,
-    runner=subprocess.run,
-    killer=os.kill,
-    pg_killer=None,
-    pgid_getter=None,
-    sleeper=time.sleep,
-    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
-    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
-) -> dict[str, Any]:
-    """Scan for orphan coordinators. Without confirm: dry-run (just classify and report).
-    With confirm: SIGTERM each orphan, wait up to _SIGTERM_WAIT_SECONDS for graceful
-    exit; if still alive, escalate to SIGKILL and wait _SIGKILL_WAIT_SECONDS. Only
-    report status='failed' (with error='alive_after_sigkill') when the process
-    survives BOTH signals — that's extremely rare and almost always indicates a
-    zombie/uninterruptible-sleep kernel state.
-
-    Mac mini 2026-05-26 evidence: real orphan coordinators have been observed alive
-    40+ hours; many of them never exit on SIGTERM (signal handler suppressed during
-    long sqlite reads, or the python interpreter is hosting an async loop that
-    swallows the term signal). SIGKILL escalation is required for production.
-
-    pg_killer / pgid_getter default to os.killpg / os.getpgid; mock them in tests.
-    If pgid_getter succeeds AND returns a pgid > 1 AND the pgid != pid (i.e. the
-    process leads its own process group with children), we signal the WHOLE group;
-    otherwise we signal the pid directly. This catches orphan coordinators that
-    spawned subprocess.Popen children which would otherwise survive a pid-only
-    SIGTERM."""
-    now = datetime.now(timezone.utc).isoformat()
-    if pg_killer is None:
-        pg_killer = getattr(os, "killpg", None)
-    if pgid_getter is None:
-        pgid_getter = getattr(os, "getpgid", None)
-    entries = find_coordinator_processes(runner=runner)
-    classified: list[dict[str, Any]] = []
-    orphans: list[dict[str, Any]] = []
-    for entry in entries:
-        is_orphan, reason = classify_orphan(entry)
-        annotated = {**entry, "is_orphan": is_orphan, "reason": reason}
-        classified.append(annotated)
-        if is_orphan:
-            orphans.append(annotated)
-    if not confirm:
-        return {
-            "ok": True,
-            "scanned": len(classified),
-            "orphans": orphans,
-            "dry_run": True,
-            "scanned_at": now,
-            "action_required": "re-run with --confirm to send SIGTERM",
-        }
-    killed: list[dict[str, Any]] = []
-    failed: list[dict[str, Any]] = []
-    for entry in orphans:
-        outcome = _terminate_orphan(
-            entry["pid"], killer=killer, pg_killer=pg_killer,
-            pgid_getter=pgid_getter, sleeper=sleeper,
-            sigterm_wait_seconds=sigterm_wait_seconds,
-            sigkill_wait_seconds=sigkill_wait_seconds,
-        )
-        annotated = {**entry, **outcome}
-        if outcome.get("status") == "killed":
-            killed.append(annotated)
-        elif outcome.get("status") == "missing":
-            killed.append(annotated)
-        else:
-            failed.append(annotated)
-    return {
-        "ok": True,
-        "scanned": len(classified),
-        "orphans": orphans,
-        "killed": killed,
-        "failed": failed,
-        "dry_run": False,
-        "scanned_at": now,
-    }
-
-
-def _terminate_orphan(
-    pid: int,
-    *,
-    killer,
-    pg_killer,
-    pgid_getter,
-    sleeper,
-    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
-    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
-) -> dict[str, Any]:
-    """SIGTERM → wait 3s → SIGKILL → wait 2s escalation. Returns one of:
-      {status: 'killed', sigkill_required: False, signaled: 'pid'|'pgid'}
-      {status: 'killed', sigkill_required: True,  signaled: 'pid'|'pgid'}
-      {status: 'missing', error: '<exc>'} — process gone before SIGTERM
-      {status: 'failed',  error: 'alive_after_sigkill'} — process survived both
-      {status: 'failed',  error: '<exc>'} — permission denied / OS error
-    """
-    pgid, pgid_error = _safe_getpgid(pid, pgid_getter)
-    use_group = bool(pg_killer and pgid is not None and pgid > 1 and pgid != pid)
-    signaled = "pgid" if use_group else "pid"
-
-    def send(sig: int) -> tuple[bool, str | None]:
-        try:
-            if use_group:
-                pg_killer(pgid, sig)
-            else:
-                killer(pid, sig)
-        except ProcessLookupError:
-            return False, "process_lookup_error"
-        except (PermissionError, OSError) as exc:
-            return False, str(exc)
-        return True, None
-
-    ok, err = send(signal.SIGTERM)
-    if not ok:
-        if err == "process_lookup_error":
-            return {"status": "missing", "signaled": signaled, "pgid": pgid}
-        return {"status": "failed", "error": err, "signaled": signaled, "pgid": pgid}
-    if _wait_for_exit(pid, sigterm_wait_seconds, killer=killer, sleeper=sleeper):
-        return {
-            "status": "killed",
-            "sigkill_required": False,
-            "signaled": signaled,
-            "pgid": pgid,
-            "pgid_error": pgid_error,
-        }
-    # SIGTERM did not work — escalate.
-    ok, err = send(signal.SIGKILL)
-    if not ok:
-        if err == "process_lookup_error":
-            # Race: died between checks.
-            return {
-                "status": "killed",
-                "sigkill_required": False,
-                "signaled": signaled,
-                "pgid": pgid,
-                "pgid_error": pgid_error,
-            }
-        return {
-            "status": "failed",
-            "error": err,
-            "signaled": signaled,
-            "pgid": pgid,
-            "sigkill_attempted": True,
-        }
-    if _wait_for_exit(pid, sigkill_wait_seconds, killer=killer, sleeper=sleeper):
-        return {
-            "status": "killed",
-            "sigkill_required": True,
-            "signaled": signaled,
-            "pgid": pgid,
-            "pgid_error": pgid_error,
-        }
-    return {
-        "status": "failed",
-        "error": "alive_after_sigkill",
-        "signaled": signaled,
-        "pgid": pgid,
-        "sigkill_required": True,
-    }
-
-
-def _safe_getpgid(pid: int, pgid_getter) -> tuple[int | None, str | None]:
-    if pgid_getter is None:
-        return None, "getpgid_unavailable"
-    try:
-        return pgid_getter(pid), None
-    except (ProcessLookupError, PermissionError, OSError) as exc:
-        return None, str(exc)
-
-
-def _wait_for_exit(pid: int, timeout: float, *, killer, sleeper) -> bool:
-    deadline = time.monotonic() + max(timeout, 0.0)
-    while time.monotonic() < deadline:
-        try:
-            killer(pid, 0)
-        except ProcessLookupError:
-            return True
-        except (PermissionError, OSError):
-            return True
-        sleeper(0.1)
-    # Final check after the deadline elapses.
-    try:
-        killer(pid, 0)
-    except ProcessLookupError:
-        return True
-    except (PermissionError, OSError):
-        return True
-    return False
-
-
-def orphan_gate(
-    *,
-    fix: bool = False,
-    confirm: bool = False,
-    runner=subprocess.run,
-    killer=os.kill,
-    pg_killer=None,
-    pgid_getter=None,
-    sleeper=time.sleep,
-    sigterm_wait_seconds: float = _SIGTERM_WAIT_SECONDS,
-    sigkill_wait_seconds: float = _SIGKILL_WAIT_SECONDS,
-) -> dict[str, Any]:
-    if fix and not confirm:
-        return {
-            "ok": False,
-            "gate": "orphans",
-            "status": "refused",
-            "reason": "fix_requires_confirm",
-            "action": "re-run with --gate orphans --fix --confirm",
-        }
-    result = cleanup_orphan_coordinators(
-        confirm=fix and confirm,
-        runner=runner,
-        killer=killer,
-        pg_killer=pg_killer,
-        pgid_getter=pgid_getter,
-        sleeper=sleeper,
-        sigterm_wait_seconds=sigterm_wait_seconds,
-        sigkill_wait_seconds=sigkill_wait_seconds,
-    )
-    orphans = result.get("orphans") or []
-    failed = result.get("failed") or []
-    passed = not orphans if not fix else not failed
-    envelope = {
-        **result,
-        "ok": passed,
-        "gate": "orphans",
-        "status": "passed" if passed else "failed",
-        "fix": bool(fix),
-    }
-    if not fix and orphans:
-        envelope["action_required"] = "re-run with --gate orphans --fix --confirm"
-    return envelope
-
-
-def format_cleanup_orphans(result: dict[str, Any]) -> str:
-    lines = [
-        f"Coordinator orphan scan @ {result.get('scanned_at')}",
-        f"  scanned: {result.get('scanned', 0)} coordinator processes",
-        f"  orphans: {len(result.get('orphans') or [])}",
-    ]
-    if result.get("dry_run"):
-        lines.append("  mode: DRY-RUN (no SIGTERM sent; re-run with --confirm)")
-    else:
-        killed_entries = result.get("killed") or []
-        escalated = sum(1 for k in killed_entries if k.get("sigkill_required"))
-        lines.append(f"  killed: {len(killed_entries)}  (sigkill_required: {escalated})")
-        lines.append(f"  failed: {len(result.get('failed') or [])}")
-    for orphan in result.get("orphans") or []:
-        lines.append(
-            f"  PID {orphan['pid']} etime={orphan['etime']} "
-            f"workspace={orphan.get('workspace') or '?'} reason={orphan.get('reason')}"
-        )
-    return "\n".join(lines)
-
-
-__all__ = [
-    "cleanup_orphan_coordinators",
-    "classify_orphan",
-    "find_coordinator_processes",
-    "format_cleanup_orphans",
-    "orphan_gate",
-]

package/src/team_agent/diagnose/preflight.py

@@ -1,194 +0,0 @@
-from __future__ import annotations
-
-import json
-import time
-from pathlib import Path
-from typing import Any
-
-from team_agent.diagnose.checks import (
-    compact_model_checks,
-    model_checks_for_agents,
-    profile_checks_for_agents,
-    profile_smoke_checks_for_agents,
-)
-from team_agent.events import EventLog
-from team_agent.paths import logs_dir, team_workspace
-from team_agent.profiles import compact_profile_check
-from team_agent.rust_core import core_binary
-from team_agent.simple_yaml import dumps
-
-
-def preflight(team_dir: Path) -> dict[str, Any]:
-    from team_agent.compiler import compile_team
-    from team_agent.profiles import profile_dir
-    from team_agent.runtime import (
-        GHOSTTY_DISPLAY_BACKENDS,
-        _attach_team_profile_dirs,
-        _ghostty_command,
-        ensure_workspace_dirs,
-        shutil_which,
-    )
-
-    team_dir = team_dir.resolve()
-    workspace = team_workspace(team_dir)
-    ensure_workspace_dirs(workspace)
-    ensure_profiles_for_roles(team_dir)
-    event_log = EventLog(workspace)
-    checks: list[dict[str, Any]] = []
-    ok = True
-    spec = None
-    try:
-        compiled = compile_team(team_dir)
-        spec = compiled["spec"]
-        _attach_team_profile_dirs(spec, team_dir / "team.spec.yaml", workspace, team_dir)
-        checks.append({"name": "compile", "ok": True, "agents": [a["id"] for a in spec.get("agents", [])]})
-    except Exception as exc:
-        ok = False
-        checks.append({"name": "compile", "ok": False, "error": str(exc)})
-    tmux_path = shutil_which("tmux")
-    checks.append({"name": "tmux", "ok": bool(tmux_path), "path": tmux_path})
-    ok = ok and bool(tmux_path)
-    ghostty = _ghostty_command()
-    ghostty_check = {"name": "ghostty", "ok": bool(ghostty), "path": ghostty, "required": False}
-    if spec and spec.get("runtime", {}).get("display_backend") in GHOSTTY_DISPLAY_BACKENDS:
-        ghostty_check["required"] = True
-        ok = ok and bool(ghostty)
-    checks.append(ghostty_check)
-    if spec:
-        profile_checks = profile_checks_for_agents(workspace, spec.get("agents", []))
-        profile_failures = [item for item in profile_checks if item.get("ok") is False]
-        checks.append({"name": "profiles", "ok": not profile_failures, "checks": [compact_profile_check(item) for item in profile_checks]})
-        ok = ok and not profile_failures
-        smoke_checks = profile_smoke_checks_for_agents(workspace, spec.get("agents", []))
-        smoke_failures = [item for item in smoke_checks if item.get("ok") is False]
-        checks.append({"name": "profile_smoke", "ok": not smoke_failures, "checks": [compact_profile_check(item) for item in smoke_checks]})
-        ok = ok and not smoke_failures
-        model_checks = model_checks_for_agents(spec.get("agents", []), workspace)
-        model_failures = [item for item in model_checks if item.get("ok") is False]
-        checks.append({"name": "models", "ok": not model_failures, "checks": compact_model_checks(model_checks)})
-        ok = ok and not model_failures
-    core = core_binary()
-    checks.append(
-        {
-            "name": "rust_core",
-            "ok": True,
-            "required": False,
-            "available": bool(core),
-            "path": str(core) if core else None,
-            "status": "available" if core else "python_fallback",
-        }
-    )
-    checks.append({"name": "profile_dir", "ok": profile_dir(workspace).exists() or (team_dir / "profiles").exists()})
-    details_log = logs_dir(workspace) / f"preflight-{int(time.time())}.json"
-    details = {"team_dir": str(team_dir), "checks": checks}
-    details_log.write_text(json.dumps(details, indent=2, ensure_ascii=False), encoding="utf-8")
-    event_log.write("preflight.complete", ok=ok, details_log=str(details_log), checks=checks)
-    blockers = [] if ok else preflight_blockers(checks)
-    return {
-        "ok": ok,
-        "summary": "preflight passed" if ok else "preflight found blockers: " + "; ".join(blockers[:3]),
-        "next_actions": [f"team-agent start --team {team_dir} --yes --json"] if ok else preflight_next_actions(blockers),
-        "details_log": str(details_log),
-        "checks": checks,
-        "blockers": blockers,
-    }
-
-
-def start(team_dir: Path, yes: bool = False) -> dict[str, Any]:
-    from team_agent.compiler import compile_team
-    from team_agent.runtime import launch
-
-    team_dir = team_dir.resolve()
-    workspace = team_workspace(team_dir)
-    spec_path = team_dir / "team.spec.yaml"
-    compiled = compile_team(team_dir, spec_path)
-    if compiled["spec"].get("context", {}).get("state_file") == "team_state.md":
-        state_file = str(team_dir.relative_to(workspace) / "team_state.md") if team_dir.is_relative_to(workspace) else "team_state.md"
-        compiled["spec"]["context"]["state_file"] = state_file
-        spec_path.write_text(dumps(compiled["spec"]), encoding="utf-8")
-    launched = launch(spec_path, auto_approve=yes)
-    details_log = logs_dir(workspace) / f"start-{int(time.time())}.json"
-    details_log.write_text(json.dumps({"compile": compiled, "launch": launched}, indent=2, ensure_ascii=False), encoding="utf-8")
-    return {
-        "ok": bool(launched.get("ok")),
-        "summary": f"compiled {team_dir} and launched {len(launched.get('agents', []))} agents",
-        "next_actions": ["team-agent wait-ready --workspace . --timeout 120 --json"],
-        "details_log": str(details_log),
-        "spec": str(spec_path),
-        "launch": launched,
-    }
-
-
-def preflight_blockers(checks: list[dict[str, Any]]) -> list[str]:
-    blockers: list[str] = []
-    for check in checks:
-        if check.get("ok", True):
-            continue
-        name = check.get("name") or "check"
-        if name == "compile":
-            blockers.append(f"compile: {check.get('error')}")
-            continue
-        for item in check.get("checks", []) or []:
-            agent = item.get("agent_id") or item.get("profile") or "-"
-            reason = item.get("reason") or item.get("status") or "failed"
-            detail = f"{name}: {agent} {reason}"
-            if item.get("endpoint"):
-                detail += f" endpoint={item['endpoint']}"
-            if item.get("proxy_configured"):
-                detail += f" proxy={item.get('proxy_url') or item.get('proxy_scheme')}"
-            if item.get("proxy_source"):
-                detail += f" proxy_source={item['proxy_source']}"
-            if item.get("proxy_mode"):
-                detail += f" proxy_mode={item['proxy_mode']}"
-            if item.get("missing_required"):
-                detail += " missing=" + ",".join(item["missing_required"])
-            if item.get("effective_model"):
-                detail += f" model={item['effective_model']}"
-            if item.get("suggestion"):
-                detail += f" suggestion={item['suggestion']}"
-            blockers.append(detail)
-        if not check.get("checks"):
-            blockers.append(f"{name}: failed")
-    return blockers or ["unknown preflight blocker"]
-
-
-def preflight_next_actions(blockers: list[str]) -> list[str]:
-    actions = ["Fix failed checks, then rerun preflight."]
-    if any("proxy_connectivity_failed" in item for item in blockers):
-        actions.insert(0, "Allow the profile BASE_URL through the configured proxy, or disable the proxy for Team Agent startup.")
-    if any("proxy_source=ambient" in item for item in blockers):
-        actions.insert(0, "Current environment proxy is being used for this compatible_api worker; either fix that proxy for BASE_URL, set HTTPS_PROXY/HTTP_PROXY in the profile, or set PROXY_MODE=direct in the profile to bypass proxy for this worker.")
-    if any("missing=" in item or "profile_required_values_missing" in item for item in blockers):
-        actions.insert(
-            0,
-            "Ask the human user to fill the local profile file; agents must inspect only with `team-agent profile show <name> --workspace . --json` or the returned --team variant and must not read .team/*/profiles/*.env.",
-        )
-    if any("model_mismatch" in item or "does not match profile MODEL" in item for item in blockers):
-        actions.insert(0, "Keep the model in the profile MODEL field or make the role model exactly match it.")
-    return actions
-
-
-def ensure_profiles_for_roles(team_dir: Path) -> None:
-    from team_agent.compiler import _read_front_matter
-    from team_agent.profiles import ensure_profile_secret_boundary, ensure_profile_secret_boundary_dir, init_profile
-
-    workspace = team_workspace(team_dir)
-    profiles_dir = team_dir / "profiles"
-    profiles_dir.mkdir(parents=True, exist_ok=True)
-    ensure_profile_secret_boundary(workspace)
-    ensure_profile_secret_boundary_dir(profiles_dir)
-    for role_doc in sorted((team_dir / "agents").glob("*.md")):
-        meta, _ = _read_front_matter(role_doc)
-        profile = meta.get("profile")
-        auth_mode = meta.get("auth_mode") or "subscription"
-        if not profile:
-            continue
-        if not (profiles_dir / f"{profile}.env").exists() and not (profiles_dir / f"{profile}.example.env").exists():
-            init_profile(workspace, str(profile), str(auth_mode))
-            if auth_mode == "subscription":
-                body = f"AUTH_MODE=subscription\nPROFILE_NAME={profile}\n"
-            elif auth_mode == "official_api":
-                body = f"AUTH_MODE=official_api\nPROFILE_NAME={profile}\nAPI_KEY=\nMODEL=\n"
-            else:
-                body = f"AUTH_MODE={auth_mode}\nPROFILE_NAME={profile}\nBASE_URL=\nAPI_KEY=\nMODEL=\n"
-            (profiles_dir / f"{profile}.example.env").write_text(body, encoding="utf-8")