@team-agent/installer 0.1.10 → 0.2.0

package/src/team_agent/{coordinator.py → coordinator/__main__.py}

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import os
 import signal
 import sys
 import time
@@ -29,7 +30,8 @@ def main(argv: list[str] | None = None) -> None:
     args = parser.parse_args(argv)
     workspace = Path(args.workspace).resolve()
     runtime.ensure_workspace_dirs(workspace)
-    runtime.coordinator_pid_path(workspace).write_text(str(__import__("os").getpid()), encoding="utf-8")
+    runtime.coordinator_pid_path(workspace).write_text(str(os.getpid()), encoding="utf-8")
+    runtime.write_coordinator_metadata(workspace, os.getpid(), source="boot")
     event_log = EventLog(workspace)
     event_log.write("coordinator.boot", workspace=str(workspace), once=args.once)
     signal.signal(signal.SIGTERM, _stop)

package/src/team_agent/coordinator/lifecycle.py

@@ -0,0 +1,319 @@
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from team_agent.coordinator.metadata import (
+    COORDINATOR_PROTOCOL_VERSION,
+    coordinator_metadata_ok,
+    pid_is_running,
+    read_coordinator_metadata,
+    write_coordinator_metadata,
+)
+from team_agent.coordinator.paths import (
+    coordinator_log_path,
+    coordinator_meta_path,
+    coordinator_pid_path,
+)
+from team_agent.events import EventLog
+from team_agent.message_store import MessageStore
+
+
+def coordinator_health(workspace: Path) -> dict[str, Any]:
+    schema = message_store_schema_health(workspace)
+    pid_path = coordinator_pid_path(workspace)
+    if not pid_path.exists():
+        return {"ok": False, "status": "missing", "pid": None, "metadata": None, "metadata_ok": False, **schema}
+    try:
+        pid = int(pid_path.read_text(encoding="utf-8").strip())
+    except ValueError:
+        return {"ok": False, "status": "invalid_pid", "pid": None, "metadata": None, "metadata_ok": False, **schema}
+    running = pid_is_running(pid)
+    metadata = read_coordinator_metadata(workspace)
+    metadata_ok = coordinator_metadata_ok(metadata, pid)
+    ok = running and metadata_ok and bool(schema.get("schema_ok"))
+    return {
+        "ok": ok,
+        "status": "running" if running else "stale",
+        "pid": pid,
+        "metadata": metadata,
+        "metadata_ok": metadata_ok,
+        **schema,
+    }
+
+
+def start_coordinator(workspace: Path) -> dict[str, Any]:
+    from team_agent.runtime import ensure_workspace_dirs
+    ensure_workspace_dirs(workspace)
+    health = coordinator_health(workspace)
+    if health["ok"]:
+        return {"ok": True, "pid": health["pid"], "status": "already_running", "log": str(coordinator_log_path(workspace))}
+    if health["status"] == "running" and not health.get("metadata_ok"):
+        EventLog(workspace).write(
+            "coordinator.restart_incompatible",
+            pid=health.get("pid"),
+            metadata=health.get("metadata"),
+            expected_protocol=COORDINATOR_PROTOCOL_VERSION,
+            expected_schema=MessageStore.SCHEMA_VERSION,
+        )
+        stopped = stop_coordinator(workspace)
+        if not stopped.get("ok"):
+            EventLog(workspace).write(
+                "coordinator.restart_incompatible_stop_failed",
+                pid=health.get("pid"),
+                stop_result=stopped,
+            )
+            return {
+                "ok": False,
+                "pid": health.get("pid"),
+                "status": "restart_incompatible_stop_failed",
+                "error": stopped.get("error") or stopped.get("status"),
+                "stop_result": stopped,
+            }
+    if not health.get("schema_ok", False):
+        EventLog(workspace).write(
+            "coordinator.schema_incompatible",
+            error=health.get("schema_error"),
+            schema=health.get("schema"),
+            reason=health.get("reason"),
+            table=health.get("table"),
+            missing_columns=health.get("missing_columns"),
+        )
+        return {
+            "ok": False,
+            "pid": None,
+            "status": "schema_incompatible",
+            "error": health.get("schema_error"),
+            "schema": health.get("schema"),
+            "action": health.get("action", _SCHEMA_ACTION_HINT),
+            "reason": health.get("reason"),
+            "table": health.get("table"),
+            "expected_columns": health.get("expected_columns"),
+            "actual_columns": health.get("actual_columns"),
+            "missing_columns": health.get("missing_columns"),
+        }
+    if health["status"] in {"stale", "invalid_pid"}:
+        coordinator_pid_path(workspace).unlink(missing_ok=True)
+        coordinator_meta_path(workspace).unlink(missing_ok=True)
+    log_path = coordinator_log_path(workspace)
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    env = dict(os.environ)
+    repo_src = str(Path(__file__).resolve().parents[2])
+    env["PYTHONPATH"] = repo_src + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
+    log = log_path.open("a", encoding="utf-8")
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "team_agent.coordinator", "--workspace", str(workspace)],
+        cwd=str(workspace),
+        stdin=subprocess.DEVNULL,
+        stdout=log,
+        stderr=log,
+        env=env,
+        start_new_session=True,
+    )
+    log.close()
+    coordinator_pid_path(workspace).write_text(str(proc.pid), encoding="utf-8")
+    write_coordinator_metadata(workspace, proc.pid, source="start")
+    EventLog(workspace).write("coordinator.started", pid=proc.pid, log=str(log_path))
+    return {"ok": True, "pid": proc.pid, "status": "started", "log": str(log_path)}
+
+
+_SCHEMA_EXPECTED_COLUMNS: dict[str, set[str]] = {}
+_SCHEMA_MIGRATABLE_COLUMNS: dict[str, set[str]] = {
+    "messages": {"delivery_attempts", "owner_team_id"},
+    "scheduled_events": {"owner_team_id"},
+    "agent_health": {"owner_team_id"},
+    "result_watchers": {"owner_team_id"},
+}
+_SCHEMA_ACTION_HINT = (
+    "use team-agent advanced repair-state --schema to re-run column migrations; "
+    "if that fails, back up .team/runtime/team.db, then delete it and rerun team-agent launch "
+    "(in-flight messages will be lost)"
+)
+
+
+def _load_expected_schema_columns() -> dict[str, set[str]]:
+    if _SCHEMA_EXPECTED_COLUMNS:
+        return _SCHEMA_EXPECTED_COLUMNS
+    from team_agent.message_store.schema import (
+        AGENT_HEALTH_COLUMNS,
+        DELIVERY_TOKEN_COLUMNS,
+        MESSAGE_COLUMNS,
+        PEER_ALLOWLIST_COLUMNS,
+        RESULT_COLUMNS,
+        RESULT_WATCHER_COLUMNS,
+        SCHEDULED_EVENT_COLUMNS,
+    )
+    _SCHEMA_EXPECTED_COLUMNS.update(
+        {
+            "messages": set(MESSAGE_COLUMNS),
+            "results": set(RESULT_COLUMNS),
+            "scheduled_events": set(SCHEDULED_EVENT_COLUMNS),
+            "delivery_tokens": set(DELIVERY_TOKEN_COLUMNS),
+            "agent_health": set(AGENT_HEALTH_COLUMNS),
+            "peer_allowlist": set(PEER_ALLOWLIST_COLUMNS),
+            "result_watchers": set(RESULT_WATCHER_COLUMNS),
+        }
+    )
+    return _SCHEMA_EXPECTED_COLUMNS
+
+
+def _diagnose_schema_mismatch(workspace: Path, *, ignore_migratable: bool = False) -> dict[str, Any] | None:
+    import sqlite3
+    from team_agent.paths import runtime_dir
+    db_path = runtime_dir(workspace) / "team.db"
+    if not db_path.exists():
+        return None
+    conn = sqlite3.connect(db_path)
+    try:
+        for table, expected in _load_expected_schema_columns().items():
+            present = conn.execute(
+                "select name from sqlite_master where type='table' and name=?",
+                (table,),
+            ).fetchone()
+            if present is None:
+                continue
+            actual = {row[1] for row in conn.execute(f"pragma table_info({table})").fetchall()}
+            missing = expected - actual
+            if ignore_migratable:
+                migratable = _SCHEMA_MIGRATABLE_COLUMNS.get(table, set())
+                missing = missing - migratable
+            if missing:
+                return {
+                    "reason": "schema_mismatch",
+                    "table": table,
+                    "expected_columns": sorted(expected),
+                    "actual_columns": sorted(actual),
+                    "missing_columns": sorted(missing),
+                }
+    finally:
+        conn.close()
+    return None
+
+
+def message_store_schema_health(workspace: Path) -> dict[str, Any]:
+    schema_version = {"message_store_schema_version": MessageStore.SCHEMA_VERSION}
+    pre_mismatch = _diagnose_schema_mismatch(workspace, ignore_migratable=True)
+    if pre_mismatch is not None:
+        return {
+            "schema_ok": False,
+            "schema_error": (
+                f"team.db table {pre_mismatch['table']} is missing required column(s): "
+                + ", ".join(pre_mismatch["missing_columns"])
+            ),
+            "schema": schema_version,
+            "action": _SCHEMA_ACTION_HINT,
+            **pre_mismatch,
+        }
+    try:
+        MessageStore(workspace)
+    except Exception as exc:
+        post_init_mismatch = _diagnose_schema_mismatch(workspace) or {}
+        return {
+            "schema_ok": False,
+            "schema_error": str(exc),
+            "schema": schema_version,
+            "action": _SCHEMA_ACTION_HINT,
+            **post_init_mismatch,
+        }
+    return {
+        "schema_ok": True,
+        "schema_error": None,
+        "schema": schema_version,
+    }
+
+
+def stop_coordinator(workspace: Path) -> dict[str, Any]:
+    pid_path = coordinator_pid_path(workspace)
+    if not pid_path.exists():
+        return {"ok": True, "status": "missing"}
+    try:
+        pid = int(pid_path.read_text(encoding="utf-8").strip())
+    except ValueError:
+        pid_path.unlink(missing_ok=True)
+        coordinator_meta_path(workspace).unlink(missing_ok=True)
+        return {"ok": True, "status": "invalid_pid_removed"}
+    if pid_is_running(pid):
+        try:
+            os.kill(pid, signal.SIGTERM)
+        except OSError as exc:
+            return {"ok": False, "status": "kill_failed", "pid": pid, "error": str(exc)}
+    pid_path.unlink(missing_ok=True)
+    coordinator_meta_path(workspace).unlink(missing_ok=True)
+    EventLog(workspace).write("coordinator.stopped", pid=pid)
+    return {"ok": True, "status": "stopped", "pid": pid}
+
+
+def coordinator_tick(workspace: Path) -> dict[str, Any]:
+    from team_agent.runtime import (
+        _capture_missing_sessions,
+        _deliver_pending_messages,
+        _detect_stuck_agents,
+        _fire_due_scheduled_events,
+        _handle_provider_runtime_prompts,
+        _handle_provider_startup_prompts,
+        _refresh_agent_runtime_statuses,
+        _sync_agent_health,
+        _tmux_session_exists,
+        _collect_results_and_notify_watchers,
+    )
+    from team_agent.messaging.idle_alerts import (
+        detect_cross_worker_deadlocks,
+        detect_idle_fallbacks,
+    )
+    from team_agent.messaging.activity_detector import detect_compaction_degradation
+    from team_agent.state import load_runtime_state, save_runtime_state
+    state = load_runtime_state(workspace)
+    event_log = EventLog(workspace)
+    store = MessageStore(workspace)
+    session_name = state.get("session_name")
+    if session_name and not _tmux_session_exists(session_name):
+        event_log.write("coordinator.session_missing", session=session_name)
+        return {"ok": False, "stop": True, "reason": "tmux_session_missing"}
+    _capture_missing_sessions(workspace, state, event_log, timeout_s=0.0, log_miss=False)
+    _refresh_agent_runtime_statuses(workspace, state, event_log)
+    _handle_provider_startup_prompts(workspace, state, event_log)
+    _handle_provider_runtime_prompts(workspace, state, event_log)
+    captures = _sync_agent_health(workspace, state, store) or {}
+    delivered = _deliver_pending_messages(workspace, state, event_log)
+    fired = _fire_due_scheduled_events(workspace, store, event_log)
+    stuck = _detect_stuck_agents(workspace, state, store, event_log)
+    idle_alerts = detect_idle_fallbacks(workspace, state, store, event_log)
+    deadlock_alerts = detect_cross_worker_deadlocks(workspace, state, store, event_log)
+    compaction_results: list[dict[str, Any]] = []
+    for agent_id, agent_state in state.get("agents", {}).items():
+        provider = str(agent_state.get("provider") or "")
+        if provider != "codex":
+            continue
+        cap = captures.get(agent_id) or {}
+        scrollback = str(cap.get("scrollback") or "")
+        if not scrollback:
+            continue
+        stuck_loop = agent_id in (stuck or [])
+        result = detect_compaction_degradation(
+            workspace,
+            state,
+            event_log,
+            agent_id=agent_id,
+            provider=provider,
+            scrollback=scrollback,
+            stuck_loop=stuck_loop,
+        )
+        if result.get("event") and result.get("event") != "compaction_threshold_crossed.none":
+            compaction_results.append(result)
+    save_runtime_state(workspace, state)
+    results = _collect_results_and_notify_watchers(workspace, event_log)
+    return {
+        "ok": True,
+        "stop": False,
+        "delivered": delivered,
+        "scheduled": fired,
+        "stuck": stuck,
+        "idle_alerts": idle_alerts,
+        "deadlock_alerts": deadlock_alerts,
+        "compaction": compaction_results,
+        "results": results,
+    }

package/src/team_agent/coordinator/metadata.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from team_agent.coordinator.paths import coordinator_meta_path
+from team_agent.message_store import MessageStore
+
+
+COORDINATOR_PROTOCOL_VERSION = 2
+
+
+def pid_is_running(pid: int) -> bool:
+    from team_agent.runtime import run_cmd
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    proc = run_cmd(["ps", "-p", str(pid), "-o", "stat="], timeout=5)
+    if proc.returncode == 0 and proc.stdout.strip().upper().startswith("Z"):
+        return False
+    return True
+
+
+def read_coordinator_metadata(workspace: Path) -> dict[str, Any] | None:
+    path = coordinator_meta_path(workspace)
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    return raw if isinstance(raw, dict) else None
+
+
+def coordinator_metadata_ok(metadata: dict[str, Any] | None, pid: int) -> bool:
+    return bool(
+        metadata
+        and metadata.get("pid") == pid
+        and metadata.get("protocol_version") == COORDINATOR_PROTOCOL_VERSION
+        and metadata.get("message_store_schema_version") == MessageStore.SCHEMA_VERSION
+    )
+
+
+def write_coordinator_metadata(workspace: Path, pid: int, source: str) -> None:
+    path = coordinator_meta_path(workspace)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(
+            {
+                "pid": pid,
+                "protocol_version": COORDINATOR_PROTOCOL_VERSION,
+                "message_store_schema_version": MessageStore.SCHEMA_VERSION,
+                "source": source,
+                "updated_at": datetime.now(timezone.utc).isoformat(),
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )

package/src/team_agent/coordinator/paths.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from team_agent.paths import runtime_dir
+
+
+def coordinator_pid_path(workspace: Path) -> Path:
+    return runtime_dir(workspace) / "coordinator.pid"
+
+
+def coordinator_meta_path(workspace: Path) -> Path:
+    return runtime_dir(workspace) / "coordinator.json"
+
+
+def coordinator_log_path(workspace: Path) -> Path:
+    return runtime_dir(workspace) / "coordinator.log"

package/src/team_agent/diagnose/__init__.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from team_agent.diagnose.checks import (
+    compact_model_checks,
+    format_model_check_failures,
+    format_profile_check_failures,
+    format_profile_smoke_failures,
+    model_checks_for_agents,
+    profile_checks_for_agents,
+    profile_smoke_checks_for_agents,
+)
+from team_agent.diagnose.health import diagnose, doctor
+from team_agent.diagnose.preflight import (
+    ensure_profiles_for_roles,
+    preflight,
+    preflight_blockers,
+    preflight_next_actions,
+    start,
+)
+from team_agent.diagnose.quick_start import (
+    prepare_quick_start_team,
+    quick_start,
+    repair_state,
+    settle,
+    wait_ready,
+)
+
+__all__ = [
+    "compact_model_checks",
+    "diagnose",
+    "doctor",
+    "ensure_profiles_for_roles",
+    "format_model_check_failures",
+    "format_profile_check_failures",
+    "format_profile_smoke_failures",
+    "model_checks_for_agents",
+    "prepare_quick_start_team",
+    "preflight",
+    "preflight_blockers",
+    "preflight_next_actions",
+    "profile_checks_for_agents",
+    "profile_smoke_checks_for_agents",
+    "quick_start",
+    "repair_state",
+    "settle",
+    "start",
+    "wait_ready",
+]

package/src/team_agent/diagnose/checks.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from team_agent.profiles import effective_model, smoke_check_agent_profile, validate_agent_profile
+
+
+def profile_checks_for_agents(workspace: Path, agents: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [validate_agent_profile(workspace, agent) for agent in agents if not agent.get("paused")]
+
+
+def profile_smoke_checks_for_agents(workspace: Path, agents: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [smoke_check_agent_profile(workspace, agent) for agent in agents if not agent.get("paused")]
+
+
+def model_checks_for_agents(agents: list[dict[str, Any]], workspace: Path | None = None) -> list[dict[str, Any]]:
+    from team_agent.runtime import get_adapter
+    checks: list[dict[str, Any]] = []
+    for agent in agents:
+        if agent.get("paused"):
+            continue
+        if agent.get("auth_mode") == "compatible_api" and agent.get("provider") == "codex":
+            checks.append(
+                {
+                    "ok": True,
+                    "status": "profile_model_deferred_to_smoke",
+                    "provider": agent["provider"],
+                    "model": effective_model(agent, workspace),
+                    "agent_id": agent["id"],
+                }
+            )
+            continue
+        adapter = get_adapter(agent["provider"])
+        validator = getattr(adapter, "validate_model", None)
+        model = effective_model(agent, workspace)
+        if not callable(validator):
+            result = {"ok": True, "status": "not_checked", "provider": agent["provider"], "model": model}
+        else:
+            result = validator(model)
+            if not isinstance(result, dict):
+                result = {"ok": True, "status": "not_checked", "provider": agent["provider"], "model": model}
+        result = dict(result)
+        result.setdefault("provider", agent["provider"])
+        result.setdefault("model", model)
+        result["agent_id"] = agent["id"]
+        checks.append(result)
+    return checks
+
+
+def compact_model_checks(checks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    compact: list[dict[str, Any]] = []
+    for item in checks:
+        compact.append(
+            {
+                key: item.get(key)
+                for key in ["agent_id", "provider", "model", "ok", "status", "reason", "suggested_model", "command"]
+                if key in item
+            }
+        )
+    return compact
+
+
+def format_model_check_failures(failures: list[dict[str, Any]]) -> str:
+    lines = ["model validation failed before starting worker windows:"]
+    for item in failures:
+        message = f"{item.get('agent_id')}: provider={item.get('provider')} model={item.get('model')!r}"
+        if item.get("suggested_model"):
+            message += f" is not an exact model id; use {item['suggested_model']!r}"
+        else:
+            message += f" is unsupported ({item.get('reason') or item.get('status')})"
+        lines.append(message)
+    return "\n".join(lines)
+
+
+def format_profile_check_failures(failures: list[dict[str, Any]]) -> str:
+    lines = ["profile validation failed before starting worker windows:"]
+    for item in failures:
+        message = f"{item.get('agent_id')}: profile={item.get('profile')!r} auth_mode={item.get('auth_mode')}"
+        if item.get("missing_required"):
+            message += f" missing {', '.join(item['missing_required'])}"
+        else:
+            message += f" failed ({item.get('reason') or item.get('status')})"
+        if item.get("suggestion"):
+            message += f"; {item['suggestion']}"
+        lines.append(message)
+    return "\n".join(lines)
+
+
+def format_profile_smoke_failures(failures: list[dict[str, Any]]) -> str:
+    lines = ["provider profile smoke check failed before starting worker windows:"]
+    for item in failures:
+        message = f"{item.get('agent_id')}: provider={item.get('provider')} profile={item.get('profile')!r}"
+        message += f" status={item.get('status')} reason={item.get('reason') or 'unknown'}"
+        if item.get("http_status"):
+            message += f" http_status={item['http_status']}"
+        if item.get("error"):
+            message += f"; {item['error']}"
+        message += "; fix the local profile file or model id, then start again"
+        lines.append(message)
+    return "\n".join(lines)