@team-agent/installer 0.1.11 → 0.2.1

package/src/team_agent/messaging/idle_alerts.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+from team_agent.events import EventLog
+from team_agent.message_store import MessageStore
+from team_agent.messaging.deps import load_spec, save_runtime_state, team_state_key
+from team_agent.messaging.internal_delivery import deliver_stored_message
+
+
+_UNDELIVERED_MESSAGE_STATUSES = {
+    "pending",
+    "accepted",
+    "queued_until_idle",
+    "queued_until_start",
+    "queued_stopped",
+    "queued_pane_missing",
+    "failed",
+    "delivery_blocked",
+    "injected_unverified",
+}
+
+
+STABLE_IDLE_SECONDS = 120
+FIRE_DEBOUNCE_SECONDS = 300
+OBLIGATION_PENDING_MIN_AGE_SECONDS = 60
+
+
+def _parse_iso(text: Any) -> datetime | None:
+    if not isinstance(text, str) or not text:
+        return None
+    try:
+        dt = datetime.fromisoformat(text.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def record_team_progress(
+    state: dict[str, Any],
+    now: datetime | None = None,
+    *,
+    source: str = "",
+    owner_team_id: str | None = None,
+) -> None:
+    coordinator = state.setdefault("coordinator", {})
+    progress = coordinator.setdefault("team_last_progress_at", {})
+    key = owner_team_id or team_state_key(state)
+    if not key:
+        return
+    progress[key] = {
+        "at": (now or datetime.now(timezone.utc)).isoformat(),
+        "source": source,
+    }
+
+
+def _team_last_progress_at(
+    state: dict[str, Any],
+    store: MessageStore,
+    owner_team_id: str,
+) -> datetime | None:
+    candidates: list[datetime] = []
+    coordinator = state.get("coordinator") or {}
+    explicit = (coordinator.get("team_last_progress_at") or {}).get(owner_team_id)
+    if isinstance(explicit, dict):
+        ts = _parse_iso(explicit.get("at"))
+        if ts:
+            candidates.append(ts)
+    elif isinstance(explicit, str):
+        ts = _parse_iso(explicit)
+        if ts:
+            candidates.append(ts)
+    health = store.agent_health(owner_team_id=owner_team_id)
+    for row in health.values():
+        ts = _parse_iso(row.get("last_output_at"))
+        if ts:
+            candidates.append(ts)
+    return max(candidates) if candidates else None
+
+
+def _team_last_idle_fallback_fire_at(state: dict[str, Any], owner_team_id: str) -> datetime | None:
+    coordinator = state.get("coordinator") or {}
+    fires = coordinator.get("team_last_idle_fallback_fire_at") or {}
+    return _parse_iso(fires.get(owner_team_id))
+
+
+def _record_idle_fallback_fire(state: dict[str, Any], owner_team_id: str, now: datetime) -> None:
+    coordinator = state.setdefault("coordinator", {})
+    fires = coordinator.setdefault("team_last_idle_fallback_fire_at", {})
+    fires[owner_team_id] = now.isoformat()
+
+
+def _team_undelivered_obligations(
+    state: dict[str, Any],
+    store: MessageStore,
+    owner_team_id: str,
+    active_task_statuses: set[str],
+    *,
+    now: datetime | None = None,
+) -> list[dict[str, Any]]:
+    now = now or datetime.now(timezone.utc)
+    min_age = timedelta(seconds=OBLIGATION_PENDING_MIN_AGE_SECONDS)
+    obligations: list[dict[str, Any]] = []
+    for message in store.messages(owner_team_id=owner_team_id):
+        if message.get("status") not in _UNDELIVERED_MESSAGE_STATUSES:
+            continue
+        created_at = _parse_iso(message.get("created_at"))
+        if created_at and (now - created_at) < min_age:
+            continue
+        obligations.append(
+            {
+                "kind": "undelivered_message",
+                "message_id": message.get("message_id"),
+                "recipient": message.get("recipient"),
+                "status": message.get("status"),
+            }
+        )
+    for watcher in store.retryable_result_watchers():
+        if watcher.get("status") not in {"pending", "notify_failed"}:
+            continue
+        created_at = _parse_iso(watcher.get("created_at"))
+        if created_at and (now - created_at) < min_age:
+            continue
+        obligations.append(
+            {
+                "kind": "pending_result_watcher",
+                "watcher_id": watcher.get("watcher_id"),
+                "task_id": watcher.get("task_id"),
+                "agent_id": watcher.get("agent_id"),
+            }
+        )
+    for task in state.get("tasks", []):
+        if task.get("status", "pending") in active_task_statuses and task.get("assignee"):
+            obligations.append(
+                {
+                    "kind": "active_task",
+                    "task_id": task.get("id"),
+                    "assignee": task.get("assignee"),
+                    "status": task.get("status"),
+                }
+            )
+    return obligations
+
+
+def _all_workers_idle(
+    state: dict[str, Any],
+    store: MessageStore,
+    owner_team_id: str,
+) -> tuple[bool, list[str]]:
+    health = store.agent_health(owner_team_id=owner_team_id)
+    worker_ids = list(state.get("agents", {}).keys()) or list(health.keys())
+    if not worker_ids:
+        return False, []
+    idle: list[str] = []
+    for agent_id in worker_ids:
+        row = health.get(agent_id) or {}
+        status = str(row.get("status") or "").lower()
+        if status != "idle":
+            return False, []
+        idle.append(agent_id)
+    return True, idle
+
+
+def _register_unified_alert(
+    state: dict[str, Any],
+    owner_team_id: str,
+    agent_id: str,
+    alert_type: str,
+    snapshot: dict[str, Any],
+    suppressed_by: str,
+    now: datetime,
+) -> dict[str, Any]:
+    coordinator = state.setdefault("coordinator", {})
+    suppressed = coordinator.setdefault("suppressed_idle_alerts", {})
+    team_suppressions = suppressed.setdefault(owner_team_id, {})
+    agent_suppressions = team_suppressions.setdefault(agent_id, {})
+    entry = {
+        "suppressed_at": now.isoformat(),
+        "suppressed_by": suppressed_by,
+        "snapshot": snapshot,
+    }
+    agent_suppressions[alert_type] = entry
+    return entry
+
+
+def detect_idle_fallbacks(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    event_log: EventLog,
+    now: datetime | None = None,
+) -> list[dict[str, Any]]:
+    from team_agent.messaging.scheduler import (
+        _ACTIVE_TASK_STATUSES,
+        _active_alert_suppression,
+        _agent_alert_snapshot,
+    )
+    now = now or datetime.now(timezone.utc)
+    owner_team_id = team_state_key(state)
+    obligations = _team_undelivered_obligations(state, store, owner_team_id, _ACTIVE_TASK_STATUSES, now=now)
+    if not obligations:
+        return []
+    all_idle, idle_workers = _all_workers_idle(state, store, owner_team_id)
+    if not all_idle:
+        record_team_progress(state, now, source="all_workers_idle:false", owner_team_id=owner_team_id)
+        save_runtime_state(workspace, state)
+        return []
+    last_progress = _team_last_progress_at(state, store, owner_team_id)
+    if last_progress and (now - last_progress) < timedelta(seconds=STABLE_IDLE_SECONDS):
+        event_log.write(
+            "coordinator.idle_fallback_skipped",
+            reason="stable_idle_window",
+            team=owner_team_id,
+            stable_idle_seconds=STABLE_IDLE_SECONDS,
+            elapsed_seconds=int((now - last_progress).total_seconds()),
+        )
+        return []
+    last_fire = _team_last_idle_fallback_fire_at(state, owner_team_id)
+    if last_fire and (now - last_fire) < timedelta(seconds=FIRE_DEBOUNCE_SECONDS):
+        event_log.write(
+            "coordinator.idle_fallback_skipped",
+            reason="fire_debounce",
+            team=owner_team_id,
+            fire_debounce_seconds=FIRE_DEBOUNCE_SECONDS,
+            elapsed_seconds=int((now - last_fire).total_seconds()),
+        )
+        return []
+    spec_path = Path(state.get("spec_path", workspace / "team.spec.yaml"))
+    spec = load_spec(spec_path) if spec_path.exists() else {}
+    leader_id = state.get("leader", {}).get("id") or spec.get("leader", {}).get("id") or "leader"
+    alerts: list[dict[str, Any]] = []
+    for agent_id in idle_workers:
+        suppression = _active_alert_suppression(state, store, event_log, agent_id, "idle_fallback")
+        if suppression:
+            continue
+        snapshot = _agent_alert_snapshot(state, store, agent_id, owner_team_id)
+        _register_unified_alert(state, owner_team_id, agent_id, "idle_fallback", snapshot, "coordinator", now)
+        alerts.append({"agent_id": agent_id, "alert_type": "idle_fallback", "obligations": obligations})
+    if not alerts:
+        return []
+    _record_idle_fallback_fire(state, owner_team_id, now)
+    save_runtime_state(workspace, state)
+    content = (
+        "There is still unfinished work. Continue coordinating, deliver a result, "
+        "or acknowledge that this idle state is intentional via team-agent acknowledge-idle."
+    )
+    try:
+        deliver_stored_message(
+            workspace,
+            leader_id,
+            content,
+            sender="coordinator",
+            requires_ack=False,
+            wait_visible=False,
+            team=owner_team_id,
+        )
+    except Exception as exc:
+        event_log.write("coordinator.idle_fallback_push_failed", error=str(exc), team=owner_team_id)
+    event_log.write(
+        "coordinator.idle_fallback",
+        team=owner_team_id,
+        idle_workers=idle_workers,
+        obligation_count=len(obligations),
+        alert_count=len(alerts),
+    )
+    return alerts
+
+
+def detect_cross_worker_deadlocks(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    event_log: EventLog,
+    now: datetime | None = None,
+) -> list[dict[str, Any]]:
+    from team_agent.messaging.scheduler import (
+        _active_alert_suppression,
+        _agent_alert_snapshot,
+    )
+    now = now or datetime.now(timezone.utc)
+    owner_team_id = team_state_key(state)
+    health = store.agent_health(owner_team_id=owner_team_id)
+    candidate_recipients: dict[str, list[dict[str, Any]]] = {}
+    for message in store.messages(owner_team_id=owner_team_id):
+        if message.get("status") not in _UNDELIVERED_MESSAGE_STATUSES:
+            continue
+        recipient = message.get("recipient")
+        if not recipient:
+            continue
+        candidate_recipients.setdefault(str(recipient), []).append(message)
+    alerts: list[dict[str, Any]] = []
+    for agent_id, messages in candidate_recipients.items():
+        row = health.get(agent_id) or {}
+        status = str(row.get("status") or "").lower()
+        if status != "idle":
+            continue
+        suppression = _active_alert_suppression(state, store, event_log, agent_id, "cross_worker_deadlock")
+        if suppression:
+            continue
+        snapshot = _agent_alert_snapshot(state, store, agent_id, owner_team_id)
+        snapshot["pending_message_ids"] = sorted(str(m.get("message_id")) for m in messages)
+        _register_unified_alert(state, owner_team_id, agent_id, "cross_worker_deadlock", snapshot, "coordinator", now)
+        alerts.append(
+            {
+                "agent_id": agent_id,
+                "alert_type": "cross_worker_deadlock",
+                "pending_messages": snapshot["pending_message_ids"],
+            }
+        )
+    if not alerts:
+        return []
+    save_runtime_state(workspace, state)
+    event_log.write(
+        "coordinator.cross_worker_deadlock",
+        team=owner_team_id,
+        agent_ids=[alert["agent_id"] for alert in alerts],
+        alert_count=len(alerts),
+    )
+    return alerts

package/src/team_agent/messaging/internal_delivery.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from team_agent.messaging.deps import (
+    EventLog,
+    _runtime_lock,
+    load_spec,
+    select_runtime_state,
+    team_state_key,
+)
+from team_agent.messaging.send import _send_single_message_unlocked
+
+
+def deliver_stored_message(
+    workspace: Path,
+    target: str | None,
+    content: str,
+    *,
+    task_id: str | None = None,
+    sender: str = "coordinator",
+    requires_ack: bool = False,
+    wait_visible: bool = False,
+    timeout: float = 30.0,
+    team: str | None = None,
+) -> dict[str, Any]:
+    with _runtime_lock(workspace, "send"):
+        state = select_runtime_state(workspace, team)
+        spec_path = Path(state.get("spec_path", workspace / "team.spec.yaml"))
+        spec = load_spec(spec_path)
+        return _send_single_message_unlocked(
+            workspace,
+            state,
+            spec,
+            EventLog(workspace),
+            target,
+            content,
+            task_id=task_id,
+            sender=sender,
+            requires_ack=requires_ack,
+            wait_visible=wait_visible,
+            timeout=timeout,
+            route_task_id=False,
+            owner_team_id=team_state_key(state),
+        )

package/src/team_agent/messaging/leader.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+from team_agent.messaging.deps import (
+    EventLog,
+    MessageStore,
+    _choose_leader_submit_key,
+    _leader_id,
+    _rediscover_leader_receiver,
+    _tmux_inject_text,
+    _validate_leader_receiver,
+    core_render_message,
+    json,
+    runtime_dir,
+    save_runtime_state,
+    time,
+)
+
+from pathlib import Path
+from typing import Any
+
+def allow_peer_talk(workspace: Path, agent_a: str, agent_b: str) -> dict[str, Any]:
+    MessageStore(workspace).allow_peer(agent_a, agent_b)
+    EventLog(workspace).write("communication.peer_allowed", a=agent_a, b=agent_b)
+    return {"ok": True, "a": agent_a, "b": agent_b, "status": "compat_noop", "reason": "team_scoped_peer_messages_enabled"}
+
+
+def _mirror_peer_message_to_leader(
+    workspace: Path,
+    state: dict[str, Any],
+    sender: str,
+    target: str,
+    content: str,
+    task_id: str | None,
+    event_log: EventLog,
+) -> None:
+    leader_id = _leader_id(state, {})
+    mirror = f"Team Agent peer message from {sender} to {target}"
+    if task_id:
+        mirror += f" for {task_id}"
+    mirror += f":\n\n{content}"
+    try:
+        result = _send_to_leader_receiver(workspace, state, leader_id, mirror, task_id, sender, False, event_log)
+        event_log.write("communication.peer_mirrored", sender=sender, target=target, ok=result.get("ok"))
+    except Exception as exc:
+        event_log.write("communication.peer_mirror_failed", sender=sender, target=target, error=str(exc))
+
+
+def _leader_inbox_path(workspace: Path) -> Path:
+    return runtime_dir(workspace) / "leader-inbox.log"
+
+
+def _send_to_leader_receiver(
+    workspace: Path,
+    state: dict[str, Any],
+    leader_id: str,
+    content: str,
+    task_id: str | None,
+    sender: str,
+    requires_ack: bool,
+    event_log: EventLog,
+) -> dict[str, Any]:
+    store = MessageStore(workspace)
+    message_id = store.create_message(task_id, sender, leader_id, content, requires_ack=False)
+    if requires_ack:
+        event_log.write("leader_receiver.no_ack_forced", message_id=message_id, requested_requires_ack=True)
+    row = _message_by_id(store, message_id)
+    if not row:
+        return {"ok": False, "message_id": message_id, "status": "failed", "to": leader_id, "reason": "message_missing"}
+    if not store.claim_for_delivery(message_id):
+        current = _message_by_id(store, message_id)
+        status = current["status"] if current else "missing"
+        event_log.write("leader_receiver.delivery_claim_skipped", message_id=message_id, status=status)
+        return {
+            "ok": status in {"submitted", "visible", "delivered", "acknowledged"},
+            "message_id": message_id,
+            "status": status,
+            "to": leader_id,
+            "channel": "direct_tmux",
+            "reason": "message_already_claimed",
+        }
+    payload = _message_payload(row)
+    rendered = core_render_message(payload)
+    text = rendered["text"]
+    receiver = state.get("leader_receiver", {})
+    if not _leader_receiver_is_direct(receiver):
+        return _fail_leader_delivery(
+            workspace,
+            state,
+            store,
+            message_id,
+            payload,
+            event_log,
+            reason="leader_not_attached",
+            error="No direct leader tmux pane is attached. Run team-agent attach-leader.",
+        )
+
+    validation = _validate_leader_receiver(receiver)
+    if not validation["ok"]:
+        owner_identity = state.get("team_owner") or None
+        rediscovery = _rediscover_leader_receiver(receiver, event_log, owner_identity)
+        if rediscovery.get("status") == "updated":
+            state["leader_receiver"].update(rediscovery["receiver"])
+            receiver = state["leader_receiver"]
+            validation = _validate_leader_receiver(receiver)
+        elif rediscovery.get("status") == "ambiguous":
+            return _fail_leader_delivery(
+                workspace,
+                state,
+                store,
+                message_id,
+                payload,
+                event_log,
+                reason="ambiguous",
+                error="multiple possible leader panes found; rerun team-agent attach-leader --pane <pane_id>",
+                message_status="ambiguous",
+            )
+    if not validation["ok"]:
+        return _fail_leader_delivery(
+            workspace,
+            state,
+            store,
+            message_id,
+            payload,
+            event_log,
+            reason=validation["reason"],
+            error=validation.get("error"),
+        )
+    state["leader_receiver"].update(validation["pane"])
+    submit_key, submit_reason = _choose_leader_submit_key(receiver.get("provider", "codex"), validation.get("capture", ""))
+    target = receiver["pane_id"]
+    event_log.write(
+        "leader_receiver.deliver_attempt",
+        message_id=message_id,
+        target=target,
+        provider=receiver.get("provider"),
+        submit_key=submit_key,
+        submit_reason=submit_reason,
+        render_engine=rendered.get("engine"),
+        visible_token=rendered.get("token"),
+        payload=payload,
+        warning=validation.get("warning"),
+    )
+    injection = _tmux_inject_text(
+        target,
+        text,
+        submit_key,
+        f"team-agent-leader-receiver-{message_id}",
+        provider=receiver.get("provider", "codex"),
+    )
+    if injection["ok"]:
+        store.mark(message_id, "submitted")
+        event_log.write(
+            "leader_receiver.submitted",
+            message_id=message_id,
+            sender=sender,
+            task_id=task_id,
+            target=target,
+            provider=receiver.get("provider"),
+            submit_key=submit_key,
+            submit_reason=submit_reason,
+            visible=True,
+            submitted=True,
+            visible_token=rendered.get("token"),
+            verification=injection.get("verification"),
+            submit_verification=injection.get("submit_verification"),
+            turn_verification=injection.get("turn_verification"),
+            attempts=injection.get("attempts"),
+            submit_attempts=injection.get("submit_attempts"),
+        )
+        save_runtime_state(workspace, state)
+        return {
+            "ok": True,
+            "message_id": message_id,
+            "status": "submitted",
+            "to": leader_id,
+            "channel": "direct_tmux",
+            "leader_receiver": state["leader_receiver"],
+            "submit_key": submit_key,
+            "visible": True,
+            "submitted": True,
+            "visible_token": rendered.get("token"),
+            "verification": injection.get("verification"),
+            "submit_verification": injection.get("submit_verification"),
+            "turn_verification": injection.get("turn_verification"),
+            "attempts": injection.get("attempts"),
+            "submit_attempts": injection.get("submit_attempts"),
+            "warning": "leader messages are no-ack; requires_ack was forced false" if requires_ack else None,
+        }
+    return _fail_leader_delivery(
+        workspace,
+        state,
+        store,
+        message_id,
+        payload,
+        event_log,
+        reason="tmux_injection_failed",
+        error=injection.get("error"),
+        stage=injection.get("stage"),
+        attempts=injection.get("attempts"),
+        submit_attempts=injection.get("submit_attempts"),
+    )
+
+
+def _fail_leader_delivery(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    message_id: str,
+    payload: dict[str, Any],
+    event_log: EventLog,
+    reason: str,
+    error: str | None = None,
+    stage: str | None = None,
+    message_status: str = "failed",
+    attempts: list[dict[str, Any]] | None = None,
+    submit_attempts: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    store.mark(message_id, message_status, error or reason)
+    fallback_path = _write_leader_fallback_audit(workspace, payload, reason, error)
+    event_log.write(
+        "leader_receiver.delivery_failed",
+        message_id=message_id,
+        target=state.get("leader_receiver", {}).get("pane_id"),
+        reason=reason,
+        error=error,
+        stage=stage,
+        attempts=attempts,
+        submit_attempts=submit_attempts,
+        fallback_path=str(fallback_path),
+        suggestion="Run team-agent attach-leader --workspace . --provider codex, or pass --pane <pane_id>.",
+    )
+    save_runtime_state(workspace, state)
+    return {
+        "ok": False,
+        "message_id": message_id,
+        "status": "fallback",
+        "message_status": message_status,
+        "to": payload["to"],
+        "channel": "fallback_inbox",
+        "reason": reason,
+        "error": error,
+        "attempts": attempts,
+        "submit_attempts": submit_attempts,
+        "fallback_path": str(fallback_path),
+        "suggestion": "Run team-agent attach-leader --workspace . --provider codex, or pass --pane <pane_id>.",
+    }
+
+
+def _write_leader_fallback_audit(workspace: Path, payload: dict[str, Any], reason: str, error: str | None) -> Path:
+    inbox_path = _leader_inbox_path(workspace)
+    inbox_path.parent.mkdir(parents=True, exist_ok=True)
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    text = core_render_message(payload)["text"]
+    with inbox_path.open("a", encoding="utf-8") as inbox:
+        inbox.write(f"\n[{stamp}] fallback reason={reason} error={error or '-'}\n{text}\n")
+    return inbox_path
+
+
+def _leader_receiver_is_direct(receiver: dict[str, Any] | None) -> bool:
+    return bool(receiver and receiver.get("mode") == "direct_tmux" and receiver.get("pane_id"))
+
+
+def _message_by_id(store: MessageStore, message_id: str) -> dict[str, Any] | None:
+    return next((m for m in store.messages() if m["message_id"] == message_id), None)
+
+
+def _message_payload(row: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "message_id": row["message_id"],
+        "task_id": row["task_id"],
+        "from": row["sender"],
+        "to": row["recipient"],
+        "reply_to": row["reply_to"],
+        "requires_ack": bool(row["requires_ack"]),
+        "artifact_refs": json.loads(row["artifact_refs"] or "[]"),
+        "content": row["content"],
+    }
+
+
+def _format_team_agent_message(payload: dict[str, Any]) -> str:
+    return core_render_message(payload)["text"]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+