@team-agent/installer 0.2.10 → 0.3.0

package/src/team_agent/messaging/scheduler.py

@@ -1,450 +0,0 @@
-from __future__ import annotations
-
-from team_agent.messaging.deps import (
-    EventLog,
-    MessageStore,
-    check_team_owner,
-    datetime,
-    json,
-    load_runtime_state,
-    load_spec,
-    save_runtime_state,
-    send_message,
-    team_state_key,
-    timedelta,
-    timezone,
-)
-from team_agent.messaging.activity_detector import classify_agent_activity, detect_compaction_degradation
-from team_agent.messaging.internal_delivery import deliver_stored_message
-from team_agent.messaging.result_delivery import delivered_result_message, result_id_from_text
-from team_agent.state import team_state_candidates
-
-from pathlib import Path
-from typing import Any
-
-_ACTIVE_TASK_STATUSES = {"pending", "assigned", "in_progress", "ready", "running", "needs_retry"}
-_INBOUND_WORK_STATUSES = {"pending", "accepted", "target_resolved", "injected"}
-_DELIVERED_MESSAGE_STATUSES = {"visible", "submitted", "delivered", "acknowledged"}
-_PROGRESS_EVENTS = {
-    "mcp.report_result",
-    "report_result.accepted",
-    "send.deliver_attempt",
-    "send.submitted",
-    "leader_receiver.deliver_attempt",
-    "leader_receiver.submitted",
-    "communication.peer_mirrored",
-}
-_RESTART_RESET_EVENTS = {"restart.agent_start", "restart.complete", "reset_agent.complete", "start_agent.complete"}
-_ALERT_TYPES = {"stuck", "idle_fallback", "cross_worker_deadlock"}
-
-
-def _fire_due_scheduled_events(workspace: Path, store: MessageStore, event_log: EventLog) -> list[int]:
-    fired: list[int] = []
-    for row in store.due_scheduled_events():
-        payload = json.loads(row["payload_json"] or "{}")
-        try:
-            if row["kind"] == "send":
-                content = str(payload.get("content") or "")
-                result_id = result_id_from_text(content)
-                existing = delivered_result_message(
-                    store,
-                    result_id or "",
-                    task_id=payload.get("task_id"),
-                    owner_team_id=row.get("owner_team_id"),
-                )
-                if existing:
-                    result = {
-                        "ok": True,
-                        "status": "already_delivered",
-                        "message_id": existing.get("message_id"),
-                        "deduped": True,
-                    }
-                    event_log.write(
-                        "coordinator.scheduled_result_deduped",
-                        id=row["id"],
-                        target=row["target"],
-                        result_id=result_id,
-                        message_id=existing.get("message_id"),
-                    )
-                    store.mark_scheduled_event(int(row["id"]), "done", result)
-                    fired.append(int(row["id"]))
-                    continue
-                deliver = deliver_stored_message if row.get("owner_team_id") else send_message
-                result = deliver(
-                    workspace,
-                    row["target"],
-                    content,
-                    task_id=payload.get("task_id"),
-                    sender=payload.get("sender", "coordinator"),
-                    requires_ack=bool(payload.get("requires_ack", True)),
-                    wait_visible=bool(payload.get("wait_visible", True)),
-                    timeout=float(payload.get("timeout", 30)),
-                    team=row.get("owner_team_id"),
-                )
-            elif row["kind"] == "health_ping":
-                result = {"ok": True, "status": "logged"}
-                event_log.write("coordinator.health_ping", target=row["target"], payload=payload)
-            elif row["kind"] == "trust_retry":
-                # Spark MEDIUM sweep #3 (2026-05-26) — bounded-backoff consumer
-                # for delivery.py:_handle_trust_retry_needed. payload carries the
-                # message_id and current attempt; _execute_trust_retry resets the
-                # row to 'accepted', re-runs _deliver_pending_message with the
-                # attempt threaded through, and either delivers, reschedules, or
-                # hits the terminal trust_auto_answer_exhausted branch.
-                from team_agent.messaging.delivery import _execute_trust_retry
-                result = _execute_trust_retry(
-                    workspace, store, event_log, payload,
-                    owner_team_id=row.get("owner_team_id"),
-                )
-            else:
-                result = {"ok": False, "error": f"unknown scheduled event kind: {row['kind']}"}
-            if not result.get("ok") and row["kind"] == "send":
-                retry = _schedule_send_retry(store, row, payload, result)
-                if retry:
-                    result = {**result, **retry}
-                    store.mark_scheduled_event(int(row["id"]), "retry_scheduled", result)
-                    event_log.write(
-                        "coordinator.scheduled_retry",
-                        id=row["id"],
-                        retry_event_id=retry["retry_event_id"],
-                        target=row["target"],
-                        attempt=retry["next_attempt"],
-                    )
-                    fired.append(int(row["id"]))
-                    continue
-            store.mark_scheduled_event(int(row["id"]), "done" if result.get("ok") else "failed", result)
-            fired.append(int(row["id"]))
-        except Exception as exc:
-            result = {"ok": False, "error": str(exc)}
-            store.mark_scheduled_event(int(row["id"]), "failed", result)
-            event_log.write("coordinator.scheduled_failed", id=row["id"], error=str(exc))
-    return fired
-
-
-def _schedule_send_retry(
-    store: MessageStore,
-    row: dict[str, Any],
-    payload: dict[str, Any],
-    result: dict[str, Any],
-) -> dict[str, Any] | None:
-    attempt = int(payload.get("attempt") or 1)
-    max_attempts = int(payload.get("max_attempts") or 1)
-    if attempt >= max_attempts:
-        return None
-    retry_payload = dict(payload)
-    retry_payload["attempt"] = attempt + 1
-    due_at = datetime.now(timezone.utc) + timedelta(seconds=min(2 * attempt, 5))
-    retry_id = store.add_scheduled_event(due_at.isoformat(), row["target"], row["kind"], retry_payload, owner_team_id=row.get("owner_team_id"))
-    return {
-        "retry_event_id": retry_id,
-        "next_attempt": attempt + 1,
-        "max_attempts": max_attempts,
-        "retry_reason": result.get("reason") or result.get("error"),
-    }
-
-
-def _detect_stuck_agents(
-    workspace: Path,
-    state: dict[str, Any],
-    store: MessageStore,
-    event_log: EventLog,
-) -> list[str]:
-    spec_path = Path(state.get("spec_path", workspace / "team.spec.yaml"))
-    spec = load_spec(spec_path) if spec_path.exists() else {}
-    runtime_cfg = spec.get("runtime", {})
-    stuck_timeout = int(runtime_cfg.get("stuck_timeout_sec", 300))
-    push_min_interval = int(runtime_cfg.get("push_min_interval_sec", 60))
-    owner_team_id = team_state_key(state)
-    health = store.agent_health(owner_team_id=owner_team_id)
-    stuck: list[str] = []
-    now = datetime.now(timezone.utc)
-    for agent_id, row in health.items():
-        if row.get("status") not in {"RUNNING", "WORKING"} or not row.get("last_output_at"):
-            continue
-        try:
-            last = datetime.fromisoformat(row["last_output_at"])
-        except ValueError:
-            continue
-        if last.tzinfo is None:
-            last = last.replace(tzinfo=timezone.utc)
-        if (now - last).total_seconds() < stuck_timeout:
-            continue
-        suppression = _active_alert_suppression(state, store, event_log, agent_id, "stuck")
-        has_work, work_reason = _agent_has_stuck_relevant_work(state, store, agent_id)
-        if not has_work:
-            event_log.write("coordinator.agent_stuck_suppressed", agent_id=agent_id, reason="idle_no_work", last_output_at=row["last_output_at"])
-            continue
-        if suppression:
-            continue
-        progress_event = _recent_agent_progress_event(event_log, agent_id, last)
-        if progress_event:
-            event_log.write(
-                "coordinator.agent_stuck_suppressed",
-                agent_id=agent_id,
-                reason="recent_progress_event",
-                progress_event=progress_event.get("event"),
-                progress_ts=progress_event.get("ts"),
-                last_output_at=row["last_output_at"],
-                work_reason=work_reason,
-            )
-            continue
-        stuck.append(agent_id)
-        state.setdefault("coordinator", {})
-        push_key = f"last_stuck_push_at:{agent_id}"
-        last_push_raw = state["coordinator"].get(push_key)
-        should_push = True
-        if last_push_raw:
-            try:
-                last_push = datetime.fromisoformat(last_push_raw)
-                if last_push.tzinfo is None:
-                    last_push = last_push.replace(tzinfo=timezone.utc)
-                should_push = (now - last_push).total_seconds() >= push_min_interval
-            except ValueError:
-                should_push = True
-        event_log.write("coordinator.agent_stuck", agent_id=agent_id, last_output_at=row["last_output_at"], work_reason=work_reason)
-        if should_push:
-            state["coordinator"][push_key] = now.isoformat()
-            try:
-                send_message(
-                    workspace,
-                    "leader",
-                    f"agent {agent_id} appears stuck: no output for {stuck_timeout}s",
-                    sender="coordinator",
-                    requires_ack=False,
-                    wait_visible=False,
-                    team=owner_team_id,
-                )
-            except Exception as exc:
-                event_log.write("coordinator.stuck_push_failed", agent_id=agent_id, error=str(exc))
-    return stuck
-
-
-def stuck_list(workspace: Path) -> dict[str, Any]:
-    state = load_runtime_state(workspace)
-    suppressed = state.get("coordinator", {}).get("suppressed_idle_alerts", {})
-    if _use_team_scoped_suppressions(state):
-        from team_agent.state import _caller_identity_from_env
-        caller = _caller_identity_from_env()
-        candidates = team_state_candidates(state)
-        caller_team = None
-        if caller.get("pane_id"):
-            for key, candidate in candidates.items():
-                owner = candidate.get("team_owner") or {}
-                if (
-                    caller["pane_id"] == (owner.get("pane_id") or "")
-                    and caller["provider"] == (owner.get("provider") or "")
-                    and caller["machine_fingerprint"] == (owner.get("machine_fingerprint") or "")
-                ):
-                    caller_team = key
-                    break
-        if caller_team is None:
-            return {
-                "ok": False,
-                "status": "refused",
-                "reason": "team_owner_unresolved",
-                "action": "set TEAM_AGENT_LEADER_PANE_ID/PROVIDER/MACHINE_FINGERPRINT to your team's claimed identity, or use team-agent takeover --confirm",
-                "candidates": sorted(list(candidates)),
-            }
-        return {"ok": True, "suppressed_idle_alerts": suppressed.get(caller_team, {}), "team": caller_team}
-    known_team_keys = set(team_state_candidates(state).keys())
-    has_team_keys = bool(known_team_keys & set(suppressed.keys()))
-    if not has_team_keys and (
-        len(suppressed) == 1
-        and all(isinstance(value, dict) for value in suppressed.values())
-        and not any(isinstance(value, dict) and set(value) & _ALERT_TYPES for value in suppressed.values())
-    ):
-        only = next(iter(suppressed.values()))
-        if all(isinstance(value, dict) for value in only.values()):
-            suppressed = only
-    return {"ok": True, "suppressed_idle_alerts": suppressed}
-
-
-def stuck_cancel(
-    workspace: Path,
-    agent_id: str,
-    alert_type: str = "stuck",
-    suppressed_by: str = "leader",
-) -> dict[str, Any]:
-    if alert_type == "all":
-        alert_types = sorted(_ALERT_TYPES)
-    elif alert_type in _ALERT_TYPES:
-        alert_types = [alert_type]
-    else:
-        return {"ok": False, "status": "refused", "reason": "invalid_alert_type", "alert_type": alert_type}
-    state = load_runtime_state(workspace)
-    gate = check_team_owner(state)
-    if gate:
-        return gate
-    store = MessageStore(workspace)
-    owner_team_id = team_state_key(state)
-    coordinator = state.setdefault("coordinator", {})
-    suppressed = coordinator.setdefault("suppressed_idle_alerts", {})
-    team_suppressions = suppressed.setdefault(owner_team_id, {}) if _use_team_scoped_suppressions(state) else suppressed
-    agent_suppressions = team_suppressions.setdefault(agent_id, {})
-    now = datetime.now(timezone.utc).isoformat()
-    snapshot = _agent_alert_snapshot(state, store, agent_id, owner_team_id)
-    for item in alert_types:
-        agent_suppressions[item] = {
-            "suppressed_at": now,
-            "suppressed_by": suppressed_by,
-            "snapshot": snapshot,
-        }
-    save_runtime_state(workspace, state)
-    EventLog(workspace).write("coordinator.idle_alert_suppressed", agent_id=agent_id, alert_types=alert_types, suppressed_by=suppressed_by)
-    return {"ok": True, "agent_id": agent_id, "alert_types": alert_types, "suppressed": agent_suppressions}
-
-
-def _active_alert_suppression(
-    state: dict[str, Any],
-    store: MessageStore,
-    event_log: EventLog,
-    agent_id: str,
-    alert_type: str,
-) -> dict[str, Any] | None:
-    owner_team_id = team_state_key(state)
-    suppressed = state.get("coordinator", {}).get("suppressed_idle_alerts", {})
-    entry = suppressed.get(owner_team_id, {}).get(agent_id, {}).get(alert_type)
-    if not isinstance(entry, dict):
-        entry = suppressed.get(agent_id, {}).get(alert_type)
-    if not isinstance(entry, dict):
-        return None
-    cleared = _suppression_clear_reason(state, store, event_log, agent_id, entry)
-    if cleared:
-        _clear_alert_suppression(state, agent_id, alert_type, owner_team_id)
-        event_log.write("coordinator.idle_alert_suppression_cleared", agent_id=agent_id, alert_type=alert_type, reason=cleared)
-        return None
-    return entry
-
-
-def _suppression_clear_reason(
-    state: dict[str, Any],
-    store: MessageStore,
-    event_log: EventLog,
-    agent_id: str,
-    entry: dict[str, Any],
-) -> str | None:
-    if entry.get("manual_acknowledge"):
-        try:
-            expires_at = datetime.fromisoformat(str(entry.get("expires_at")))
-        except ValueError:
-            return "invalid_suppression_timestamp"
-        if expires_at.tzinfo is None:
-            expires_at = expires_at.replace(tzinfo=timezone.utc)
-        if datetime.now(timezone.utc) < expires_at:
-            return None
-        return "manual_acknowledge_expired"
-    previous = entry.get("snapshot") if isinstance(entry.get("snapshot"), dict) else {}
-    current = _agent_alert_snapshot(state, store, agent_id)
-    if current.get("assigned_task_ids") != previous.get("assigned_task_ids"):
-        return "task_assignment_changed"
-    if current.get("delivered_message_ids") != previous.get("delivered_message_ids"):
-        return "inbound_delivery_changed"
-    try:
-        suppressed_at = datetime.fromisoformat(str(entry.get("suppressed_at")))
-    except ValueError:
-        return "invalid_suppression_timestamp"
-    if suppressed_at.tzinfo is None:
-        suppressed_at = suppressed_at.replace(tzinfo=timezone.utc)
-    if _recent_agent_progress_event(event_log, agent_id, suppressed_at):
-        return "progress_event"
-    if _recent_restart_or_reset_event(event_log, agent_id, suppressed_at):
-        return "restart_or_reset"
-    return None
-
-
-def _clear_alert_suppression(state: dict[str, Any], agent_id: str, alert_type: str, owner_team_id: str | None = None) -> None:
-    suppressed = state.get("coordinator", {}).get("suppressed_idle_alerts", {})
-    if agent_id in suppressed:
-        agent_suppressions = suppressed.get(agent_id, {})
-        agent_suppressions.pop(alert_type, None)
-        if not agent_suppressions:
-            suppressed.pop(agent_id, None)
-        return
-    team_suppressions = suppressed.get(owner_team_id or team_state_key(state), {})
-    agent_suppressions = team_suppressions.get(agent_id, {})
-    agent_suppressions.pop(alert_type, None)
-    if not agent_suppressions:
-        team_suppressions.pop(agent_id, None)
-    if not team_suppressions:
-        suppressed.pop(owner_team_id or team_state_key(state), None)
-
-
-def _use_team_scoped_suppressions(state: dict[str, Any]) -> bool:
-    return len(team_state_candidates(state)) > 1
-
-
-def _agent_alert_snapshot(state: dict[str, Any], store: MessageStore, agent_id: str, owner_team_id: str | None = None) -> dict[str, Any]:
-    assigned_task_ids = sorted(str(task.get("id")) for task in state.get("tasks", []) if task.get("assignee") == agent_id)
-    delivered_message_ids = sorted(
-        str(message.get("message_id"))
-        for message in store.messages(owner_team_id=owner_team_id or team_state_key(state))
-        if message.get("recipient") == agent_id and message.get("status") in _DELIVERED_MESSAGE_STATUSES
-    )
-    return {"assigned_task_ids": assigned_task_ids, "delivered_message_ids": delivered_message_ids}
-
-
-def _agent_has_stuck_relevant_work(state: dict[str, Any], store: MessageStore, agent_id: str) -> tuple[bool, str]:
-    for task in state.get("tasks", []):
-        if task.get("assignee") == agent_id and task.get("status", "pending") in _ACTIVE_TASK_STATUSES:
-            return True, "active_task"
-    for message in store.messages(owner_team_id=team_state_key(state)):
-        if message.get("recipient") == agent_id and message.get("status") in _INBOUND_WORK_STATUSES:
-            return True, "inbound_message"
-    return False, "idle_no_work"
-
-
-def _recent_agent_progress_event(event_log: EventLog, agent_id: str, since: datetime) -> dict[str, Any] | None:
-    for event in reversed(event_log.tail(200)):
-        if event.get("event") not in _PROGRESS_EVENTS:
-            continue
-        if not _event_mentions_agent(event, agent_id):
-            continue
-        try:
-            ts = datetime.fromisoformat(str(event.get("ts")))
-        except ValueError:
-            continue
-        if ts.tzinfo is None:
-            ts = ts.replace(tzinfo=timezone.utc)
-        if ts >= since:
-            return event
-    return None
-
-
-def _event_mentions_agent(event: dict[str, Any], agent_id: str) -> bool:
-    if event.get("agent_id") == agent_id or event.get("sender") == agent_id or event.get("target") == agent_id:
-        return True
-    payload = event.get("payload")
-    return isinstance(payload, dict) and (payload.get("from") == agent_id or payload.get("to") == agent_id)
-
-
-def _recent_restart_or_reset_event(event_log: EventLog, agent_id: str, since: datetime) -> dict[str, Any] | None:
-    for event in reversed(event_log.tail(200)):
-        if event.get("event") not in _RESTART_RESET_EVENTS:
-            continue
-        if event.get("agent_id") != agent_id:
-            agents_field = event.get("agents") or []
-            agent_ids: set[str] = set()
-            for entry in agents_field:
-                if isinstance(entry, str):
-                    agent_ids.add(entry)
-                elif isinstance(entry, dict):
-                    aid = entry.get("agent_id")
-                    if isinstance(aid, str):
-                        agent_ids.add(aid)
-            if agent_id not in agent_ids:
-                continue
-        try:
-            ts = datetime.fromisoformat(str(event.get("ts")))
-        except ValueError:
-            continue
-        if ts.tzinfo is None:
-            ts = ts.replace(tzinfo=timezone.utc)
-        if ts >= since:
-            return event
-    return None
-
-
-from team_agent.messaging.idle_alerts import (
-    detect_cross_worker_deadlocks,
-    detect_idle_fallbacks,
-)