@team-agent/installer 0.2.1 → 0.2.3

package/src/team_agent/messaging/delivery.py

@@ -10,15 +10,28 @@ from team_agent.messaging.deps import (
     core_render_message,
 )
 
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
+
+# Spark MEDIUM sweep #3 (2026-05-26): retry_needed bounded backoff. Each entry is
+# the delay (seconds) BEFORE the attempt with that number runs; attempt 1 was the
+# original delivery, attempt 2 fires 5s after retry_needed, attempt 3 fires 15s
+# after the previous, attempt 4 fires 30s after the previous. _TRUST_RETRY_MAX_ATTEMPTS
+# bounds the total — the 4th retry_needed is terminal and emits
+# leader_panes.trust_auto_answer_exhausted.
+_TRUST_RETRY_BACKOFF_SECONDS = {2: 5, 3: 15, 4: 30}
+_TRUST_RETRY_MAX_ATTEMPTS = 4
+
 def _deliver_pending_message(
     workspace: Path,
     state: dict[str, Any],
     message_id: str,
     wait_visible: bool = True,
     timeout: float = 30.0,
+    *,
+    _trust_retry_attempt: int = 1,
 ) -> dict[str, Any]:
     store = MessageStore(workspace)
     row = next((m for m in store.messages() if m["message_id"] == message_id), None)
@@ -65,9 +78,49 @@ def _deliver_pending_message(
         attempts=3 if wait_visible else 1,
         provider=agent_state.get("provider", "fake"),
     )
+    if not injection.get("ok") and injection.get("detected") == "codex_trust_prompt":
+        # Gap 29 (Stage 2): opt-in trust auto-answer. The helper enforces both the
+        # opt-in flag and a workspace-dir match before sending '1'+Enter, then we
+        # retry the original paste once the prompt has actually been dismissed.
+        # Bypassed entirely when opt-out (default) — the existing failed envelope
+        # is preserved.
+        from team_agent.messaging.leader_panes import attempt_trust_auto_answer
+        answer = attempt_trust_auto_answer(
+            workspace,
+            injection.get("pane_id") or target,
+            injection.get("pane_capture_tail") or "",
+            EventLog(workspace),
+            state=state,
+        )
+        if answer.get("answered"):
+            # Spark MEDIUM #4 (2026-05-26): replace the fixed 0.3s sleep with a
+            # bounded poll. Slow terminals can take well over a second to clear
+            # the trust prompt; sleeping a fixed amount races dismissal and
+            # leaves the retry hitting the same codex_trust_prompt state. We
+            # poll for prompt dismissal up to 3s; if still present, return a
+            # retry_needed envelope and let the upstream scheduler decide
+            # whether to back off and try again later.
+            dismissed = _wait_for_trust_prompt_dismissal(
+                injection.get("pane_id") or target, timeout=3.0,
+            )
+            if not dismissed:
+                return _handle_trust_retry_needed(
+                    workspace, state, store, message_id, target, injection,
+                    attempt=_trust_retry_attempt,
+                )
+            injection = _tmux_inject_text(
+                target,
+                text,
+                "Enter",
+                f"team-agent-send-{message_id}-trust-retry",
+                attempts=3 if wait_visible else 1,
+                provider=agent_state.get("provider", "fake"),
+            )
     if injection["ok"]:
         store.mark(message_id, "submitted")
-        EventLog(workspace).write(
+        send_event_log = EventLog(workspace)
+        _stamp_first_send_at_if_leader_to_worker(state, row, send_event_log)
+        send_event_log.write(
             "send.submitted",
             message_id=message_id,
             target=target,
@@ -112,9 +165,248 @@ def _deliver_pending_message(
         "turn_verification": injection.get("turn_verification"),
         "paste_attempts": injection.get("attempts"),
         "submit_attempts": injection.get("submit_attempts"),
+        "detected": injection.get("detected"),
+        "pane_id": injection.get("pane_id"),
+        "pane_mode": injection.get("pane_mode"),
+        "pane_capture_tail": injection.get("pane_capture_tail"),
     }
 
 
+def _handle_trust_retry_needed(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    message_id: str,
+    target: str,
+    injection: dict[str, Any],
+    *,
+    attempt: int,
+) -> dict[str, Any]:
+    """Spark MEDIUM sweep #3: replace the dead-end failed mark with a real
+    bounded-backoff consumer. attempt is the number of the delivery that JUST
+    failed (1 = the original delivery; 2..4 = the scheduler-fired retries).
+
+    Behaviour:
+      * attempt < _TRUST_RETRY_MAX_ATTEMPTS: schedule a trust_retry
+        scheduled_event for the message, holding the message in 'failed' status
+        so _deliver_pending_messages does not race the scheduler. Emit
+        leader_panes.trust_auto_answer_retry_scheduled. Return status='retry_scheduled'.
+      * attempt >= _TRUST_RETRY_MAX_ATTEMPTS: terminal. Mark the message failed
+        and emit leader_panes.trust_auto_answer_exhausted. Return
+        status='trust_auto_answer_exhausted'.
+    """
+    event_log = EventLog(workspace)
+    next_attempt = attempt + 1
+    if next_attempt > _TRUST_RETRY_MAX_ATTEMPTS:
+        store.mark(message_id, "failed", "trust_auto_answer_exhausted")
+        event_log.write(
+            "leader_panes.trust_auto_answer_exhausted",
+            message_id=message_id,
+            workspace=str(workspace),
+            attempts=attempt,
+            target=target,
+            pane_id=injection.get("pane_id"),
+            reason="trust_auto_answer_exhausted",
+        )
+        return {
+            "ok": False,
+            "status": "trust_auto_answer_exhausted",
+            "reason": "trust_auto_answer_exhausted",
+            "attempts": attempt,
+            "detected": injection.get("detected"),
+            "pane_id": injection.get("pane_id"),
+            "pane_mode": injection.get("pane_mode"),
+            "pane_capture_tail": injection.get("pane_capture_tail"),
+        }
+    backoff = _TRUST_RETRY_BACKOFF_SECONDS.get(next_attempt, _TRUST_RETRY_BACKOFF_SECONDS[_TRUST_RETRY_MAX_ATTEMPTS])
+    due_at = (datetime.now(timezone.utc) + timedelta(seconds=backoff)).isoformat()
+    owner_team_id = _message_owner_team_id(store, message_id)
+    event_id = store.add_scheduled_event(
+        due_at,
+        message_id,
+        "trust_retry",
+        {
+            "message_id": message_id,
+            "attempt": next_attempt,
+            "max_attempts": _TRUST_RETRY_MAX_ATTEMPTS,
+            "first_target": target,
+        },
+        owner_team_id=owner_team_id,
+    )
+    # Hold the message in 'failed' so _deliver_pending_messages does not race
+    # the scheduled retry. The scheduler consumer resets it to 'accepted' just
+    # before re-delivery.
+    store.mark(message_id, "failed", "trust_retry_scheduled")
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_needed",
+        message_id=message_id,
+        workspace=str(workspace),
+        pane_id=injection.get("pane_id") or target,
+        target=target,
+        reason="trust_prompt_not_dismissed_after_answer",
+        attempt=attempt,
+    )
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_scheduled",
+        message_id=message_id,
+        workspace=str(workspace),
+        scheduled_event_id=event_id,
+        due_at=due_at,
+        next_attempt=next_attempt,
+        max_attempts=_TRUST_RETRY_MAX_ATTEMPTS,
+        backoff_seconds=backoff,
+    )
+    return {
+        "ok": False,
+        "status": "retry_scheduled",
+        "reason": "trust_prompt_not_dismissed_after_answer",
+        "stage": "trust_auto_answer_dismissal_wait",
+        "verification": "trust_prompt_not_dismissed_after_answer",
+        "scheduled_event_id": event_id,
+        "scheduled_retry_at": due_at,
+        "next_attempt": next_attempt,
+        "max_attempts": _TRUST_RETRY_MAX_ATTEMPTS,
+        "detected": injection.get("detected"),
+        "pane_id": injection.get("pane_id"),
+        "pane_mode": injection.get("pane_mode"),
+        "pane_capture_tail": injection.get("pane_capture_tail"),
+    }
+
+
+def _message_owner_team_id(store: MessageStore, message_id: str) -> str | None:
+    row = _message_by_id(store, message_id)
+    if not row:
+        return None
+    owner = row.get("owner_team_id")
+    return str(owner) if owner else None
+
+
+def _execute_trust_retry(
+    workspace: Path,
+    store: MessageStore,
+    event_log: EventLog,
+    payload: dict[str, Any],
+    *,
+    owner_team_id: str | None = None,
+) -> dict[str, Any]:
+    """Scheduler-side consumer for kind='trust_retry'. Resets the message back
+    to 'accepted' so claim_for_delivery succeeds, re-runs _deliver_pending_message,
+    and either succeeds, escalates to a further retry (via _handle_trust_retry_needed),
+    or hits the terminal exhausted branch.
+    """
+    from team_agent.state import load_runtime_state
+    message_id = str(payload.get("message_id") or "")
+    if not message_id:
+        return {"ok": False, "reason": "trust_retry_missing_message_id"}
+    attempt = int(payload.get("attempt") or 1)
+    row = _message_by_id(store, message_id)
+    if not row:
+        event_log.write(
+            "leader_panes.trust_auto_answer_retry_skipped",
+            message_id=message_id,
+            reason="message_missing",
+            attempt=attempt,
+        )
+        return {"ok": False, "reason": "message_missing"}
+    # Reset to accepted so claim_for_delivery succeeds. The previous attempt
+    # left the row in 'failed' status with reason='trust_retry_scheduled'.
+    store.mark(message_id, "accepted", "trust_retry_resuming")
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_attempted",
+        message_id=message_id,
+        workspace=str(workspace),
+        attempt=attempt,
+        max_attempts=int(payload.get("max_attempts") or _TRUST_RETRY_MAX_ATTEMPTS),
+    )
+    state = load_runtime_state(workspace)
+    if owner_team_id and isinstance(state.get("teams"), dict):
+        scoped = state["teams"].get(owner_team_id)
+        if isinstance(scoped, dict):
+            state = scoped
+    delivery_result = _deliver_pending_message(
+        workspace, state, message_id,
+        wait_visible=True, timeout=30.0,
+        _trust_retry_attempt=attempt,
+    )
+    return delivery_result
+
+
+def _stamp_first_send_at_if_leader_to_worker(
+    state: dict[str, Any],
+    row: dict[str, Any],
+    event_log: EventLog | None = None,
+) -> None:
+    """Route B atomicity (2026-05-27): record the first time the leader
+    successfully sends work to each worker. The presence of this stamp drives
+    restart's resumability decision — a worker the leader has interacted with
+    has accumulated conversation state, so a missing session_id at restart
+    time IS an atomicity violation. A worker that has never received work
+    legitimately fresh-starts during restart.
+
+    Only stamped once per worker (idempotent across re-sends). Only fires on
+    leader -> worker sends; worker-to-worker peer messages do not count.
+    The mutation lives on the state dict the caller already saves
+    (`save_team_scoped_state` in send.py, or `save_runtime_state` after
+    coordinator_tick), so persistence is automatic.
+
+    C1 (cr verdict, 2026-05-27): when the stamp transitions null -> ts (the
+    one-time write), emit a `worker.first_interaction` audit event with
+    worker_id, first_send_at, message_id. Re-sends to the same worker hit the
+    idempotency guard above and do NOT re-emit. Worker-to-worker peer sends
+    short-circuit at the sender check and do NOT emit.
+    """
+    sender = str(row.get("sender") or "")
+    recipient = str(row.get("recipient") or "")
+    if not recipient:
+        return
+    leader_id = str((state.get("leader") or {}).get("id") or "leader")
+    if sender not in {"leader", "Leader", leader_id}:
+        return
+    agents = state.get("agents")
+    if not isinstance(agents, dict):
+        return
+    agent_state = agents.get(recipient)
+    if not isinstance(agent_state, dict):
+        return
+    if agent_state.get("first_send_at"):
+        return
+    stamp = datetime.now(timezone.utc).isoformat()
+    agent_state["first_send_at"] = stamp
+    if event_log is not None:
+        event_log.write(
+            "worker.first_interaction",
+            worker_id=recipient,
+            first_send_at=stamp,
+            message_id=str(row.get("message_id") or ""),
+        )
+
+
+def _wait_for_trust_prompt_dismissal(target: str, *, timeout: float = 3.0, poll_interval: float = 0.1) -> bool:
+    """Spark MEDIUM #4: bounded poll for trust prompt dismissal. Returns True once
+    the pane no longer matches detect_non_input_scrollback, False if the prompt
+    is still present after `timeout` seconds. Uses the same detector the inject
+    path uses so behaviour stays consistent."""
+    import time as _time
+    from team_agent.messaging.tmux_prompt import detect_non_input_scrollback
+    deadline = _time.monotonic() + max(timeout, 0.0)
+    while True:
+        capture = _capture_pane_tail(target)
+        detected = detect_non_input_scrollback(capture)
+        if detected != "codex_trust_prompt":
+            return True
+        if _time.monotonic() >= deadline:
+            return False
+        _time.sleep(poll_interval)
+
+
+def _capture_pane_tail(target: str) -> str:
+    from team_agent.messaging.deps import _capture_tmux_pane_text
+    capture = _capture_tmux_pane_text(target)
+    if not capture.get("ok"):
+        return ""
+    return str(capture.get("capture") or "")
+
+
 def _deliver_pending_messages(workspace: Path, state: dict[str, Any], event_log: EventLog) -> list[str]:
     store = MessageStore(workspace)
     delivered: list[str] = []

package/src/team_agent/messaging/idle_alerts.py

@@ -6,7 +6,7 @@ from typing import Any
 
 from team_agent.events import EventLog
 from team_agent.message_store import MessageStore
-from team_agent.messaging.deps import load_spec, save_runtime_state, team_state_key
+from team_agent.messaging.deps import load_runtime_state, load_spec, save_runtime_state, team_state_key
 from team_agent.messaging.internal_delivery import deliver_stored_message
 
 
@@ -27,6 +27,21 @@ STABLE_IDLE_SECONDS = 120
 FIRE_DEBOUNCE_SECONDS = 300
 OBLIGATION_PENDING_MIN_AGE_SECONDS = 60
 
+# Event-log progress signal (Gap 32 §"Idle-Detector False Positive Continues Post Phase G hotfix-3"):
+# the team_last_progress_at calculation must also count leader-side sends and worker MCP calls
+# as recent team activity, not only agent_health.last_output_at. Without this, a worker that has
+# called MCP but not yet emitted a visible turn shows up as idle and the idle reminder fires
+# spuriously inside the stable-idle window.
+_PROGRESS_EVENT_TYPES = frozenset({
+    "send.deliver_attempt",
+    "leader_receiver.deliver_attempt",
+    "mcp.report_result",
+    "mcp.send_message",
+})
+_PROGRESS_EVENT_PREFIXES = ("mcp.read_",)
+_PROGRESS_EVENT_WINDOW_SECONDS = 300
+_PROGRESS_EVENT_TAIL_LIMIT = 1000
+
 
 def _parse_iso(text: Any) -> datetime | None:
     if not isinstance(text, str) or not text:
@@ -62,24 +77,105 @@ def _team_last_progress_at(
     state: dict[str, Any],
     store: MessageStore,
     owner_team_id: str,
-) -> datetime | None:
-    candidates: list[datetime] = []
+    event_log: EventLog | None = None,
+    now: datetime | None = None,
+    workspace: Path | None = None,
+) -> tuple[datetime | None, str | None]:
+    sources: list[tuple[datetime, str]] = []
     coordinator = state.get("coordinator") or {}
     explicit = (coordinator.get("team_last_progress_at") or {}).get(owner_team_id)
     if isinstance(explicit, dict):
         ts = _parse_iso(explicit.get("at"))
         if ts:
-            candidates.append(ts)
+            sources.append((ts, "explicit_marker"))
     elif isinstance(explicit, str):
         ts = _parse_iso(explicit)
         if ts:
-            candidates.append(ts)
+            sources.append((ts, "explicit_marker"))
     health = store.agent_health(owner_team_id=owner_team_id)
     for row in health.values():
         ts = _parse_iso(row.get("last_output_at"))
         if ts:
-            candidates.append(ts)
-    return max(candidates) if candidates else None
+            sources.append((ts, "agent_health.last_output_at"))
+    if event_log is not None:
+        # Spark MEDIUM #3 (d9f740d): in multi-team workspaces an unscoped progress event in
+        # team A's activity must NOT suppress team B's idle_fallback. require_team_scope=True
+        # when the workspace has more than one team so unscoped events are ignored. The
+        # team-scoped state passed in here does not carry the workspace-level `teams` dict, so
+        # we re-read the workspace state from disk to detect multi-team shape.
+        require_team_scope = False
+        teams = state.get("teams")
+        if isinstance(teams, dict) and len(teams) > 1:
+            require_team_scope = True
+        elif workspace is not None:
+            try:
+                ws_teams = (load_runtime_state(workspace).get("teams") or {})
+            except Exception:
+                ws_teams = {}
+            if isinstance(ws_teams, dict) and len(ws_teams) > 1:
+                require_team_scope = True
+        event_ts = _scan_event_progress_signals(
+            event_log, owner_team_id, now or datetime.now(timezone.utc),
+            require_team_scope=require_team_scope,
+        )
+        if event_ts:
+            sources.append((event_ts, "event_log"))
+    if not sources:
+        return None, None
+    sources.sort(key=lambda item: item[0], reverse=True)
+    return sources[0]
+
+
+# Stage 14 (Gap 36b) — mtime cache per (workspace_path, owner_team_id, require_team_scope).
+# Mac mini 2026-05-26 evidence: _scan_event_progress_signals was a 22% CPU hot path because
+# every 2-second coordinator tick parsed up to 1000 events from a 28 MB events.jsonl. With
+# the cache, the parse only re-runs when the file changes; quiet workspaces pay zero file
+# I/O between writes.
+_PROGRESS_SCAN_CACHE: dict[tuple[str, str, bool], tuple[float, datetime | None]] = {}
+
+
+def _scan_event_progress_signals(
+    event_log: EventLog,
+    owner_team_id: str,
+    now: datetime,
+    *,
+    require_team_scope: bool = False,
+) -> datetime | None:
+    cache_key = (str(event_log.path), owner_team_id, require_team_scope)
+    try:
+        current_mtime = event_log.path.stat().st_mtime
+    except FileNotFoundError:
+        _PROGRESS_SCAN_CACHE.pop(cache_key, None)
+        return None
+    cached = _PROGRESS_SCAN_CACHE.get(cache_key)
+    if cached is not None and cached[0] == current_mtime:
+        return cached[1]
+    window_start = now - timedelta(seconds=_PROGRESS_EVENT_WINDOW_SECONDS)
+    latest: datetime | None = None
+    for event in event_log.tail(_PROGRESS_EVENT_TAIL_LIMIT):
+        event_type = str(event.get("event") or "")
+        if event_type not in _PROGRESS_EVENT_TYPES and not any(
+            event_type.startswith(prefix) for prefix in _PROGRESS_EVENT_PREFIXES
+        ):
+            continue
+        event_team = event.get("team") or event.get("owner_team_id")
+        if event_team is None:
+            if require_team_scope:
+                continue
+        elif event_team != owner_team_id:
+            continue
+        ts = _parse_iso(event.get("ts"))
+        if not ts or ts < window_start:
+            continue
+        if latest is None or ts > latest:
+            latest = ts
+    _PROGRESS_SCAN_CACHE[cache_key] = (current_mtime, latest)
+    return latest
+
+
+def _reset_progress_scan_cache() -> None:
+    """Test-only hook to force re-scan."""
+    _PROGRESS_SCAN_CACHE.clear()
 
 
 def _team_last_idle_fallback_fire_at(state: dict[str, Any], owner_team_id: str) -> datetime | None:
@@ -209,14 +305,18 @@ def detect_idle_fallbacks(
         record_team_progress(state, now, source="all_workers_idle:false", owner_team_id=owner_team_id)
         save_runtime_state(workspace, state)
         return []
-    last_progress = _team_last_progress_at(state, store, owner_team_id)
+    last_progress, progress_source = _team_last_progress_at(
+        state, store, owner_team_id, event_log=event_log, now=now, workspace=workspace,
+    )
     if last_progress and (now - last_progress) < timedelta(seconds=STABLE_IDLE_SECONDS):
+        reason = "recent_team_progress" if progress_source == "event_log" else "stable_idle_window"
         event_log.write(
             "coordinator.idle_fallback_skipped",
-            reason="stable_idle_window",
+            reason=reason,
             team=owner_team_id,
             stable_idle_seconds=STABLE_IDLE_SECONDS,
             elapsed_seconds=int((now - last_progress).total_seconds()),
+            progress_source=progress_source,
         )
         return []
     last_fire = _team_last_idle_fallback_fire_at(state, owner_team_id)