@team-agent/installer 0.2.1 → 0.2.3

package/src/team_agent/messaging/leader.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import hashlib
+
 from team_agent.messaging.deps import (
     EventLog,
     MessageStore,
@@ -10,8 +12,10 @@ from team_agent.messaging.deps import (
     _validate_leader_receiver,
     core_render_message,
     json,
+    os,
     runtime_dir,
     save_runtime_state,
+    team_state_key,
     time,
 )
 
@@ -49,6 +53,19 @@ def _leader_inbox_path(workspace: Path) -> Path:
     return runtime_dir(workspace) / "leader-inbox.log"
 
 
+def _extract_result_id_from_content(content: str) -> str | None:
+    """Stage 12: result-notification messages embed a `Result id: <id>` line; the gate
+    parses it from content so callers that did NOT plumb the result_id kwarg through
+    still consult the dedupe gate. Format mirrors _format_report_result_notification and
+    format_result_watcher_notification."""
+    if not content:
+        return None
+    for line in content.splitlines():
+        if line.startswith("Result id: "):
+            return line.removeprefix("Result id: ").strip() or None
+    return None
+
+
 def _send_to_leader_receiver(
     workspace: Path,
     state: dict[str, Any],
@@ -58,6 +75,8 @@ def _send_to_leader_receiver(
     sender: str,
     requires_ack: bool,
     event_log: EventLog,
+    *,
+    result_id: str | None = None,
 ) -> dict[str, Any]:
     store = MessageStore(workspace)
     message_id = store.create_message(task_id, sender, leader_id, content, requires_ack=False)
@@ -94,10 +113,30 @@ def _send_to_leader_receiver(
             error="No direct leader tmux pane is attached. Run team-agent attach-leader.",
         )
 
-    validation = _validate_leader_receiver(receiver)
+    owner_identity = state.get("team_owner") or None
+    side_pane_refusal = _side_pane_owner_refusal(state, owner_identity)
+    if side_pane_refusal:
+        event_log.write("leader_receiver.side_pane_refused", **side_pane_refusal)
+        return {
+            "ok": False,
+            "message_id": message_id,
+            "status": "refused",
+            "to": leader_id,
+            "channel": "direct_tmux",
+            **side_pane_refusal,
+        }
+    receiver_for_validation = dict(receiver)
+    if owner_identity and owner_identity.get("leader_session_uuid") and not receiver_for_validation.get("leader_session_uuid"):
+        receiver_for_validation["leader_session_uuid"] = owner_identity["leader_session_uuid"]
+    validation = _validate_leader_receiver(receiver_for_validation)
     if not validation["ok"]:
-        owner_identity = state.get("team_owner") or None
-        rediscovery = _rediscover_leader_receiver(receiver, event_log, owner_identity)
+        rediscovery = _rediscover_leader_receiver(
+            receiver_for_validation,
+            event_log,
+            owner_identity,
+            invalidation_reason=validation.get("reason"),
+            team_id=team_state_key(state),
+        )
         if rediscovery.get("status") == "updated":
             state["leader_receiver"].update(rediscovery["receiver"])
             receiver = state["leader_receiver"]
@@ -111,7 +150,7 @@ def _send_to_leader_receiver(
                 payload,
                 event_log,
                 reason="ambiguous",
-                error="multiple possible leader panes found; rerun team-agent attach-leader --pane <pane_id>",
+                error="multiple possible leader panes found; run team-agent claim-leader --confirm from the intended pane",
                 message_status="ambiguous",
             )
     if not validation["ok"]:
@@ -128,6 +167,69 @@ def _send_to_leader_receiver(
     state["leader_receiver"].update(validation["pane"])
     submit_key, submit_reason = _choose_leader_submit_key(receiver.get("provider", "codex"), validation.get("capture", ""))
     target = receiver["pane_id"]
+    # Stage 12 (Gap 26 ∩ Gap 32 roundtable 2026-05-26) — injection-boundary dedupe gate.
+    # Result-notification injections route through claim_leader_notification_delivery; the
+    # gate suppresses a second inject for the same (result_id, leader_session_uuid).
+    # Non-result messages (peer mirror, idle reminder, ambiguous-prompt) lack a "Result id:"
+    # line in their text and bypass the gate.
+    effective_result_id = result_id or _extract_result_id_from_content(content)
+    leader_uuid_for_gate = str(
+        (state.get("team_owner") or {}).get("leader_session_uuid")
+        or (state.get("leader_receiver") or {}).get("leader_session_uuid")
+        or ""
+    )
+    if effective_result_id and leader_uuid_for_gate:
+        from team_agent.message_store.leader_notification_log import claim_leader_notification_delivery
+        envelope_hash = hashlib.sha256(content.encode("utf-8", errors="ignore")).hexdigest()[:16]
+        claim = claim_leader_notification_delivery(
+            store,
+            result_id=effective_result_id,
+            leader_session_uuid=leader_uuid_for_gate,
+            proposed_message_id=message_id,
+            envelope_hash=envelope_hash,
+            owner_team_id=team_state_key(state),
+            pane_id=target,
+        )
+        if claim["status"] == "already_notified_by":
+            prev_msg = claim.get("notified_message_id")
+            prev_hash = claim.get("envelope_content_hash")
+            if envelope_hash == prev_hash:
+                event_log.write(
+                    "leader_notification.dedupe_skip",
+                    result_id=effective_result_id,
+                    leader_session_uuid=leader_uuid_for_gate,
+                    prev_message_id=prev_msg,
+                    this_message_id=message_id,
+                    prev_ts=claim.get("notified_at"),
+                    pane_id=target,
+                    team_id=team_state_key(state),
+                )
+            else:
+                event_log.write(
+                    "leader_notification.legitimate_duplicate_suspected",
+                    result_id=effective_result_id,
+                    leader_session_uuid=leader_uuid_for_gate,
+                    prev_message_id=prev_msg,
+                    this_message_id=message_id,
+                    prev_envelope_hash=prev_hash,
+                    this_envelope_hash=envelope_hash,
+                    pane_id=target,
+                    team_id=team_state_key(state),
+                )
+            store.mark(message_id, "submitted", "dedupe_suppressed_by_leader_notification_log")
+            save_runtime_state(workspace, state)
+            return {
+                "ok": True,
+                "message_id": message_id,
+                "status": "submitted",
+                "to": leader_id,
+                "channel": "direct_tmux",
+                "leader_receiver": state["leader_receiver"],
+                "visible": False,
+                "submitted": False,
+                "deduped": True,
+                "canonical_message_id": prev_msg,
+            }
     event_log.write(
         "leader_receiver.deliver_attempt",
         message_id=message_id,
@@ -139,6 +241,8 @@ def _send_to_leader_receiver(
         visible_token=rendered.get("token"),
         payload=payload,
         warning=validation.get("warning"),
+        result_id=effective_result_id,
+        leader_session_uuid=leader_uuid_for_gate or None,
     )
     injection = _tmux_inject_text(
         target,
@@ -147,6 +251,19 @@ def _send_to_leader_receiver(
         f"team-agent-leader-receiver-{message_id}",
         provider=receiver.get("provider", "codex"),
     )
+    if not injection.get("ok") and injection.get("detected") == "codex_trust_prompt":
+        from team_agent.messaging.trust_auto_answer import retry_injection_after_trust_auto_answer
+        injection = retry_injection_after_trust_auto_answer(
+            workspace,
+            state,
+            event_log,
+            injection,
+            target,
+            text,
+            submit_key,
+            f"team-agent-leader-receiver-{message_id}-trust-retry",
+            receiver.get("provider", "codex"),
+        )
     if injection["ok"]:
         store.mark(message_id, "submitted")
         event_log.write(
@@ -201,6 +318,64 @@ def _send_to_leader_receiver(
     )
 
 
+def _side_pane_owner_refusal(state: dict[str, Any], owner_identity: dict[str, Any] | None) -> dict[str, Any] | None:
+    owner_uuid = str((owner_identity or {}).get("leader_session_uuid") or "")
+    caller_uuid = os.environ.get("TEAM_AGENT_LEADER_SESSION_UUID") or os.environ.get("TEAM_AGENT_LEADER_SESSION_UUID_OVERRIDE") or ""
+    if not owner_uuid or not caller_uuid or caller_uuid == owner_uuid:
+        return None
+    bound_pane = (state.get("leader_receiver") or {}).get("pane_id") or (owner_identity or {}).get("pane_id")
+    team_id = team_state_key(state)
+    return {
+        "reason": "team_owner_mismatch",
+        "error": (
+            f"This workspace's team `{team_id}` is already bound to pane `{bound_pane}`. "
+            "To work in this window either start a new team with a different team_id, operate through the bound pane, "
+            "or run `team-agent claim-leader --confirm` only if you intend to forcibly take over."
+        ),
+        "bound_pane_id": bound_pane,
+        "caller_uuid_prefix": caller_uuid[:8],
+        "uuid_prefix": owner_uuid[:8],
+        "action": "team-agent claim-leader --confirm",
+    }
+
+
+def claim_leader_receiver(
+    workspace: Path,
+    state: dict[str, Any],
+    candidate: dict[str, Any],
+    event_log: EventLog,
+    *,
+    confirm: bool,
+    expected_epoch: int | None = None,
+) -> dict[str, Any]:
+    from team_agent.messaging.leader_panes import _leader_command_looks_usable, _receiver_from_target, _target_matches_owner_identity, _uuid_prefix
+    if not confirm:
+        return {"ok": False, "status": "refused", "reason": "confirm_required", "action": "team-agent claim-leader --confirm"}
+    owner = state.setdefault("team_owner", {})
+    receiver = state.get("leader_receiver") or {}
+    current_epoch = int(owner.get("owner_epoch") or receiver.get("owner_epoch") or 0)
+    if expected_epoch is not None and current_epoch != expected_epoch:
+        event_log.write("leader_receiver.claim_refused", reason="owner_epoch_advanced", owner_epoch=current_epoch, bound_pane_id=receiver.get("pane_id"))
+        return {"ok": False, "status": "refused", "reason": "owner_epoch_advanced", "owner_epoch": current_epoch, "bound_pane_id": receiver.get("pane_id")}
+    if receiver.get("pane_id") == candidate.get("pane_id"):
+        return {"ok": True, "status": "already_bound", "leader_receiver": receiver, "owner_epoch": current_epoch}
+    if not _target_matches_owner_identity(candidate, owner):
+        event_log.write("leader_receiver.claim_refused", reason="uuid_mismatch", candidate_pane_id=candidate.get("pane_id"))
+        return {"ok": False, "status": "refused", "reason": "uuid_mismatch"}
+    provider = str(candidate.get("provider") or receiver.get("provider") or "codex")
+    if not _leader_command_looks_usable(str(candidate.get("pane_current_command", "")), provider):
+        return {"ok": False, "status": "refused", "reason": "wrong_command", "candidate_pane_id": candidate.get("pane_id")}
+    next_epoch = current_epoch + 1
+    new_receiver = _receiver_from_target(candidate, provider, owner.get("leader_session_uuid"), next_epoch)
+    owner["owner_epoch"] = next_epoch
+    state["leader_receiver"] = new_receiver
+    from team_agent.runtime import _runtime_lock, save_runtime_state
+    with _runtime_lock(workspace, "leader_receiver"):
+        save_runtime_state(workspace, state)
+    event_log.write("leader_receiver.claimed", pane_id=new_receiver["pane_id"], owner_epoch=next_epoch, uuid_prefix=_uuid_prefix(owner))
+    return {"ok": True, "status": "claimed", "leader_receiver": new_receiver, "owner_epoch": next_epoch}
+
+
 def _fail_leader_delivery(
     workspace: Path,
     state: dict[str, Any],
@@ -302,12 +477,6 @@ def _format_team_agent_message(payload: dict[str, Any]) -> str:
 
 
 
-
-
-
-
-
-
 
 
 

package/src/team_agent/messaging/leader_api_errors.py

@@ -0,0 +1,216 @@
+"""Gap 28 (Slice 2 Stage 2): observe-only detection of leader-pane API errors.
+
+The coordinator tick captures the leader pane scrollback once per cycle, scans it for
+known upstream-API error patterns (Claude/Codex CLI errors that occur mid-turn), and
+emits a structured `leader.api_error` audit event. The intent is observability — auto-
+retry belongs to the upstream CLI; this module never touches the pane.
+
+Event schema (logged via EventLog.write):
+
+  event:                          'leader.api_error'
+  ts:                             ISO-8601 UTC (added by EventLog)
+  leader_session_uuid:            str | None
+  error_class:                    'Overloaded' | 'RateLimit' | 'Timeout' |
+                                  'NetworkError' | 'Unknown'
+  provider:                       'claude' | 'codex' | 'claude_code' | str | None
+  partial_response_streamed:      bool  (heuristic: assistant text before the error)
+  worker_dispatch_just_before:    list[str]  (leader→worker msg_ids in the prior 60s)
+  retry_count:                    int   (always 0 — the framework does not retry today)
+  matched_pattern_snippet:        str   (the captured error line, ≤160 chars)
+
+Detection dedupes within the coordinator state via a (error_class, snippet-tail)
+fingerprint stored under `state['coordinator']['last_api_error_fingerprint']`. A
+clean tick (no error pattern present) clears the fingerprint so the next genuine
+error re-emits. This keeps event volume bounded while still catching distinct
+errors as they occur.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from team_agent.events import EventLog
+from team_agent.message_store import MessageStore
+
+
+# Spark MEDIUM sweeps (2026-05-26):
+# (#3) Require an API/provider context marker near the error keyword. Bare '503' /
+#      'fetch failed' / 'timed out' in user text used to false-fire.
+# (#7) Match across short sliding windows of 1-3 adjacent lines so wrapped tmux
+#      output ("claude:\n  request timed out") still resolves to a single
+#      detection. Window joined with a single space; capped at _WINDOW_MAX_CHARS
+#      so the scan stays bounded.
+_API_CONTEXT = (
+    r"(?:API\s+Error|HTTP\s*Error|HTTPError|request\s+failed|"
+    r"codex|claude|Anthropic|OpenAI|TypeError)"
+)
+
+# Patterns operate against a sliding window of up to 3 joined lines. The window
+# never contains '\n' (lines are joined with a single space), so `[^\n]` and `.`
+# behave the same; we use `[^\n]` for self-documentation.
+_ERROR_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    # Overloaded — keyword itself already includes the "API Error:" prefix.
+    (re.compile(r"API\s+Error:\s*Overloaded", re.IGNORECASE), "Overloaded"),
+    # RateLimit — 429 with "Too Many Requests" is sufficiently specific; require it
+    # appear AFTER an API context marker OR before "Too Many Requests" tightly.
+    (re.compile(rf"(?:{_API_CONTEXT}[^\n]*\b429\b|\b429\s+Too\s+Many\s+Requests)", re.IGNORECASE), "RateLimit"),
+    # 5xx — must share a window with an API-context marker on either side.
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}\b5(?:00|02|03|04)\b", re.IGNORECASE), "NetworkError"),
+    (re.compile(rf"\b5(?:00|02|03|04)\b[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "NetworkError"),
+    # fetch failed — needs an API-context marker in the same window. The TypeError
+    # marker on its own counts (Node fetch frames the error this way).
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}fetch\s+failed", re.IGNORECASE), "NetworkError"),
+    (re.compile(rf"fetch\s+failed[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "NetworkError"),
+    # Timeout — likewise requires an API-context marker in the window, except for
+    # the unambiguous syscall token ETIMEDOUT.
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}(?:request|connection)\s+(?:timed\s+out|timeout)", re.IGNORECASE), "Timeout"),
+    (re.compile(rf"(?:request|connection)\s+(?:timed\s+out|timeout)[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "Timeout"),
+    (re.compile(r"\bETIMEDOUT\b", re.IGNORECASE), "Timeout"),
+]
+
+_RECENT_LINE_WINDOW = 100        # scan only the most recent N lines
+_SLIDING_WINDOW_LINES = 3        # join up to 3 adjacent lines per scan window
+_WINDOW_MAX_CHARS = 400          # discard windows beyond this length to bound work
+_DISPATCH_WINDOW_SECONDS = 60    # leader→worker sends counted within this lookback
+_PARTIAL_RESPONSE_HEAD_BYTES = 4000
+
+_PARTIAL_RESPONSE_HINT = re.compile(
+    r"(?:^|\n)\s*(?:Assistant|⏺|●|> |I'll |I will |I'm |I am |Let me )",
+    re.IGNORECASE,
+)
+
+
+def detect_leader_api_errors(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    event_log: EventLog,
+    *,
+    capture_fn: Callable[[str], dict[str, Any]] | None = None,
+    now_fn: Callable[[], datetime] | None = None,
+) -> list[dict[str, Any]]:
+    """Coordinator-tick entry point. Returns a list of emitted events (0 or 1)."""
+    receiver = state.get("leader_receiver") or {}
+    pane = receiver.get("pane_id") if receiver.get("mode") == "direct_tmux" else None
+    if not pane:
+        return []
+    capture_fn = capture_fn or _default_capture_fn()
+    capture = capture_fn(str(pane))
+    if not capture.get("ok"):
+        return []
+    scrollback = str(capture.get("capture") or "")
+    coordinator_state = state.setdefault("coordinator", {})
+    found = _match_first_error(scrollback)
+    if not found:
+        if coordinator_state.get("last_api_error_fingerprint"):
+            coordinator_state["last_api_error_fingerprint"] = None
+        return []
+    error_class, snippet = found
+    fingerprint = f"{error_class}::{snippet[-120:]}"
+    if coordinator_state.get("last_api_error_fingerprint") == fingerprint:
+        return []
+    coordinator_state["last_api_error_fingerprint"] = fingerprint
+    now = (now_fn() if now_fn else datetime.now(timezone.utc))
+    cutoff_iso = (now - timedelta(seconds=_DISPATCH_WINDOW_SECONDS)).isoformat()
+    leader_uuid = (
+        str((state.get("team_owner") or {}).get("leader_session_uuid") or "")
+        or str(receiver.get("leader_session_uuid") or "")
+        or None
+    )
+    provider = str(receiver.get("provider") or "") or None
+    event = event_log.write(
+        "leader.api_error",
+        leader_session_uuid=leader_uuid,
+        error_class=error_class,
+        provider=provider,
+        partial_response_streamed=_scrollback_has_partial_response(scrollback, snippet),
+        worker_dispatch_just_before=_recent_leader_dispatches(store, cutoff_iso),
+        retry_count=0,
+        matched_pattern_snippet=snippet[:160],
+    )
+    return [event]
+
+
+def _default_capture_fn() -> Callable[[str], dict[str, Any]]:
+    from team_agent.messaging.deps import _capture_tmux_pane_text
+    return _capture_tmux_pane_text
+
+
+def _match_first_error(scrollback: str) -> tuple[str, str] | None:
+    """Spark MEDIUM #7: sliding window of 1..N adjacent lines. Lines inside a
+    window are joined with a single space so a wrapped pair such as
+        claude:
+          request timed out
+    is detected as one event without permitting unbounded cross-line matches.
+    Latest window wins so the freshest error is reported."""
+    if not scrollback:
+        return None
+    lines = [line.strip() for line in scrollback.splitlines()[-_RECENT_LINE_WINDOW:]]
+    if not lines:
+        return None
+    best: tuple[int, str, str] | None = None
+    for start in range(len(lines)):
+        for size in range(1, _SLIDING_WINDOW_LINES + 1):
+            end = start + size
+            if end > len(lines):
+                break
+            window = " ".join(line for line in lines[start:end] if line)
+            if not window:
+                continue
+            # Spark MEDIUM sweep #3 (2026-05-26): tail-preserve instead of
+            # dropping the window wholesale. Errors land at the END of verbose
+            # diagnostics (stack traces, retry chatter, etc.). If we discarded
+            # any window over the cap we silently lost recall on long wrapped
+            # output. Scanning the LAST _WINDOW_MAX_CHARS still bounds regex
+            # cost while keeping the freshest context — the bit most likely to
+            # contain the actual provider error keyword.
+            if len(window) > _WINDOW_MAX_CHARS:
+                window = window[-_WINDOW_MAX_CHARS:]
+            for pattern, error_class in _ERROR_PATTERNS:
+                match = pattern.search(window)
+                if not match:
+                    continue
+                snippet = window[:240]
+                if best is None or start > best[0]:
+                    best = (start, error_class, snippet)
+                # First match per window is enough; later windows may override.
+                break
+    if best is None:
+        return None
+    return best[1], best[2]
+
+
+def _scrollback_has_partial_response(scrollback: str, error_snippet: str) -> bool:
+    idx = scrollback.rfind(error_snippet)
+    if idx == -1:
+        return False
+    head = scrollback[max(0, idx - _PARTIAL_RESPONSE_HEAD_BYTES): idx]
+    return bool(_PARTIAL_RESPONSE_HINT.search(head))
+
+
+def _recent_leader_dispatches(store: MessageStore, cutoff_iso: str) -> list[str]:
+    out: list[str] = []
+    try:
+        rows = store.messages()
+    except Exception:
+        return out
+    for row in rows:
+        sender = str(row.get("sender") or "")
+        if sender not in {"leader", "Leader"} and not _looks_like_leader_sender(sender):
+            continue
+        created = str(row.get("created_at") or "")
+        if not created or created < cutoff_iso:
+            continue
+        msg_id = str(row.get("message_id") or "")
+        if msg_id:
+            out.append(msg_id)
+    return out
+
+
+def _looks_like_leader_sender(sender: str) -> bool:
+    return sender.startswith("leader") or sender.lower() == "leader"
+
+
+__all__ = ["detect_leader_api_errors"]