@team-agent/installer 0.2.2 → 0.2.3

package/src/team_agent/messaging/delivery.py

@@ -10,15 +10,28 @@ from team_agent.messaging.deps import (
     core_render_message,
 )
 
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
+
+# Spark MEDIUM sweep #3 (2026-05-26): retry_needed bounded backoff. Each entry is
+# the delay (seconds) BEFORE the attempt with that number runs; attempt 1 was the
+# original delivery, attempt 2 fires 5s after retry_needed, attempt 3 fires 15s
+# after the previous, attempt 4 fires 30s after the previous. _TRUST_RETRY_MAX_ATTEMPTS
+# bounds the total — the 4th retry_needed is terminal and emits
+# leader_panes.trust_auto_answer_exhausted.
+_TRUST_RETRY_BACKOFF_SECONDS = {2: 5, 3: 15, 4: 30}
+_TRUST_RETRY_MAX_ATTEMPTS = 4
+
 def _deliver_pending_message(
     workspace: Path,
     state: dict[str, Any],
     message_id: str,
     wait_visible: bool = True,
     timeout: float = 30.0,
+    *,
+    _trust_retry_attempt: int = 1,
 ) -> dict[str, Any]:
     store = MessageStore(workspace)
     row = next((m for m in store.messages() if m["message_id"] == message_id), None)
@@ -65,9 +78,49 @@ def _deliver_pending_message(
         attempts=3 if wait_visible else 1,
         provider=agent_state.get("provider", "fake"),
     )
+    if not injection.get("ok") and injection.get("detected") == "codex_trust_prompt":
+        # Gap 29 (Stage 2): opt-in trust auto-answer. The helper enforces both the
+        # opt-in flag and a workspace-dir match before sending '1'+Enter, then we
+        # retry the original paste once the prompt has actually been dismissed.
+        # Bypassed entirely when opt-out (default) — the existing failed envelope
+        # is preserved.
+        from team_agent.messaging.leader_panes import attempt_trust_auto_answer
+        answer = attempt_trust_auto_answer(
+            workspace,
+            injection.get("pane_id") or target,
+            injection.get("pane_capture_tail") or "",
+            EventLog(workspace),
+            state=state,
+        )
+        if answer.get("answered"):
+            # Spark MEDIUM #4 (2026-05-26): replace the fixed 0.3s sleep with a
+            # bounded poll. Slow terminals can take well over a second to clear
+            # the trust prompt; sleeping a fixed amount races dismissal and
+            # leaves the retry hitting the same codex_trust_prompt state. We
+            # poll for prompt dismissal up to 3s; if still present, return a
+            # retry_needed envelope and let the upstream scheduler decide
+            # whether to back off and try again later.
+            dismissed = _wait_for_trust_prompt_dismissal(
+                injection.get("pane_id") or target, timeout=3.0,
+            )
+            if not dismissed:
+                return _handle_trust_retry_needed(
+                    workspace, state, store, message_id, target, injection,
+                    attempt=_trust_retry_attempt,
+                )
+            injection = _tmux_inject_text(
+                target,
+                text,
+                "Enter",
+                f"team-agent-send-{message_id}-trust-retry",
+                attempts=3 if wait_visible else 1,
+                provider=agent_state.get("provider", "fake"),
+            )
     if injection["ok"]:
         store.mark(message_id, "submitted")
-        EventLog(workspace).write(
+        send_event_log = EventLog(workspace)
+        _stamp_first_send_at_if_leader_to_worker(state, row, send_event_log)
+        send_event_log.write(
             "send.submitted",
             message_id=message_id,
             target=target,
@@ -112,9 +165,248 @@ def _deliver_pending_message(
         "turn_verification": injection.get("turn_verification"),
         "paste_attempts": injection.get("attempts"),
         "submit_attempts": injection.get("submit_attempts"),
+        "detected": injection.get("detected"),
+        "pane_id": injection.get("pane_id"),
+        "pane_mode": injection.get("pane_mode"),
+        "pane_capture_tail": injection.get("pane_capture_tail"),
     }
 
 
+def _handle_trust_retry_needed(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    message_id: str,
+    target: str,
+    injection: dict[str, Any],
+    *,
+    attempt: int,
+) -> dict[str, Any]:
+    """Spark MEDIUM sweep #3: replace the dead-end failed mark with a real
+    bounded-backoff consumer. attempt is the number of the delivery that JUST
+    failed (1 = the original delivery; 2..4 = the scheduler-fired retries).
+
+    Behaviour:
+      * attempt < _TRUST_RETRY_MAX_ATTEMPTS: schedule a trust_retry
+        scheduled_event for the message, holding the message in 'failed' status
+        so _deliver_pending_messages does not race the scheduler. Emit
+        leader_panes.trust_auto_answer_retry_scheduled. Return status='retry_scheduled'.
+      * attempt >= _TRUST_RETRY_MAX_ATTEMPTS: terminal. Mark the message failed
+        and emit leader_panes.trust_auto_answer_exhausted. Return
+        status='trust_auto_answer_exhausted'.
+    """
+    event_log = EventLog(workspace)
+    next_attempt = attempt + 1
+    if next_attempt > _TRUST_RETRY_MAX_ATTEMPTS:
+        store.mark(message_id, "failed", "trust_auto_answer_exhausted")
+        event_log.write(
+            "leader_panes.trust_auto_answer_exhausted",
+            message_id=message_id,
+            workspace=str(workspace),
+            attempts=attempt,
+            target=target,
+            pane_id=injection.get("pane_id"),
+            reason="trust_auto_answer_exhausted",
+        )
+        return {
+            "ok": False,
+            "status": "trust_auto_answer_exhausted",
+            "reason": "trust_auto_answer_exhausted",
+            "attempts": attempt,
+            "detected": injection.get("detected"),
+            "pane_id": injection.get("pane_id"),
+            "pane_mode": injection.get("pane_mode"),
+            "pane_capture_tail": injection.get("pane_capture_tail"),
+        }
+    backoff = _TRUST_RETRY_BACKOFF_SECONDS.get(next_attempt, _TRUST_RETRY_BACKOFF_SECONDS[_TRUST_RETRY_MAX_ATTEMPTS])
+    due_at = (datetime.now(timezone.utc) + timedelta(seconds=backoff)).isoformat()
+    owner_team_id = _message_owner_team_id(store, message_id)
+    event_id = store.add_scheduled_event(
+        due_at,
+        message_id,
+        "trust_retry",
+        {
+            "message_id": message_id,
+            "attempt": next_attempt,
+            "max_attempts": _TRUST_RETRY_MAX_ATTEMPTS,
+            "first_target": target,
+        },
+        owner_team_id=owner_team_id,
+    )
+    # Hold the message in 'failed' so _deliver_pending_messages does not race
+    # the scheduled retry. The scheduler consumer resets it to 'accepted' just
+    # before re-delivery.
+    store.mark(message_id, "failed", "trust_retry_scheduled")
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_needed",
+        message_id=message_id,
+        workspace=str(workspace),
+        pane_id=injection.get("pane_id") or target,
+        target=target,
+        reason="trust_prompt_not_dismissed_after_answer",
+        attempt=attempt,
+    )
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_scheduled",
+        message_id=message_id,
+        workspace=str(workspace),
+        scheduled_event_id=event_id,
+        due_at=due_at,
+        next_attempt=next_attempt,
+        max_attempts=_TRUST_RETRY_MAX_ATTEMPTS,
+        backoff_seconds=backoff,
+    )
+    return {
+        "ok": False,
+        "status": "retry_scheduled",
+        "reason": "trust_prompt_not_dismissed_after_answer",
+        "stage": "trust_auto_answer_dismissal_wait",
+        "verification": "trust_prompt_not_dismissed_after_answer",
+        "scheduled_event_id": event_id,
+        "scheduled_retry_at": due_at,
+        "next_attempt": next_attempt,
+        "max_attempts": _TRUST_RETRY_MAX_ATTEMPTS,
+        "detected": injection.get("detected"),
+        "pane_id": injection.get("pane_id"),
+        "pane_mode": injection.get("pane_mode"),
+        "pane_capture_tail": injection.get("pane_capture_tail"),
+    }
+
+
+def _message_owner_team_id(store: MessageStore, message_id: str) -> str | None:
+    row = _message_by_id(store, message_id)
+    if not row:
+        return None
+    owner = row.get("owner_team_id")
+    return str(owner) if owner else None
+
+
+def _execute_trust_retry(
+    workspace: Path,
+    store: MessageStore,
+    event_log: EventLog,
+    payload: dict[str, Any],
+    *,
+    owner_team_id: str | None = None,
+) -> dict[str, Any]:
+    """Scheduler-side consumer for kind='trust_retry'. Resets the message back
+    to 'accepted' so claim_for_delivery succeeds, re-runs _deliver_pending_message,
+    and either succeeds, escalates to a further retry (via _handle_trust_retry_needed),
+    or hits the terminal exhausted branch.
+    """
+    from team_agent.state import load_runtime_state
+    message_id = str(payload.get("message_id") or "")
+    if not message_id:
+        return {"ok": False, "reason": "trust_retry_missing_message_id"}
+    attempt = int(payload.get("attempt") or 1)
+    row = _message_by_id(store, message_id)
+    if not row:
+        event_log.write(
+            "leader_panes.trust_auto_answer_retry_skipped",
+            message_id=message_id,
+            reason="message_missing",
+            attempt=attempt,
+        )
+        return {"ok": False, "reason": "message_missing"}
+    # Reset to accepted so claim_for_delivery succeeds. The previous attempt
+    # left the row in 'failed' status with reason='trust_retry_scheduled'.
+    store.mark(message_id, "accepted", "trust_retry_resuming")
+    event_log.write(
+        "leader_panes.trust_auto_answer_retry_attempted",
+        message_id=message_id,
+        workspace=str(workspace),
+        attempt=attempt,
+        max_attempts=int(payload.get("max_attempts") or _TRUST_RETRY_MAX_ATTEMPTS),
+    )
+    state = load_runtime_state(workspace)
+    if owner_team_id and isinstance(state.get("teams"), dict):
+        scoped = state["teams"].get(owner_team_id)
+        if isinstance(scoped, dict):
+            state = scoped
+    delivery_result = _deliver_pending_message(
+        workspace, state, message_id,
+        wait_visible=True, timeout=30.0,
+        _trust_retry_attempt=attempt,
+    )
+    return delivery_result
+
+
+def _stamp_first_send_at_if_leader_to_worker(
+    state: dict[str, Any],
+    row: dict[str, Any],
+    event_log: EventLog | None = None,
+) -> None:
+    """Route B atomicity (2026-05-27): record the first time the leader
+    successfully sends work to each worker. The presence of this stamp drives
+    restart's resumability decision — a worker the leader has interacted with
+    has accumulated conversation state, so a missing session_id at restart
+    time IS an atomicity violation. A worker that has never received work
+    legitimately fresh-starts during restart.
+
+    Only stamped once per worker (idempotent across re-sends). Only fires on
+    leader -> worker sends; worker-to-worker peer messages do not count.
+    The mutation lives on the state dict the caller already saves
+    (`save_team_scoped_state` in send.py, or `save_runtime_state` after
+    coordinator_tick), so persistence is automatic.
+
+    C1 (cr verdict, 2026-05-27): when the stamp transitions null -> ts (the
+    one-time write), emit a `worker.first_interaction` audit event with
+    worker_id, first_send_at, message_id. Re-sends to the same worker hit the
+    idempotency guard above and do NOT re-emit. Worker-to-worker peer sends
+    short-circuit at the sender check and do NOT emit.
+    """
+    sender = str(row.get("sender") or "")
+    recipient = str(row.get("recipient") or "")
+    if not recipient:
+        return
+    leader_id = str((state.get("leader") or {}).get("id") or "leader")
+    if sender not in {"leader", "Leader", leader_id}:
+        return
+    agents = state.get("agents")
+    if not isinstance(agents, dict):
+        return
+    agent_state = agents.get(recipient)
+    if not isinstance(agent_state, dict):
+        return
+    if agent_state.get("first_send_at"):
+        return
+    stamp = datetime.now(timezone.utc).isoformat()
+    agent_state["first_send_at"] = stamp
+    if event_log is not None:
+        event_log.write(
+            "worker.first_interaction",
+            worker_id=recipient,
+            first_send_at=stamp,
+            message_id=str(row.get("message_id") or ""),
+        )
+
+
+def _wait_for_trust_prompt_dismissal(target: str, *, timeout: float = 3.0, poll_interval: float = 0.1) -> bool:
+    """Spark MEDIUM #4: bounded poll for trust prompt dismissal. Returns True once
+    the pane no longer matches detect_non_input_scrollback, False if the prompt
+    is still present after `timeout` seconds. Uses the same detector the inject
+    path uses so behaviour stays consistent."""
+    import time as _time
+    from team_agent.messaging.tmux_prompt import detect_non_input_scrollback
+    deadline = _time.monotonic() + max(timeout, 0.0)
+    while True:
+        capture = _capture_pane_tail(target)
+        detected = detect_non_input_scrollback(capture)
+        if detected != "codex_trust_prompt":
+            return True
+        if _time.monotonic() >= deadline:
+            return False
+        _time.sleep(poll_interval)
+
+
+def _capture_pane_tail(target: str) -> str:
+    from team_agent.messaging.deps import _capture_tmux_pane_text
+    capture = _capture_tmux_pane_text(target)
+    if not capture.get("ok"):
+        return ""
+    return str(capture.get("capture") or "")
+
+
 def _deliver_pending_messages(workspace: Path, state: dict[str, Any], event_log: EventLog) -> list[str]:
     store = MessageStore(workspace)
     delivered: list[str] = []

package/src/team_agent/messaging/leader.py

@@ -251,6 +251,19 @@ def _send_to_leader_receiver(
         f"team-agent-leader-receiver-{message_id}",
         provider=receiver.get("provider", "codex"),
     )
+    if not injection.get("ok") and injection.get("detected") == "codex_trust_prompt":
+        from team_agent.messaging.trust_auto_answer import retry_injection_after_trust_auto_answer
+        injection = retry_injection_after_trust_auto_answer(
+            workspace,
+            state,
+            event_log,
+            injection,
+            target,
+            text,
+            submit_key,
+            f"team-agent-leader-receiver-{message_id}-trust-retry",
+            receiver.get("provider", "codex"),
+        )
     if injection["ok"]:
         store.mark(message_id, "submitted")
         event_log.write(
@@ -466,10 +479,6 @@ def _format_team_agent_message(payload: dict[str, Any]) -> str:
 
 
 
-
-
-
-
 
 
 

package/src/team_agent/messaging/leader_api_errors.py

@@ -0,0 +1,216 @@
+"""Gap 28 (Slice 2 Stage 2): observe-only detection of leader-pane API errors.
+
+The coordinator tick captures the leader pane scrollback once per cycle, scans it for
+known upstream-API error patterns (Claude/Codex CLI errors that occur mid-turn), and
+emits a structured `leader.api_error` audit event. The intent is observability — auto-
+retry belongs to the upstream CLI; this module never touches the pane.
+
+Event schema (logged via EventLog.write):
+
+  event:                          'leader.api_error'
+  ts:                             ISO-8601 UTC (added by EventLog)
+  leader_session_uuid:            str | None
+  error_class:                    'Overloaded' | 'RateLimit' | 'Timeout' |
+                                  'NetworkError' | 'Unknown'
+  provider:                       'claude' | 'codex' | 'claude_code' | str | None
+  partial_response_streamed:      bool  (heuristic: assistant text before the error)
+  worker_dispatch_just_before:    list[str]  (leader→worker msg_ids in the prior 60s)
+  retry_count:                    int   (always 0 — the framework does not retry today)
+  matched_pattern_snippet:        str   (the captured error line, ≤160 chars)
+
+Detection dedupes within the coordinator state via a (error_class, snippet-tail)
+fingerprint stored under `state['coordinator']['last_api_error_fingerprint']`. A
+clean tick (no error pattern present) clears the fingerprint so the next genuine
+error re-emits. This keeps event volume bounded while still catching distinct
+errors as they occur.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from team_agent.events import EventLog
+from team_agent.message_store import MessageStore
+
+
+# Spark MEDIUM sweeps (2026-05-26):
+# (#3) Require an API/provider context marker near the error keyword. Bare '503' /
+#      'fetch failed' / 'timed out' in user text used to false-fire.
+# (#7) Match across short sliding windows of 1-3 adjacent lines so wrapped tmux
+#      output ("claude:\n  request timed out") still resolves to a single
+#      detection. Window joined with a single space; capped at _WINDOW_MAX_CHARS
+#      so the scan stays bounded.
+_API_CONTEXT = (
+    r"(?:API\s+Error|HTTP\s*Error|HTTPError|request\s+failed|"
+    r"codex|claude|Anthropic|OpenAI|TypeError)"
+)
+
+# Patterns operate against a sliding window of up to 3 joined lines. The window
+# never contains '\n' (lines are joined with a single space), so `[^\n]` and `.`
+# behave the same; we use `[^\n]` for self-documentation.
+_ERROR_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    # Overloaded — keyword itself already includes the "API Error:" prefix.
+    (re.compile(r"API\s+Error:\s*Overloaded", re.IGNORECASE), "Overloaded"),
+    # RateLimit — 429 with "Too Many Requests" is sufficiently specific; require it
+    # appear AFTER an API context marker OR before "Too Many Requests" tightly.
+    (re.compile(rf"(?:{_API_CONTEXT}[^\n]*\b429\b|\b429\s+Too\s+Many\s+Requests)", re.IGNORECASE), "RateLimit"),
+    # 5xx — must share a window with an API-context marker on either side.
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}\b5(?:00|02|03|04)\b", re.IGNORECASE), "NetworkError"),
+    (re.compile(rf"\b5(?:00|02|03|04)\b[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "NetworkError"),
+    # fetch failed — needs an API-context marker in the same window. The TypeError
+    # marker on its own counts (Node fetch frames the error this way).
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}fetch\s+failed", re.IGNORECASE), "NetworkError"),
+    (re.compile(rf"fetch\s+failed[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "NetworkError"),
+    # Timeout — likewise requires an API-context marker in the window, except for
+    # the unambiguous syscall token ETIMEDOUT.
+    (re.compile(rf"{_API_CONTEXT}[^\n]{{0,200}}(?:request|connection)\s+(?:timed\s+out|timeout)", re.IGNORECASE), "Timeout"),
+    (re.compile(rf"(?:request|connection)\s+(?:timed\s+out|timeout)[^\n]{{0,200}}{_API_CONTEXT}", re.IGNORECASE), "Timeout"),
+    (re.compile(r"\bETIMEDOUT\b", re.IGNORECASE), "Timeout"),
+]
+
+_RECENT_LINE_WINDOW = 100        # scan only the most recent N lines
+_SLIDING_WINDOW_LINES = 3        # join up to 3 adjacent lines per scan window
+_WINDOW_MAX_CHARS = 400          # discard windows beyond this length to bound work
+_DISPATCH_WINDOW_SECONDS = 60    # leader→worker sends counted within this lookback
+_PARTIAL_RESPONSE_HEAD_BYTES = 4000
+
+_PARTIAL_RESPONSE_HINT = re.compile(
+    r"(?:^|\n)\s*(?:Assistant|⏺|●|> |I'll |I will |I'm |I am |Let me )",
+    re.IGNORECASE,
+)
+
+
+def detect_leader_api_errors(
+    workspace: Path,
+    state: dict[str, Any],
+    store: MessageStore,
+    event_log: EventLog,
+    *,
+    capture_fn: Callable[[str], dict[str, Any]] | None = None,
+    now_fn: Callable[[], datetime] | None = None,
+) -> list[dict[str, Any]]:
+    """Coordinator-tick entry point. Returns a list of emitted events (0 or 1)."""
+    receiver = state.get("leader_receiver") or {}
+    pane = receiver.get("pane_id") if receiver.get("mode") == "direct_tmux" else None
+    if not pane:
+        return []
+    capture_fn = capture_fn or _default_capture_fn()
+    capture = capture_fn(str(pane))
+    if not capture.get("ok"):
+        return []
+    scrollback = str(capture.get("capture") or "")
+    coordinator_state = state.setdefault("coordinator", {})
+    found = _match_first_error(scrollback)
+    if not found:
+        if coordinator_state.get("last_api_error_fingerprint"):
+            coordinator_state["last_api_error_fingerprint"] = None
+        return []
+    error_class, snippet = found
+    fingerprint = f"{error_class}::{snippet[-120:]}"
+    if coordinator_state.get("last_api_error_fingerprint") == fingerprint:
+        return []
+    coordinator_state["last_api_error_fingerprint"] = fingerprint
+    now = (now_fn() if now_fn else datetime.now(timezone.utc))
+    cutoff_iso = (now - timedelta(seconds=_DISPATCH_WINDOW_SECONDS)).isoformat()
+    leader_uuid = (
+        str((state.get("team_owner") or {}).get("leader_session_uuid") or "")
+        or str(receiver.get("leader_session_uuid") or "")
+        or None
+    )
+    provider = str(receiver.get("provider") or "") or None
+    event = event_log.write(
+        "leader.api_error",
+        leader_session_uuid=leader_uuid,
+        error_class=error_class,
+        provider=provider,
+        partial_response_streamed=_scrollback_has_partial_response(scrollback, snippet),
+        worker_dispatch_just_before=_recent_leader_dispatches(store, cutoff_iso),
+        retry_count=0,
+        matched_pattern_snippet=snippet[:160],
+    )
+    return [event]
+
+
+def _default_capture_fn() -> Callable[[str], dict[str, Any]]:
+    from team_agent.messaging.deps import _capture_tmux_pane_text
+    return _capture_tmux_pane_text
+
+
+def _match_first_error(scrollback: str) -> tuple[str, str] | None:
+    """Spark MEDIUM #7: sliding window of 1..N adjacent lines. Lines inside a
+    window are joined with a single space so a wrapped pair such as
+        claude:
+          request timed out
+    is detected as one event without permitting unbounded cross-line matches.
+    Latest window wins so the freshest error is reported."""
+    if not scrollback:
+        return None
+    lines = [line.strip() for line in scrollback.splitlines()[-_RECENT_LINE_WINDOW:]]
+    if not lines:
+        return None
+    best: tuple[int, str, str] | None = None
+    for start in range(len(lines)):
+        for size in range(1, _SLIDING_WINDOW_LINES + 1):
+            end = start + size
+            if end > len(lines):
+                break
+            window = " ".join(line for line in lines[start:end] if line)
+            if not window:
+                continue
+            # Spark MEDIUM sweep #3 (2026-05-26): tail-preserve instead of
+            # dropping the window wholesale. Errors land at the END of verbose
+            # diagnostics (stack traces, retry chatter, etc.). If we discarded
+            # any window over the cap we silently lost recall on long wrapped
+            # output. Scanning the LAST _WINDOW_MAX_CHARS still bounds regex
+            # cost while keeping the freshest context — the bit most likely to
+            # contain the actual provider error keyword.
+            if len(window) > _WINDOW_MAX_CHARS:
+                window = window[-_WINDOW_MAX_CHARS:]
+            for pattern, error_class in _ERROR_PATTERNS:
+                match = pattern.search(window)
+                if not match:
+                    continue
+                snippet = window[:240]
+                if best is None or start > best[0]:
+                    best = (start, error_class, snippet)
+                # First match per window is enough; later windows may override.
+                break
+    if best is None:
+        return None
+    return best[1], best[2]
+
+
+def _scrollback_has_partial_response(scrollback: str, error_snippet: str) -> bool:
+    idx = scrollback.rfind(error_snippet)
+    if idx == -1:
+        return False
+    head = scrollback[max(0, idx - _PARTIAL_RESPONSE_HEAD_BYTES): idx]
+    return bool(_PARTIAL_RESPONSE_HINT.search(head))
+
+
+def _recent_leader_dispatches(store: MessageStore, cutoff_iso: str) -> list[str]:
+    out: list[str] = []
+    try:
+        rows = store.messages()
+    except Exception:
+        return out
+    for row in rows:
+        sender = str(row.get("sender") or "")
+        if sender not in {"leader", "Leader"} and not _looks_like_leader_sender(sender):
+            continue
+        created = str(row.get("created_at") or "")
+        if not created or created < cutoff_iso:
+            continue
+        msg_id = str(row.get("message_id") or "")
+        if msg_id:
+            out.append(msg_id)
+    return out
+
+
+def _looks_like_leader_sender(sender: str) -> bool:
+    return sender.startswith("leader") or sender.lower() == "leader"
+
+
+__all__ = ["detect_leader_api_errors"]