@team-agent/installer 0.1.11 → 0.2.1

package/src/team_agent/message_store/result_watchers.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import uuid
+from contextlib import closing
+from typing import Any
+
+from team_agent.message_store.schema import utcnow
+
+
+def create_result_watcher(
+    self,
+    task_id: str | None,
+    agent_id: str | None,
+    message_id: str | None,
+    leader_id: str = "leader",
+    owner_team_id: str | None = None,
+) -> str:
+    watcher_id = f"watch_{uuid.uuid4().hex[:12]}"
+    with closing(self.connect()) as conn:
+        with conn:
+            conn.execute(
+                """
+                insert into result_watchers(
+                  watcher_id, owner_team_id, task_id, agent_id, message_id, leader_id, status, created_at
+                )
+                values (?, ?, ?, ?, ?, ?, 'pending', ?)
+                """,
+                (watcher_id, owner_team_id, task_id, agent_id, message_id, leader_id, utcnow()),
+            )
+    return watcher_id
+
+def pending_result_watchers(self, owner_team_id: str | None = None) -> list[dict[str, Any]]:
+    with closing(self.connect()) as conn:
+        if owner_team_id is None:
+            rows = conn.execute("select * from result_watchers where status = 'pending' order by created_at").fetchall()
+        else:
+            rows = conn.execute(
+                """
+                select * from result_watchers
+                where status = 'pending' and (owner_team_id = ? or owner_team_id is null)
+                order by created_at
+                """,
+                (owner_team_id,),
+            ).fetchall()
+    return [dict(row) for row in rows]
+
+def retryable_result_watchers(self) -> list[dict[str, Any]]:
+    with closing(self.connect()) as conn:
+        rows = conn.execute(
+            "select * from result_watchers where status in ('pending', 'notify_failed') order by created_at"
+        ).fetchall()
+    return [dict(row) for row in rows]
+
+def result_watchers(self, owner_team_id: str | None = None) -> list[dict[str, Any]]:
+    with closing(self.connect()) as conn:
+        if owner_team_id is None:
+            rows = conn.execute("select * from result_watchers order by created_at").fetchall()
+        else:
+            rows = conn.execute(
+                "select * from result_watchers where owner_team_id = ? or owner_team_id is null order by created_at",
+                (owner_team_id,),
+            ).fetchall()
+    return [dict(row) for row in rows]
+
+def mark_result_watcher(
+    self,
+    watcher_id: str,
+    status: str,
+    result_id: str | None = None,
+    notified_message_id: str | None = None,
+    error: str | None = None,
+) -> None:
+    with closing(self.connect()) as conn:
+        with conn:
+            conn.execute(
+                """
+                update result_watchers
+                set status = ?,
+                    completed_at = ?,
+                    result_id = coalesce(?, result_id),
+                    notified_message_id = coalesce(?, notified_message_id),
+                    error = coalesce(?, error)
+                where watcher_id = ?
+                """,
+                (status, utcnow(), result_id, notified_message_id, error, watcher_id),
+            )
+
+
+def requeue_delivery_exhausted_watchers(self) -> list[str]:
+    with closing(self.connect()) as conn:
+        with conn:
+            rows = conn.execute(
+                "select watcher_id from result_watchers where status = 'delivery_exhausted'"
+            ).fetchall()
+            watcher_ids = [row[0] for row in rows]
+            if watcher_ids:
+                conn.execute(
+                    "update result_watchers "
+                    "set status = 'notify_failed', error = null, notified_message_id = null, completed_at = null "
+                    "where status = 'delivery_exhausted'"
+                )
+    return watcher_ids

package/src/team_agent/message_store/schema.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import sqlite3
+from datetime import datetime, timezone
+
+
+MESSAGE_COLUMNS = {
+    "owner_team_id",
+    "message_id",
+    "task_id",
+    "sender",
+    "recipient",
+    "reply_to",
+    "requires_ack",
+    "status",
+    "content",
+    "artifact_refs",
+    "created_at",
+    "updated_at",
+    "delivered_at",
+    "acknowledged_at",
+    "error",
+    "delivery_attempts",
+}
+RESULT_COLUMNS = {"result_id", "task_id", "agent_id", "envelope", "status", "created_at"}
+SCHEDULED_EVENT_COLUMNS = {
+    "id",
+    "owner_team_id",
+    "due_at",
+    "target",
+    "kind",
+    "payload_json",
+    "status",
+    "created_at",
+    "fired_at",
+    "result_json",
+}
+DELIVERY_TOKEN_COLUMNS = {
+    "message_id",
+    "unique_token",
+    "injected_at",
+    "visible_at",
+    "consumed_at",
+    "failed_at",
+    "failure_reason",
+}
+AGENT_HEALTH_COLUMNS = {
+    "owner_team_id",
+    "agent_id",
+    "status",
+    "last_output_at",
+    "context_usage_pct",
+    "current_task_id",
+    "updated_at",
+}
+PEER_ALLOWLIST_COLUMNS = {"a", "b", "created_at"}
+RESULT_WATCHER_COLUMNS = {
+    "owner_team_id",
+    "watcher_id",
+    "task_id",
+    "agent_id",
+    "message_id",
+    "leader_id",
+    "status",
+    "created_at",
+    "completed_at",
+    "result_id",
+    "notified_message_id",
+    "error",
+}
+
+
+def utcnow() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+SCHEMA_VERSION = 3
+
+
+def _table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
+    return {row[1] for row in conn.execute(f"pragma table_info({table})").fetchall()}
+
+
+def _ensure_table_columns(
+    conn: sqlite3.Connection,
+    table: str,
+    required: set[str],
+    migrations: dict[str, str] | None = None,
+) -> None:
+    columns = _table_columns(conn, table)
+    missing = required - columns
+    migrations = migrations or {}
+    unsupported = missing - set(migrations)
+    if unsupported:
+        names = ", ".join(sorted(unsupported))
+        raise RuntimeError(f"team.db table {table} is missing required column(s): {names}")
+    for name in sorted(missing):
+        conn.execute(migrations[name])
+
+
+def initialize_schema(conn: sqlite3.Connection) -> None:
+    with conn:
+        conn.execute(
+            """
+            create table if not exists messages (
+              message_id text primary key,
+              owner_team_id text,
+              task_id text,
+              sender text,
+              recipient text,
+              reply_to text,
+              requires_ack integer,
+              status text,
+              content text,
+              artifact_refs text,
+              created_at text,
+              updated_at text,
+              delivered_at text,
+              acknowledged_at text,
+              error text,
+              delivery_attempts integer not null default 0
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists results (
+              result_id text primary key,
+              task_id text not null,
+              agent_id text not null,
+              envelope text not null,
+              status text not null,
+              created_at text not null
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists scheduled_events (
+              id integer primary key,
+              owner_team_id text,
+              due_at text not null,
+              target text not null,
+              kind text not null,
+              payload_json text not null,
+              status text not null,
+              created_at text not null,
+              fired_at text,
+              result_json text
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists delivery_tokens (
+              message_id text primary key,
+              unique_token text not null,
+              injected_at text not null,
+              visible_at text,
+              consumed_at text,
+              failed_at text,
+              failure_reason text
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists agent_health (
+              owner_team_id text,
+              agent_id text not null,
+              status text not null,
+              last_output_at text,
+              context_usage_pct integer,
+              current_task_id text,
+              updated_at text not null,
+              unique(owner_team_id, agent_id)
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists peer_allowlist (
+              a text not null,
+              b text not null,
+              created_at text not null,
+              primary key (a, b)
+            )
+            """
+        )
+        conn.execute(
+            """
+            create table if not exists result_watchers (
+              watcher_id text primary key,
+              owner_team_id text,
+              task_id text,
+              agent_id text,
+              message_id text,
+              leader_id text not null,
+              status text not null,
+              created_at text not null,
+              completed_at text,
+              result_id text,
+              notified_message_id text,
+              error text
+            )
+            """
+        )
+        _ensure_table_columns(
+            conn,
+            "messages",
+            MESSAGE_COLUMNS,
+            {
+                "delivery_attempts": (
+                    "alter table messages add column delivery_attempts integer not null default 0"
+                ),
+                "owner_team_id": "alter table messages add column owner_team_id text",
+            },
+        )
+        _ensure_table_columns(conn, "results", RESULT_COLUMNS)
+        _ensure_table_columns(
+            conn,
+            "scheduled_events",
+            SCHEDULED_EVENT_COLUMNS,
+            {"owner_team_id": "alter table scheduled_events add column owner_team_id text"},
+        )
+        _ensure_table_columns(conn, "delivery_tokens", DELIVERY_TOKEN_COLUMNS)
+        _migrate_agent_health_owner_team_id(conn)
+        _ensure_table_columns(conn, "peer_allowlist", PEER_ALLOWLIST_COLUMNS)
+        _ensure_table_columns(
+            conn,
+            "result_watchers",
+            RESULT_WATCHER_COLUMNS,
+            {"owner_team_id": "alter table result_watchers add column owner_team_id text"},
+        )
+        conn.execute("create index if not exists idx_messages_owner_team_id on messages(owner_team_id)")
+        conn.execute("create index if not exists idx_scheduled_events_owner_team_id on scheduled_events(owner_team_id)")
+        conn.execute("create index if not exists idx_agent_health_owner_team_id on agent_health(owner_team_id)")
+        conn.execute("create index if not exists idx_result_watchers_owner_team_id on result_watchers(owner_team_id)")
+        conn.execute(f"pragma user_version = {SCHEMA_VERSION}")
+
+
+def _migrate_agent_health_owner_team_id(conn: sqlite3.Connection) -> None:
+    columns = _table_columns(conn, "agent_health")
+    if "owner_team_id" not in columns:
+        conn.execute(
+            """
+            create table agent_health_new (
+              owner_team_id text,
+              agent_id text not null,
+              status text not null,
+              last_output_at text,
+              context_usage_pct integer,
+              current_task_id text,
+              updated_at text not null,
+              unique(owner_team_id, agent_id)
+            )
+            """
+        )
+        conn.execute(
+            """
+            insert into agent_health_new(owner_team_id, agent_id, status, last_output_at, context_usage_pct, current_task_id, updated_at)
+            select null, agent_id, status, last_output_at, context_usage_pct, current_task_id, updated_at from agent_health
+            """
+        )
+        conn.execute("drop table agent_health")
+        conn.execute("alter table agent_health_new rename to agent_health")
+    _ensure_table_columns(conn, "agent_health", AGENT_HEALTH_COLUMNS)

package/src/team_agent/messaging/__init__.py

@@ -0,0 +1 @@
+"""Messaging/delivery/leader-receiver lane package."""

package/src/team_agent/messaging/activity_detector.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from pathlib import Path
+import re
+from typing import Any
+
+from team_agent.events import EventLog
+from team_agent.messaging.deps import datetime, save_runtime_state, team_state_key, timezone
+
+_COMPACTION_RESET_THRESHOLD_DEFAULT = 3
+
+
+def _compaction_reset_threshold(state: dict[str, Any]) -> int:
+    from team_agent.messaging.deps import load_spec
+    spec_path = state.get("spec_path")
+    if spec_path:
+        try:
+            spec = load_spec(spec_path)
+        except Exception:
+            spec = {}
+        runtime_cfg = spec.get("runtime", {}) if isinstance(spec, dict) else {}
+        raw = runtime_cfg.get("compaction_reset_threshold")
+        if raw is not None:
+            try:
+                value = int(raw)
+                if value > 0:
+                    return value
+            except (TypeError, ValueError):
+                pass
+    return _COMPACTION_RESET_THRESHOLD_DEFAULT
+_PROVIDER_COMMANDS = {"claude", "claude-code", "codex", "node", "node.exe", "claude.exe"}
+_COMPACTION_PATTERNS = (
+    re.compile(r"context compacted", re.IGNORECASE),
+    re.compile(r"compaction occurred", re.IGNORECASE),
+)
+_IDLE_PROMPT_PATTERNS = (
+    re.compile(r"›\s*Find and fix a bug in @filename"),
+    re.compile(r"─\s*for agents"),
+    re.compile(r"^›[^\n]*\n(?:\s*\n){0,8}\s*gpt-[\w.-]+\s+\S+\s+·", re.MULTILINE),
+)
+_WORKING_PATTERNS = (
+    re.compile(r"\bWorking(?:\s*\((?P<working_seconds>\d+)s\))?", re.IGNORECASE),
+    re.compile(r"\bReticulating\b", re.IGNORECASE),
+    re.compile(r"\bBaked for (?P<baked_seconds>\d+)s\b", re.IGNORECASE),
+    re.compile(r"\bThinking\b", re.IGNORECASE),
+    re.compile(r"esc to interrupt", re.IGNORECASE),
+    re.compile(r"[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]"),
+)
+
+
+def classify_agent_activity(
+    agent_id: str,
+    provider: str,
+    last_output_at: str | None,
+    pane: dict[str, Any] | None,
+    scrollback: str,
+    *,
+    now: datetime | None = None,
+    stuck_timeout_sec: int = 300,
+) -> dict[str, Any]:
+    _ = agent_id, provider
+    now = now or datetime.now(timezone.utc)
+    pane = pane or {}
+    pane_in_mode = str(pane.get("pane_in_mode") or "0")
+    if pane_in_mode != "0":
+        return {"status": "uncertain", "confidence": 0.9, "rationale": f"pane_in_mode={pane_in_mode}"}
+    command = str(pane.get("pane_current_command") or "").split("/")[-1]
+    if command and command not in _PROVIDER_COMMANDS:
+        return {"status": "uncertain", "confidence": 0.75, "rationale": f"unexpected pane current_command={command}"}
+    working = _latest_working_match(scrollback)
+    idle_pos = _latest_idle_prompt_position(scrollback)
+    if idle_pos is not None and (working is None or idle_pos > working[0]):
+        return {"status": "idle", "confidence": 0.9, "rationale": "provider idle prompt is the latest scrollback signal"}
+    if working:
+        _pos, label, elapsed = working
+        if elapsed is not None and elapsed >= stuck_timeout_sec:
+            return {"status": "stuck", "confidence": 0.85, "rationale": f"stale {label} indicator for {elapsed}s"}
+        return {"status": "working", "confidence": 0.9, "rationale": f"{label} indicator is the latest scrollback signal"}
+    age = _last_output_age_seconds(last_output_at, now)
+    if age is not None and age >= stuck_timeout_sec:
+        return {"status": "stuck", "confidence": 0.85, "rationale": "last_output_at exceeded timeout with no idle prompt"}
+    if age is not None and age <= 120 and (not command or command in _PROVIDER_COMMANDS):
+        return {"status": "working", "confidence": 0.7, "rationale": "recent output from provider command"}
+    return {"status": "uncertain", "confidence": 0.5, "rationale": "no decisive prompt or working signal"}
+
+
+def _latest_idle_prompt_position(scrollback: str) -> int | None:
+    best: int | None = None
+    for pattern in _IDLE_PROMPT_PATTERNS:
+        for match in pattern.finditer(scrollback):
+            if best is None or match.start() > best:
+                best = match.start()
+    return best
+
+
+def detect_compaction_degradation(
+    workspace: Path,
+    state: dict[str, Any],
+    event_log: EventLog,
+    *,
+    agent_id: str,
+    provider: str,
+    scrollback: str,
+    stuck_loop: bool = False,
+) -> dict[str, Any]:
+    count = _count_compaction_markers(scrollback)
+    owner_team_id = team_state_key(state)
+    team_counts = state.setdefault("coordinator", {}).setdefault("compaction_counts", {}).setdefault(owner_team_id, {})
+    current = max(int(team_counts.get(agent_id) or 0), count)
+    team_counts[agent_id] = current
+    save_runtime_state(workspace, state)
+    if current <= 0:
+        return {"ok": True, "event": "compaction_threshold_crossed.none", "compaction_count": current}
+    event_log.write(
+        "coordinator.compaction_observed",
+        agent_id=agent_id,
+        provider=provider,
+        team=owner_team_id,
+        compaction_count=current,
+        stuck_loop=stuck_loop,
+    )
+    if provider != "codex":
+        event = "compaction_threshold_crossed.ignored_lossless_provider"
+        event_log.write(event, agent_id=agent_id, provider=provider, team=owner_team_id, compaction_count=current)
+        return {"ok": True, "event": event, "agent_id": agent_id, "provider": provider, "compaction_count": current}
+    threshold = _compaction_reset_threshold(state)
+    if current < threshold and not (current >= 1 and stuck_loop):
+        return {"ok": True, "event": "compaction_threshold_crossed.below_threshold", "agent_id": agent_id, "compaction_count": current, "threshold": threshold}
+    return _reset_or_recommend(workspace, state, event_log, agent_id, provider, owner_team_id, current, threshold)
+
+
+def _reset_or_recommend(
+    workspace: Path,
+    state: dict[str, Any],
+    event_log: EventLog,
+    agent_id: str,
+    provider: str,
+    owner_team_id: str,
+    compaction_count: int,
+    threshold: int,
+) -> dict[str, Any]:
+    from team_agent.runtime import reset_agent
+    reset = reset_agent(workspace, agent_id, discard_session=True)
+    if reset.get("ok"):
+        team_counts = state.setdefault("coordinator", {}).setdefault("compaction_counts", {}).setdefault(owner_team_id, {})
+        team_counts[agent_id] = 0
+        save_runtime_state(workspace, state)
+        event = "compaction_threshold_crossed.auto_reset"
+        event_log.write(event, agent_id=agent_id, provider=provider, team=owner_team_id, compaction_count=compaction_count, threshold=threshold)
+        return {"ok": True, "event": event, "agent_id": agent_id, "compaction_count": compaction_count, "threshold": threshold, "reset": reset}
+    event = "compaction_threshold_crossed.recommend_reset"
+    message = f"agent {agent_id} crossed Codex compaction threshold; run team-agent reset-agent {agent_id} --discard-session"
+    event_log.write(
+        event,
+        agent_id=agent_id,
+        provider=provider,
+        team=owner_team_id,
+        compaction_count=compaction_count,
+        threshold=threshold,
+        leader_visible_message=message,
+        reset_error=reset.get("error") or reset.get("reason"),
+    )
+    return {"ok": True, "event": event, "agent_id": agent_id, "compaction_count": compaction_count, "threshold": threshold, "leader_visible_message": message, "reset": reset}
+
+
+def _latest_working_match(scrollback: str) -> tuple[int, str, int | None] | None:
+    best: tuple[int, str, int | None] | None = None
+    for pattern in _WORKING_PATTERNS:
+        for match in pattern.finditer(scrollback):
+            elapsed_raw = match.groupdict().get("working_seconds") or match.groupdict().get("baked_seconds")
+            elapsed = int(elapsed_raw) if elapsed_raw else None
+            if best is None or match.start() > best[0]:
+                best = (match.start(), match.group(0), elapsed)
+    return best
+
+
+def _last_output_age_seconds(last_output_at: str | None, now: datetime) -> float | None:
+    if not last_output_at:
+        return None
+    try:
+        last = datetime.fromisoformat(last_output_at)
+    except ValueError:
+        return None
+    if last.tzinfo is None:
+        last = last.replace(tzinfo=timezone.utc)
+    return max(0.0, (now - last).total_seconds())
+
+
+def _count_compaction_markers(scrollback: str) -> int:
+    return sum(len(pattern.findall(scrollback)) for pattern in _COMPACTION_PATTERNS)