@maudecode/talaria-web 1.18.0 → 1.20.0

package/sidecar/talaria_sidecar/methods/process.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import json
 import logging
 import queue
+import re
 import sqlite3
 import threading
 import time
@@ -14,7 +15,7 @@ from contextlib import closing
 from pathlib import Path
 
 from ..errors import InvalidParams, RpcError
-from ..home import profile_home_param, scoped_home
+from ..home import _PROCESS_HOME, _is_named_profile, profile_home_param, scoped_home
 from ..rpc import CallContext
 
 log = logging.getLogger("talaria_sidecar.process")
@@ -28,11 +29,23 @@ def _registry():
     return process_registry
 
 
+def _is_child_process(row: dict) -> bool:
+    """TAL-933: a subagent's own process notice, which the server surfaces only when the owning profile opts in."""
+    try:
+        from tools.process_registry_notifications import is_child_process_notification
+    except Exception:  # noqa: BLE001 - before Agent 0.21.7, the same rule inline
+        if row.get("type") in ("async_delegation", "async_delegation_work_closeout"):
+            return False
+        return str(row.get("owner_task_id") or row.get("task_id") or "").startswith("sa-")
+    return bool(is_child_process_notification(row))
+
+
 def _event_row(evt: dict) -> dict:
     registry = _registry()
     row = {str(k): v for k, v in evt.items()} if isinstance(evt, dict) else {}
     process_id = str(row.get("delegation_id") or row.get("session_id") or row.get("task_id") or "") if row.get("type") == "async_delegation" else str(row.get("session_id") or "")
     row["process_id"] = process_id
+    row["child_process"] = _is_child_process(row)
     if registry is not None and process_id and (not row.get("session_key") or not row.get("origin_ui_session_id")):
         try:
             proc = registry.get(process_id)
@@ -102,12 +115,112 @@ def _publish_recovered_exits(registry) -> None:
                 _RECOVERED.setdefault(session.id, session)
 
 
+_COMPLETION_BASE_HOME: Path | None = None
+_RESTORED_COMPLETION_HOMES: set[Path] = set()
+
+
+def _completion_homes() -> list[Path]:
+    from hermes_constants import get_hermes_home
+    from .profiles import named_profile_homes
+
+    homes = [_PROCESS_HOME, Path(get_hermes_home())]
+    if _COMPLETION_BASE_HOME is not None:
+        # The server drains one global queue. Re-enumerate so new profiles join and removed/escaped ones leave.
+        homes += [_COMPLETION_BASE_HOME, *(home for _, home in named_profile_homes(_COMPLETION_BASE_HOME))]
+    return list(dict.fromkeys(home.resolve() for home in homes))
+
+
+def _restore_and_sweep_completions(registry) -> None:
+    from tools import async_delegation
+
+    restore = getattr(registry, "restore_completions", None)
+    if not callable(restore):
+        return  # Older Agents already restore at registry import and have no orphan sweep.
+    # The Agent binds the launch home itself, even when the first drain is in a named profile.
+    restore()
+    for home in _completion_homes():
+        try:
+            with scoped_home(home):
+                if _is_named_profile(home) and home not in _RESTORED_COMPLETION_HOMES:
+                    async_delegation.restore_undelivered_completions(registry.completion_queue)
+                    _RESTORED_COMPLETION_HOMES.add(home)
+                sweep = getattr(async_delegation, "maybe_sweep_orphaned_completions", None)
+                if callable(sweep):
+                    sweep(registry.completion_queue)
+        except Exception:  # noqa: BLE001 - retry this home without blocking other profiles or queued events
+            log.debug("Failed to restore or sweep delegation completions for %s", home, exc_info=True)
+
+
+_DELIVERY_RECONCILE_INTERVAL_S = 30
+_LAST_DELIVERY_RECONCILE: dict[Path, float] = {}
+_DELIVERY_RECONCILE_CURSOR: dict[Path, str] = {}
+_DELIVERY_RECONCILE_PAGE_SIZE = 200
+
+
+def _delivered_notifications(home: Path, delegation_id: str | None = None, after: str = "") -> tuple[list[dict], str]:
+    """Only typed transcript deliveries, never every delivered ledger row. No Agent state is changed."""
+    db_path = home / "state.db"
+    if not db_path.exists():
+        return [], ""
+    with closing(sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True, timeout=5)) as conn:
+        tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+        if not {"async_delegations", "messages", "sessions"} <= tables:
+            return [], ""
+        rows = conn.execute("""WITH RECURSIVE candidates AS (
+            SELECT delegation_id, origin_ui_session_id, parent_session_id, origin_session_id, event_json
+            FROM async_delegations WHERE delivery_state='delivered' AND state NOT IN ('running','finalizing')
+            AND event_json IS NOT NULL AND origin_ui_session_id != '' AND delivered_at >= ?
+            AND (? IS NULL OR delegation_id=?) AND delegation_id > ?
+        ), lineage(delegation_id, id) AS (
+            SELECT delegation_id, parent_session_id FROM candidates WHERE parent_session_id != ''
+            UNION SELECT delegation_id, origin_session_id FROM candidates WHERE origin_session_id != ''
+            UNION SELECT l.delegation_id, s.id FROM lineage l
+                JOIN sessions parent ON parent.id=l.id AND parent.end_reason='compression'
+                JOIN sessions s ON s.parent_session_id=l.id
+        ) SELECT DISTINCT c.delegation_id, c.origin_ui_session_id, c.event_json
+        FROM candidates c JOIN lineage l ON l.delegation_id=c.delegation_id
+        JOIN messages m ON m.session_id=l.id
+        WHERE m.role='user' AND m.display_kind='async_delegation_complete'
+        AND json_valid(m.display_metadata)
+        AND json_extract(m.display_metadata, '$.delegation_id')=c.delegation_id
+        AND coalesce(json_extract(m.display_metadata, '$.delivery_notice'), '')=''
+        AND coalesce(json_extract(m.display_metadata, '$.presentation_suppressed'), 0)=0
+        ORDER BY c.delegation_id LIMIT ?""", (time.time() - 48 * 3600, delegation_id, delegation_id, after, _DELIVERY_RECONCILE_PAGE_SIZE)).fetchall()
+    events = []
+    for did, owner, payload in rows:
+        event = _json(payload)
+        if event.get("type") != "async_delegation" or event.get("delegation_id") != did or event.get("origin_ui_session_id") != owner or event.get("task_failure_notice"):
+            continue
+        events.append({**event, "reconciliation_profile_home": str(home.resolve())})
+    return events, rows[-1][0] if len(rows) == _DELIVERY_RECONCILE_PAGE_SIZE else ""
+
+
+def _reconcile_delivered_notifications() -> list[dict]:
+    out = []
+    for home in _completion_homes():
+        now = time.monotonic()
+        if now - _LAST_DELIVERY_RECONCILE.get(home, -float("inf")) < _DELIVERY_RECONCILE_INTERVAL_S:
+            continue
+        try:
+            events, cursor = _delivered_notifications(home, after=_DELIVERY_RECONCILE_CURSOR.get(home, ""))
+            out.extend(events)
+            _DELIVERY_RECONCILE_CURSOR[home] = cursor
+            _LAST_DELIVERY_RECONCILE[home] = now
+        except Exception:  # retry unreadable homes; never block live queue delivery
+            log.warning("Failed to reconcile typed delegation deliveries for %s", home, exc_info=True)
+    return out
+
+
 def drain(max_events: int) -> list[dict]:
     """Pop every queued completion event; the server requeues what it does not own."""
     registry = _registry()
     completion_queue = getattr(registry, "completion_queue", None) if registry is not None else None
     if completion_queue is None:
         return []
+    try:
+        _restore_and_sweep_completions(registry)
+    except Exception:  # noqa: BLE001 - maintenance must not prevent delivery of live queued events
+        log.debug("Failed to reconcile delegation completions", exc_info=True)
     try:
         # A recovered process has no reader thread; only this probe notices its exit and queues its completion (TAL-533).
         with registry._lock:
@@ -117,6 +230,12 @@ def drain(max_events: int) -> list[dict]:
         _publish_recovered_exits(registry)
     except Exception:  # noqa: BLE001
         log.debug("Failed to reconcile recovered processes", exc_info=True)
+    # Use the existing queue and its bound, so reconciliation cannot starve live or foreign events.
+    try:
+        for event in _reconcile_delivered_notifications():
+            completion_queue.put_nowait(event)
+    except Exception:
+        log.warning("Failed to offer typed delegation deliveries", exc_info=True)
     out = []
     while len(out) < max_events:
         try:
@@ -138,7 +257,7 @@ def requeue(events: list[dict]) -> int:
     count = 0
     for evt in events:
         try:
-            completion_queue.put_nowait({k: v for k, v in evt.items() if k not in ("process_id", "consumed")})
+            completion_queue.put_nowait({k: v for k, v in evt.items() if k not in ("process_id", "consumed", "child_process")})
             count += 1
         except Exception:  # noqa: BLE001
             log.debug("Failed to requeue process completion", exc_info=True)
@@ -196,6 +315,15 @@ def claim_delivery(evt: dict, consumer: str) -> str | None:
     if api is None:
         return ""
     claim_id = api.claim_event_delivery(_ledger_event(evt), consumer)
+    if claim_id is None and consumer == "webui" and evt.get("origin_ui_session_id"):
+        from hermes_constants import get_hermes_home
+        home = Path(get_hermes_home()).resolve()
+        if evt.get("reconciliation_profile_home", str(home)) == str(home):
+            # A live queue copy can race the transcript-only consumer too. Recheck the same proof here.
+            transport = {"process_id", "consumed", "child_process", "restored", "startup_replay", "reconciliation_profile_home"}
+            for candidate in _delivered_notifications(home, str(evt.get("delegation_id") or ""))[0]:
+                if {k: v for k, v in candidate.items() if k not in transport} == {k: v for k, v in evt.items() if k not in transport}:
+                    return ""  # Web's session receipt acknowledges this admission, not the Agent ledger.
     return None if claim_id is None else str(claim_id)
 
 
@@ -230,11 +358,14 @@ def defer_delivery(evt: dict, claim_id: str) -> bool:
 
 
 def format_notification(evt: dict) -> str:
-    if evt.get("type") == "async_delegation":
+    if evt.get("type") in ("async_delegation", "heartbeat", "watch_match", "watch_disabled", "watch_overflow_tripped", "watch_overflow_released"):
         try:
-            from tools.process_registry import format_process_notification
+            from tools import process_registry
 
-            return format_process_notification(evt) or ""
+            # Older formatters fall through unknown events to an exit notification.
+            if evt.get("type") == "heartbeat" and not callable(getattr(getattr(process_registry, "ProcessRegistry", None), "_emit_heartbeat", None)):
+                return ""
+            return process_registry.format_process_notification(evt) or ""
         except Exception:  # noqa: BLE001
             return ""
     if evt.get("type") != "completion":
@@ -319,7 +450,7 @@ def _live_delegations() -> dict[str, dict]:
         items = list_async_delegations()
     except Exception:  # noqa: BLE001 - an Agent without the live registry reports from the ledger alone
         return {}
-    return {str(i.get("delegation_id")): {"status": str(i.get("status") or "")} for i in items if isinstance(i, dict) and i.get("delegation_id")}
+    return {str(i.get("delegation_id")): i for i in items if isinstance(i, dict) and i.get("delegation_id")}
 
 
 def _owned_processes(session_ids: set[str]) -> list[dict]:
@@ -369,7 +500,7 @@ def _live_children() -> list[dict]:
     return out
 
 
-def _ledger_children(home: Path, owners: list[str]) -> list[dict]:
+def _ledger_children(home: Path, owners: list[str], dispatched_at: float) -> list[dict]:
     """Finished subagents' own sessions in state.db: ``source = subagent`` rows tagged ``_delegate_from`` with their
     parent's session, each with its start time and first user message (the child's goal, sent verbatim)."""
     db_path = home / "state.db"
@@ -378,10 +509,11 @@ def _ledger_children(home: Path, owners: list[str]) -> list[dict]:
     marks = ",".join("?" * len(owners))
     with closing(sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True, timeout=5)) as conn:
         try:
-            # Only these chats' children: a long-lived profile keeps every subagent session it ever ran.
+            # Bound history before parsing owner JSON; allow 60 seconds of dispatch/start clock skew.
             rows = conn.execute(
                 "SELECT s.id, s.model_config, s.started_at, (SELECT m.content FROM messages m WHERE m.session_id = s.id AND m.role = 'user' ORDER BY m.id LIMIT 1)"
-                f" FROM sessions s WHERE s.source = 'subagent' AND json_extract(s.model_config, '$._delegate_from') IN ({marks})", owners).fetchall()
+                f" FROM sessions s WHERE s.source = 'subagent' AND s.started_at >= ? AND json_extract(s.model_config, '$._delegate_from') IN ({marks})",
+                [dispatched_at - 60, *owners]).fetchall()
         except sqlite3.OperationalError:
             return []
     out = []
@@ -404,12 +536,21 @@ def _goal_text(content) -> str:
         return ""
     if content.startswith(_CONTENT_JSON_PREFIX):
         try:
-            parts = json.loads(content[len(_CONTENT_JSON_PREFIX):])
-        except ValueError:
-            return ""
-        texts = [p.get("text") for p in parts if isinstance(p, dict) and p.get("type") == "text"] if isinstance(parts, list) else []
+            from hermes_state import SessionDB
+            decode = getattr(SessionDB, "_decode_content", None)
+        except ImportError:
+            decode = None
+        if callable(decode):
+            content = decode(content)
+        else:
+            try:
+                content = json.loads(content[len(_CONTENT_JSON_PREFIX):])
+            except ValueError:
+                pass  # Legacy malformed marker rows remain literal text, as in the Agent.
+    if isinstance(content, list):
+        texts = [p.get("text") for p in content if isinstance(p, dict) and p.get("type") == "text"]
         return str(texts[0]) if texts and texts[0] else ""
-    return content.split(_IMAGE_HINTS, 1)[0]
+    return content.split(_IMAGE_HINTS, 1)[0] if isinstance(content, str) else ""
 
 
 def unit_children(unit: dict, owner: str, live: list[dict], finished: list[dict], now: float) -> list[dict]:
@@ -444,7 +585,7 @@ def background_list(home: Path, session_ids: list[str]) -> dict:
     live = _live_delegations()
     delegations = _ledger_rows(home, session_ids)
     live_children = _live_children()
-    finished = _ledger_children(home, session_ids) if delegations else []
+    finished = _ledger_children(home, session_ids, min(row["dispatched_at"] for row in delegations)) if delegations else []
     now = time.time()
     for row in delegations:
         row["live_status"] = live.get(row["delegation_id"], {}).get("status")
@@ -472,6 +613,8 @@ def recover(base_home: Path) -> int:
     recovery fails the call, so the server retries."""
     from .profiles import named_profile_homes
 
+    global _COMPLETION_BASE_HOME
+    _COMPLETION_BASE_HOME = base_home
     named = [home for _, home in named_profile_homes(base_home)]
     failed = []
     for home in [base_home, *named]:
@@ -489,6 +632,60 @@ def recover(base_home: Path) -> int:
     return 1 + len(named)
 
 
+def stop(home: Path, session_id: str, process_id: str) -> str:
+    """Request cancellation only for work this profile/session owns; let Agent completions settle its row."""
+    registry = _registry()
+    proc = registry.get(process_id) if registry is not None else None
+    if proc is not None:
+        if str(getattr(proc, "session_key", "") or "") != session_id:
+            return "not_found"
+        if proc.exited:
+            return "not_running"
+        # Keep the completion visible to the parent, just like other autonomous exits.
+        result = registry.kill_process(process_id, consume_output=False)
+        if result.get("status") == "error" or result.get("error"):
+            raise RpcError(str(result.get("error") or "Process could not be stopped"))
+        if result.get("status") == "killed":
+            return "stopped"
+        if result.get("completion_reason") == "already_exited" or result.get("status") == "already_exited":
+            return "not_running"
+        raise RpcError("Process stop was not confirmed")
+
+    row = next((r for r in _ledger_rows(home, [session_id]) if r["delegation_id"] == process_id), None)
+    if row is None:
+        return "not_found"
+    if row["state"] not in {"running", "stalling", "finalizing"}:
+        return "not_running"
+    from tools.delegate_tool_registry import interrupt_subagent, list_active_subagents
+
+    live = _live_delegations().get(process_id)
+    if not live or live.get("origin_ui_session_id") != session_id:
+        return "not_running"
+    indexes = live.get("task_indexes")
+    # Independent-completion units are <call-id>-<unit-number>; slot_key is the first unit, not the call.
+    split = re.fullmatch(r"(.+)-[1-9][0-9]*", process_id) if indexes is not None else None
+    if indexes is not None and split is None:
+        raise RpcError("Delegation unit identity could not be confirmed")
+    call_id = split[1] if split else process_id
+    # The dispatch parent is the Agent execution id; compression can rotate it away from the stable Web id.
+    owner_id = live.get("parent_session_id") or session_id
+
+    def belongs(child: dict) -> bool:
+        if child.get("delegation_id") != call_id or child.get("owner_agent_session_id") != owner_id:
+            return False
+        # Split units share a call id; Agent subagent ids encode their original task index.
+        index = re.fullmatch(r"sa-(\d+)-[a-zA-Z0-9]+", str(child.get("subagent_id") or ""))
+        return indexes is None or (index is not None and int(index[1]) in indexes)
+
+    children = [r for r in list_active_subagents() if belongs(r)]
+    if not children:
+        return "not_running"
+    interrupted = [interrupt_subagent(r["subagent_id"]) for r in children]
+    if not all(interrupted):
+        raise RpcError("Some subagents could not be interrupted; refresh and retry")
+    return "stopped"
+
+
 def register(registry_) -> None:
     @registry_.method("process.drain")
     def drain_(ctx: CallContext, params: dict) -> dict:
@@ -533,7 +730,15 @@ def register(registry_) -> None:
         if not consumer:
             raise InvalidParams("consumer is required")
         with scoped_home(profile_home_param(params)):
-            return {"claim_id": claim_delivery(evt, consumer)}
+            claim = claim_delivery(evt, consumer)
+            result = {"claim_id": claim}
+            if claim is None:
+                api = _delivery_api()
+                lookup = getattr(api, "get_durable_delegation", None)
+                row = lookup(str(evt.get("delegation_id") or "")) if callable(lookup) else None
+                if row and row.get("delivery_state") == "pending":
+                    result["retryable"] = True
+            return result
 
     @registry_.method("process.complete_delivery")
     def complete_(ctx: CallContext, params: dict) -> dict:
@@ -580,6 +785,14 @@ def register(registry_) -> None:
         with scoped_home(profile_home_param(params)) as home:
             return {"text": delegation_result(home, sid, did)}
 
+    @registry_.method("process.stop")
+    def stop_(ctx: CallContext, params: dict) -> dict:
+        sid, pid = params.get("session_id"), params.get("process_id")
+        if not isinstance(sid, str) or not sid.strip() or not isinstance(pid, str) or not pid.strip():
+            raise InvalidParams("session_id and process_id are required")
+        with scoped_home(profile_home_param(params)) as home:
+            return {"outcome": stop(home, sid, pid)}
+
     @registry_.method("process.list")
     def list_(ctx: CallContext, params: dict) -> dict:
         with scoped_home(profile_home_param(params)):

package/sidecar/talaria_sidecar/methods/profiles.py

@@ -151,20 +151,58 @@ def canonical_bot_chat(home: Path) -> dict | None:
             pass
 
 
+GATEWAY_RUNNING_MODES = ("own", "served")
+
+
+def gateway_mode(home: Path, name: str, is_default: bool) -> str:
+    """The profile's gateway under the host multiplexer (TAL-805): ``served`` by the live host gateway, ``own`` gateway
+    process, ``parked`` (``hermes -p X gateway stop`` left its marker and nothing runs; the marker records intent, so a
+    live ``--force`` gateway still counts, as in ``hermes -p X gateway status``), ``standalone`` (opted out of the host and
+    not running), else ``stopped``. The default profile is the host itself.
+    A helper the installed Agent lacks answers no, so an Agent without the multiplexer helpers reports ``own`` or
+    ``stopped``; a helper that fails answers no too, except the parked check, which fails closed to ``stopped``."""
+    from hermes_cli import profiles as agent
+
+    def probe(helper: str, arg) -> bool | None:
+        check = getattr(agent, helper, None)
+        if check is None:
+            return False
+        try:
+            return bool(check(arg))
+        except Exception:  # noqa: BLE001
+            log.debug("gateway mode probe %s failed for %s", helper, home, exc_info=True)
+            return None
+
+    def asks(helper: str, arg) -> bool:
+        return probe(helper, arg) is True
+
+    named = not is_default
+    parked = probe("profile_is_parked", home) if named else False
+    if parked is None:
+        return "stopped"
+    if named and asks("_served_by_running_multiplexer", name):
+        return "served"
+    if asks("_check_gateway_running", home):
+        return "own"
+    if parked:
+        return "parked"
+    if named and asks("profile_is_standalone", home):
+        return "standalone"
+    return "stopped"
+
+
 def _row(home: Path, name: str, is_default: bool) -> dict:
-    from hermes_cli.profiles import _check_gateway_running, _read_config_model
+    from hermes_cli.profiles import _read_config_model
 
     try:
         model, provider = _read_config_model(home)
     except Exception:  # noqa: BLE001
         model, provider = None, None
-    try:
-        gateway_running = bool(_check_gateway_running(home))
-    except Exception:  # noqa: BLE001
-        gateway_running = False
+    mode = gateway_mode(home, name, is_default)
     enabled, total = skills_stats(home)
     return {
-        "name": name, "path": str(home), "is_default": is_default, "gateway_running": gateway_running, "model": model, "provider": provider,
+        "name": name, "path": str(home), "is_default": is_default, "gateway_running": mode in GATEWAY_RUNNING_MODES, "gateway_mode": mode,
+        "model": model, "provider": provider,
         "has_env": (home / ".env").exists(), "visible": _visible(home), "skill_count": enabled, "enabled_skills": enabled, "total_skills": total,
         **bot_identity(home), "canonical_session": canonical_bot_chat(home),
     }
@@ -314,10 +352,11 @@ def register(registry) -> None:
         base_home, name = profile_home_param(params, "base_home"), str(params.get("name") or "").strip()
         validate_name(name)
         # Idle cached agents end their memory sessions while the profile's home still exists.
-        from .chat import release_profile_agents
+        from .chat import forget_profile_hooks, release_profile_agents
 
-        release_profile_agents(base_home / "profiles" / name)
-        delete_profile(base_home, name)
+        with release_profile_agents(base_home / "profiles" / name):
+            forget_profile_hooks(base_home / "profiles" / name)
+            delete_profile(base_home, name)
         return {"ok": True}
 
     @registry.method("profiles.runtime_env")