@maudecode/talaria-web 1.18.0 → 1.20.0

package/sidecar/talaria_sidecar/methods/worktree.py

@@ -1,10 +1,14 @@
-"""``worktree.*``: Hermes Agent git worktree creation (``hermes_cli.worktree_ops._setup_worktree``)."""
+"""``worktree.*``: Hermes Agent git worktree creation (``hermes_cli.worktree_ops._setup_worktree``) and lock retention."""
 
 from __future__ import annotations
 
+import contextlib
 import io
 import logging
+import os
+import re
 import subprocess
+import uuid
 from contextlib import redirect_stderr, redirect_stdout
 from pathlib import Path
 
@@ -15,26 +19,136 @@ from ..rpc import CallContext
 log = logging.getLogger("talaria_sidecar.worktree")
 
 
-def _setup_worktree_fn():
-    try:
-        from hermes_cli.worktree_ops import _setup_worktree
+# Repository config can name a program for a filter driver its .gitattributes selects; the checkout runs it. A
+# conditional include (onbranch:, gitdir:) can hold one back until the new worktree's checkout, so it counts too.
+_REPO_FILTER_KEYS = r"^(filter\..*\.(smudge|clean|process)|includeif\..*\.path)$"
+
+
+def _refuse_unneutralized_repo_filters(repo_root: str) -> None:
+    """Agents before v0.21.6 check out with the repository's filter programs live; refuse a repo that configures one.
 
-        return _setup_worktree
-    except Exception:  # noqa: BLE001 - older Agents keep the helper in cli.py
+    Only global and system config are the user's own; any other scope (local, worktree, an include from either) is
+    the repository's. Unreadable config fails closed, matching the Agent's own refusal from v0.21.6, and so does any
+    repository conditional include: this probe cannot evaluate it for the branch and gitdir the worktree will have.
+    """
+    try:
+        from hermes_cli._subprocess_compat import noninteractive_repo_git_env  # noqa: F401 - v0.21.6+ neutralizes them
+        return
+    except ImportError:
         pass
+    probe = subprocess.run(["git", "-C", repo_root, "config", "--includes", "--show-scope", "-z", "--get-regexp", _REPO_FILTER_KEYS],
+                           text=True, capture_output=True, timeout=5, check=False, stdin=subprocess.DEVNULL)
+    # -z --show-scope answers "<scope>\0<key>\n<value>\0" per match.
+    scopes = set(probe.stdout.split("\0")[::2]) - {""}
+    if probe.returncode not in (0, 1) or scopes - {"global", "system"}:
+        raise RpcError("This repository's git config can run filter programs; update Hermes Agent to v0.21.6 or later to create a worktree here",
+                       condition="worktree_unsafe_repo")
+
+
+# The Agent's own lock contract (``worktree_ops._worktree_lock_is_live``): its sweep keeps a tree whose ``hermes pid=`` owner runs.
+_HERMES_PID_LOCK = re.compile(r"hermes pid=(\d+)(?: db=.+)?", re.DOTALL)
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except (ProcessLookupError, OverflowError):  # gone, or a pid no process can have
+        return False
+    except OSError:
+        return True  # exists, owned by someone else
+    return True
+
+
+# Only these say the tree or its registration is gone; any other read failure leaves the tree unconfirmed (raised).
+_GONE = (FileNotFoundError, NotADirectoryError, IsADirectoryError)
+
+
+def _admin_dir(tree: Path) -> Path | None:
+    """The tree's ``<common-dir>/worktrees/<name>`` entry when git still registers it as a linked worktree."""
+    try:
+        link = (tree / ".git").read_text(encoding="utf-8-sig")
+    except _GONE:
+        return None
+    if not link.startswith("gitdir:"):
+        return None
+    admin = (tree / link[len("gitdir:"):].strip()).resolve()
+    try:
+        registered = Path((admin / "gitdir").read_text(encoding="utf-8-sig").strip()).resolve()
+    except _GONE:
+        return None
+    return admin if admin.parent.name == "worktrees" and registered == (tree / ".git").resolve() else None
+
+
+def _take_lock(admin: Path, expected: str) -> bool:
+    """Remove the tree's lock only while it still reads ``expected``: it is moved aside first, so a lock written since it
+    was read is never deleted; anything else is put back unless a newer lock already holds the tree."""
+    lock, aside = admin / "locked", admin / f"locked.talaria-{uuid.uuid4().hex}"
+    try:
+        os.rename(lock, aside)
+    except FileNotFoundError:
+        return True
     try:
-        import importlib.util
+        stale = aside.read_text(encoding="utf-8-sig", errors="replace").strip(" \t\n\v\f\r") == expected
+    except OSError:
+        stale = False
+    if not stale:
+        try:
+            os.link(aside, lock)
+        except FileExistsError:
+            pass  # a newer lock holds the tree
+        except OSError:
+            # No hard link here: move it back whole, never leave the tree unlocked by dropping it.
+            with contextlib.suppress(OSError):
+                os.rename(aside, lock)
+            return False
+    aside.unlink(missing_ok=True)
+    return stale
 
-        import hermes_cli
 
-        spec = importlib.util.spec_from_file_location("hermes_cli_worktree", str(Path(hermes_cli.__file__).resolve().parent.parent / "cli.py"))
-        if spec is None or spec.loader is None:
-            raise RuntimeError("cli.py not found")
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        return module._setup_worktree
-    except Exception as exc:  # noqa: BLE001
-        raise RpcError("Hermes Agent worktree helper is unavailable", condition="worktree_unavailable") from exc
+def retain(path: str) -> str:
+    """Lock a Talaria worktree for this process, so the Agent's sweep reads its lock live (TAL-1254).
+
+    ``missing`` when the tree is gone or git no longer registers it; ``foreign_lock`` when a lock that is not a ``hermes
+    pid=`` lock (a user's own) holds it, which stays untouched. A ``hermes pid=`` lock of an exited process is replaced;
+    one of a live other process (a reused pid, another instance) is left alone and raises, so the caller retries until
+    this sidecar holds the lock.
+    """
+    tree = Path(path)
+    admin = _admin_dir(tree)
+    if admin is None:
+        return "missing"
+    ours = f"hermes pid={os.getpid()}"
+    try:
+        reason = (admin / "locked").read_text(encoding="utf-8-sig", errors="replace").strip(" \t\n\v\f\r")
+    except FileNotFoundError:
+        reason = None
+    if reason == ours:
+        return "retained"
+    if reason is not None:
+        lock = _HERMES_PID_LOCK.fullmatch(reason)
+        if lock is None:
+            return "foreign_lock"
+        if int(lock.group(1)) != os.getpid() and _pid_alive(int(lock.group(1))):
+            raise OSError(f"locked by running pid {lock.group(1)}")
+        if not _take_lock(admin, reason):
+            raise OSError("the lock changed while it was being replaced")
+    # git refuses to lock a tree someone locked meanwhile, so that lock stays too.
+    locked = subprocess.run(["git", "-C", str(tree), "worktree", "lock", "--reason", ours, str(tree)],
+                            capture_output=True, timeout=10, check=False, stdin=subprocess.DEVNULL)
+    if locked.returncode != 0:
+        raise OSError(locked.stderr.decode(errors="replace").strip() or "git worktree lock failed")
+    return "retained"
+
+
+def retain_all(paths: list[str]) -> dict[str, str]:
+    """``retain`` per path; a tree this sidecar cannot lock now is left out (unknown), never reported missing."""
+    results = {}
+    for path in dict.fromkeys(paths):
+        try:
+            results[path] = retain(path)
+        except (OSError, subprocess.SubprocessError) as exc:
+            log.warning("Could not retain worktree %s: %s", path, exc)
+    return results
 
 
 def create(repo_root: str) -> dict:
@@ -44,17 +158,22 @@ def create(repo_root: str) -> dict:
     probe = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=str(root), text=True, capture_output=True, timeout=5, check=False)
     if probe.returncode != 0 or not probe.stdout.strip():
         raise RpcError("Workspace is not inside a git repository", condition="not_a_repo")
-    setup = _setup_worktree_fn()
+    repo = str(Path(probe.stdout.strip()).resolve())
+    _refuse_unneutralized_repo_filters(repo)
+    from hermes_cli.worktree_ops import _setup_worktree
+
     output = io.StringIO()
     with redirect_stdout(output), redirect_stderr(output):
-        info = setup(str(Path(probe.stdout.strip()).resolve()))
+        info = _setup_worktree(repo)
     emitted = output.getvalue().strip()
     if emitted:
         log.debug("Hermes Agent worktree helper output: %s", emitted)
     if not info or not info.get("path") or not info.get("branch"):
         raise RpcError("Hermes Agent failed to create a git worktree", condition="worktree_failed")
+    path = str(Path(info["path"]).expanduser().resolve())
+    retain_all([path])  # one owner for Talaria's lock text
     return {
-        "path": str(Path(info["path"]).expanduser().resolve()),
+        "path": path,
         "branch": str(info["branch"]),
         "repo_root": str(Path(info.get("repo_root") or probe.stdout.strip()).expanduser().resolve()),
         "base": str(info["base"]) if info.get("base") else None,
@@ -69,3 +188,10 @@ def register(registry) -> None:
             raise InvalidParams("repo_root is required")
         with scoped_home(profile_home_param(params)):
             return create(repo_root)
+
+    @registry.method("worktree.retain")
+    def retain_(ctx: CallContext, params: dict) -> dict:
+        paths = params.get("paths")
+        if not isinstance(paths, list) or not all(isinstance(p, str) and p for p in paths):
+            raise InvalidParams("paths must be a list of worktree paths")
+        return {"results": retain_all(paths)}

package/sidecar/talaria_sidecar/methods/write_approval.py

@@ -0,0 +1,181 @@
+"""``write_approval.*``: the Agent's staged skill and memory writes (TAL-819).
+
+With ``skills.write_approval`` or ``memory.write_approval`` on, the Agent stages writes under
+``<home>/pending/<kind>/<id>.json`` instead of committing them. These methods list them with
+the diff an approval would apply, and approve or reject them through the Agent's own
+``/skills`` and ``/memory`` review handlers.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import difflib
+import hashlib
+import re
+import tempfile
+import threading
+from pathlib import Path
+
+from ..errors import InvalidParams, RpcError
+from ..home import profile_home_param, scoped_home
+from ..rpc import CallContext
+
+KINDS = ("skills", "memory")
+# The Agent names records ``uuid4().hex[:8]`` and builds their path from the id; anything else could leave the pending dir.
+_ID = re.compile(r"[A-Za-z0-9_-]{1,64}")
+# One review at a time in this process, so two approvals of one record cannot both replay it.
+_LOCK = threading.Lock()
+
+
+def _agent():
+    try:
+        from hermes_cli import write_approval_commands
+        from tools import write_approval
+    except ImportError as exc:
+        raise RpcError("This Hermes Agent cannot review staged writes", condition="write_approval_unavailable") from exc
+    return write_approval, write_approval_commands
+
+
+def _kind(params: dict) -> str:
+    kind = str(params.get("kind") or "")
+    if kind not in KINDS:
+        raise InvalidParams("kind must be skills or memory")
+    return kind
+
+
+def _memory_diff(payload: dict) -> str:
+    """Replay the record on a scratch copy of its file through the Agent's approve path; the diff is what approval commits."""
+    from tools.memory_tool import ENTRY_DELIMITER, apply_memory_pending, load_on_disk_store
+
+    store = load_on_disk_store()
+    target = "user" if payload.get("target") == "user" else "memory"
+    real = store._path_for(target)
+    before = list(store._entries_for(target))
+    with tempfile.TemporaryDirectory(prefix="talaria-pending-") as scratch:
+        copy = Path(scratch) / real.name
+        if real.is_file():
+            copy.write_bytes(real.read_bytes())
+        # The store resolves every read, lock and write through ``self._path_for``.
+        store._path_for = lambda t: Path(scratch) / ("USER.md" if t == "user" else "MEMORY.md")
+        result = apply_memory_pending(payload, store)
+        after = list(store._entries_for(target))
+    if not result.get("success"):
+        raise _Unapprovable(f"(approve would fail: {result.get('error') or 'unknown error'})")
+    lines = lambda entries: ENTRY_DELIMITER.join(entries).splitlines(keepends=True)  # noqa: E731
+    diff = difflib.unified_diff(lines(before), lines(after), fromfile=f"a/{real.name}", tofile=f"b/{real.name}")
+    return "".join(diff) or "(no textual change)"
+
+
+class _Unapprovable(Exception):
+    """A write with no diff the user could review: its message replaces the diff and approval is refused."""
+
+
+def _skill_names(payload: dict) -> list[str]:
+    ops = payload.get("operations") or [] if payload.get("action") == "batch" else [payload]
+    return list(dict.fromkeys(str(op.get("name")) for op in ops if isinstance(op, dict) and op.get("name")))
+
+
+def _item(wa, kind: str, record: dict) -> dict:
+    payload = record.get("payload") if isinstance(record.get("payload"), dict) else {}
+    if kind == "memory":
+        target = "user" if payload.get("target") == "user" else "memory"
+    else:
+        target = ", ".join(_skill_names(payload))
+    diff, approvable = _diff(wa, kind, record)
+    return {
+        "id": str(record.get("id") or ""), "kind": kind, "target": target, "op": str(payload.get("action") or record.get("action") or ""),
+        "summary": str(record.get("summary") or ""), "origin": str(record.get("origin") or "foreground"),
+        "created_at": float(record.get("created_at") or 0), "diff": diff,
+        "approvable": approvable, "revision": _revision(diff) if approvable else "",
+    }
+
+
+def _diff(wa, kind: str, record: dict) -> tuple[str, bool]:
+    """The diff approving ``record`` applies and whether it may be approved; a diff that cannot be built is never approvable."""
+    payload = record.get("payload") if isinstance(record.get("payload"), dict) else {}
+    try:
+        return (_memory_diff(payload) if kind == "memory" else wa.skill_pending_diff(record)), True
+    except _Unapprovable as exc:
+        return str(exc), False
+    except Exception as exc:  # noqa: BLE001 - one unreadable record must not hide the rest
+        return f"(diff unavailable: {exc})", False
+
+
+def _apply_lock(kind: str, record: dict):
+    """Hold the Agent's own per-skill locks across the revision check and the apply (they are re-entrant in this thread, so
+    the Agent's apply takes them again). Memory needs none: the Agent applies a staged remove/patch only to the exact entry
+    it pinned at staging and an add only appends, so a concurrent write can never be dropped by an approval."""
+    if kind != "skills":
+        return contextlib.nullcontext()
+    from tools.skill_manager_tool import _skill_mutation_locks
+
+    return _skill_mutation_locks(_skill_names(record.get("payload") if isinstance(record.get("payload"), dict) else {}))
+
+
+def _revision(diff: str) -> str:
+    """What the reviewer saw: the diff changes whenever the record or the file it applies to does."""
+    return hashlib.sha256(diff.encode("utf-8")).hexdigest()
+
+
+def pending(kind: str) -> dict:
+    try:
+        wa, _ = _agent()
+    except RpcError:
+        return {"supported": False, "enabled": False, "items": []}
+    items = [_item(wa, kind, record) for record in wa.list_pending(kind) if _ID.fullmatch(str(record.get("id") or ""))]
+    return {"supported": True, "enabled": wa.write_approval_enabled(kind), "items": items}
+
+
+def review(kind: str, verb: str, writes: list[dict]) -> dict:
+    """``approve`` or ``reject`` each write through the Agent's handler; a record that is gone afterwards was applied or dropped.
+
+    An approval applies only while the write's diff still has the ``revision`` the reviewer saw; a changed file or record
+    is refused, so nothing lands that the user did not review.
+    """
+    wa, commands = _agent()
+    if not writes or any(not (isinstance(w, dict) and _ID.fullmatch(str(w.get("id") or "")) and isinstance(w.get("revision"), str)) for w in writes):
+        raise InvalidParams("writes must be pending write ids with the revision reviewed")
+    results = []
+    with _LOCK:
+        for write in writes:
+            pending_id = write["id"]
+            record = wa.get_pending(kind, pending_id)
+            if record is None:
+                results.append({"id": pending_id, "done": False, "message": f"No pending {kind} write with id '{pending_id}'."})
+                continue
+            if verb == "reject":
+                message = commands.handle_pending_subcommand(kind, [verb, pending_id]) or ""
+            else:
+                with _apply_lock(kind, record):
+                    diff, approvable = _diff(wa, kind, record)
+                    if not approvable:
+                        results.append({"id": pending_id, "done": False, "message": f"This write cannot be approved: {diff}"})
+                        continue
+                    if _revision(diff) != write["revision"]:
+                        results.append({"id": pending_id, "done": False, "message": "This write's diff changed after you reviewed it. Review the new diff, then approve again."})
+                        continue
+                    store = None
+                    if kind == "memory":
+                        from tools.memory_tool import load_on_disk_store
+
+                        store = load_on_disk_store()
+                    message = commands.handle_pending_subcommand(kind, [verb, pending_id], memory_store=store) or ""
+            results.append({"id": pending_id, "done": wa.get_pending(kind, pending_id) is None, "message": message})
+    return {"results": results}
+
+
+def register(registry) -> None:
+    @registry.method("write_approval.pending")
+    def pending_(ctx: CallContext, params: dict) -> dict:
+        kind = _kind(params)
+        with scoped_home(profile_home_param(params)):
+            return pending(kind)
+
+    for verb in ("approve", "reject"):
+        def handler(ctx: CallContext, params: dict, verb: str = verb) -> dict:
+            kind = _kind(params)
+            writes = list(params.get("writes") or [])
+            with scoped_home(profile_home_param(params)):
+                return review(kind, verb, writes)
+
+        registry.method(f"write_approval.{verb}")(handler)

package/sidecar/talaria_sidecar/rpc.py

@@ -13,11 +13,16 @@ from __future__ import annotations
 
 import json
 import logging
+import os
+import queue
+import time
 import sys
 import threading
 import traceback
 from typing import Any, Callable
 
+from .startup_timing import StartupTiming, outcome_for
+
 from .errors import (
     APPLICATION_ERROR,
     INTERNAL_ERROR,
@@ -39,11 +44,12 @@ DEFAULT_MAX_CALLS = 64
 class CallContext:
     """Per-request handle: stream emission and cooperative cancellation."""
 
-    def __init__(self, server: "RpcServer", request_id: Any, method: str):
+    def __init__(self, server: "RpcServer", request_id: Any, method: str, request: dict | None = None):
         self.server = server
         self.id = request_id
         self.method = method
         self.cancel_event = threading.Event()
+        self.timing = StartupTiming(request) if method == "chat.start" and request is not None else None
         self._seq = 0
         self._seq_lock = threading.Lock()
 
@@ -60,6 +66,8 @@ class CallContext:
         with self._seq_lock:
             self._seq += 1
             seq = self._seq
+        if self.timing:
+            self.timing.frame(event, data)
         self.server.notify("stream", {"id": self.id, "seq": seq, "event": event, "data": data if data is not None else {}})
 
 
@@ -121,6 +129,11 @@ class RpcServer:
     def request_shutdown(self, exit_code: int = 0) -> None:
         self.exit_code = exit_code
         self._closed.set()
+        with self._active_lock:
+            for ctx in self._active.values():
+                ctx.cancel_event.set()
+                if ctx.timing:
+                    ctx.timing.finish("unknown")
 
     # ── dispatch ──────────────────────────────────────────────────────────
     def _register(self, request: dict) -> CallContext | None:
@@ -129,7 +142,7 @@ class RpcServer:
         method = request.get("method")
         if request_id is None or not isinstance(method, str):
             return None
-        ctx = CallContext(self, request_id, method)
+        ctx = CallContext(self, request_id, method, request)
         with self._active_lock:
             self._active[request_id] = ctx
         return ctx
@@ -141,6 +154,8 @@ class RpcServer:
         if params is None:
             params = {}
         if not isinstance(method, str) or not isinstance(params, dict):
+            if ctx and ctx.timing:
+                ctx.timing.finish("refused")
             if request_id is not None:
                 with self._active_lock:
                     self._active.pop(request_id, None)
@@ -154,19 +169,28 @@ class RpcServer:
                 self._error(request_id, METHOD_NOT_FOUND, f"unknown method {method}")
             return
         if ctx is None:
-            ctx = CallContext(self, request_id, method)
+            ctx = CallContext(self, request_id, method, request)
             if request_id is not None:
                 with self._active_lock:
                     self._active[request_id] = ctx
+        outcome = "error"
+        if ctx.timing:
+            ctx.timing.end("rpc_wait")
+            ctx.timing.observe("rpc_start")
         try:
             result = handler(ctx, params)
+            status = result.get("status") if isinstance(result, dict) else None
+            failed = isinstance(result, dict) and bool(result.get("failed") or result.get("partial") or result.get("error"))
+            outcome = "cancelled" if ctx.cancelled or status == "cancelled" else "error" if status == "error" or failed else "ok"
             if request_id is not None:
                 self._result(request_id, result if result is not None else {})
         except RpcError as exc:
+            outcome = outcome_for(exc)
             if request_id is not None:
                 error = exc.to_json()
                 self._error(request_id, error["code"], error["message"], error.get("data"))
         except TypeError as exc:
+            outcome = "refused"
             # Wrong keyword arguments reaching a wrapped Agent function.
             log.debug("invalid params for %s", method, exc_info=True)
             if request_id is not None:
@@ -176,18 +200,46 @@ class RpcServer:
             if request_id is not None:
                 self._error(request_id, INTERNAL_ERROR, f"{type(exc).__name__}: {exc}")
         finally:
+            if ctx.timing:
+                ctx.timing.finish(outcome)
             if request_id is not None:
                 with self._active_lock:
                     self._active.pop(request_id, None)
 
     def serve_forever(self) -> int:
         """Read requests until stdin closes or shutdown is requested."""
-        while not self._closed.is_set():
+        incoming = queue.Queue(maxsize=1)
+        def read_input():
             try:
-                raw = self._in.readline()
+                try:
+                    fd = self._in.fileno()
+                except (AttributeError, OSError, ValueError):
+                    fd = None
+                if fd is None:
+                    while raw := self._in.readline():
+                        incoming.put(raw)
+                else:
+                    # An unbuffered OS read cannot hold sys.stdin's Python buffer lock at interpreter exit.
+                    pending = []
+                    while chunk := os.read(fd, 65536):
+                        lines = chunk.split(b"\n")
+                        pending.append(lines[0])
+                        for line in lines[1:]:
+                            incoming.put(b"".join(pending))
+                            pending = [line]
+                    if pending:
+                        incoming.put(b"".join(pending))
             except (OSError, ValueError):
-                break
-            if not raw:
+                pass
+            finally:
+                incoming.put(None)
+        threading.Thread(target=read_input, name="rpc-input", daemon=True).start()
+        while not self._closed.is_set():
+            try:
+                raw = incoming.get(timeout=.1)
+            except queue.Empty:
+                continue
+            if raw is None or self._closed.is_set():
                 break
             raw = raw.strip()
             if not raw:
@@ -205,10 +257,21 @@ class RpcServer:
                 self._dispatch(request)
                 continue
             if not self._slots.acquire(blocking=False):
+                if request.get("method") == "chat.start":
+                    StartupTiming(request).finish("refused")
                 self._error(request.get("id"), APPLICATION_ERROR, "sidecar is at its concurrent call limit", {"condition": "sidecar_busy"})
                 continue
             ctx = self._register(request)
             threading.Thread(target=self._dispatch_slot, args=(request, ctx), name=f"rpc-{request.get('method')}", daemon=True).start()
+        if self._closed.is_set():
+            # Let the terminal reply reach stdout before the process returns its final exit code.
+            deadline = time.monotonic() + 5
+            while time.monotonic() < deadline:
+                with self._active_lock:
+                    terminal_reply = any(ctx.method in {"runtime.shutdown", "runtime.handshake"} for ctx in self._active.values())
+                if not terminal_reply:
+                    break
+                threading.Event().wait(.01)
         return self.exit_code
 
     def _dispatch_slot(self, request: dict, ctx: CallContext | None) -> None: