@maudecode/talaria-web 1.18.0 → 1.19.0

package/sidecar/scripts/suite_runner.py

@@ -0,0 +1,302 @@
+"""Bounded, module-preserving pytest subprocesses and verified JUnit aggregation.
+
+Also loaded as a pytest plugin to record collection and actual test starts. Only
+the existing pytest dependency is used; orchestration itself is Python stdlib.
+"""
+
+from collections import Counter, defaultdict
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import traceback
+import xml.etree.ElementTree as ET
+
+WORKERS = "TALARIA_SIDECAR_WORKERS"
+
+
+def worker_count(value=None):
+    if value is None:
+        cpu = getattr(os, "process_cpu_count", os.cpu_count)() or 1
+        if hasattr(os, "sched_getaffinity"):
+            cpu = min(cpu, len(os.sched_getaffinity(0)))
+        return min(4, cpu)
+    if not value.isdecimal() or int(value) < 1:
+        raise ValueError(f"{WORKERS} must be a positive integer")
+    return int(value)
+
+
+def partition(nodeids, count):
+    modules = defaultdict(list)
+    for nodeid in nodeids:
+        modules[nodeid.split("::", 1)[0]].append(nodeid)
+    groups = [[] for _ in range(min(count, len(modules)))]
+    for _, items in sorted(modules.items(), key=lambda pair: (-len(pair[1]), pair[0])):
+        min(groups, key=len).extend(items)
+    return groups
+
+
+# Pytest plugin hooks. The collection and execution journal is independent of
+# JUnit, which can contain an extra testcase for a teardown error.
+_collected, _started = [], []
+
+
+def pytest_collection_modifyitems(items):
+    _collected.extend(item.nodeid for item in items)
+    for item in items:
+        item.user_properties.append(("talaria_nodeid", item.nodeid))
+
+
+def pytest_runtest_logstart(nodeid):
+    _started.append(nodeid)
+
+
+def pytest_sessionfinish():
+    Path(os.environ["TALARIA_PYTEST_MANIFEST"]).write_text(json.dumps({"collected": _collected, "started": _started}))
+
+
+def report_argument(args):
+    """Only an unfiltered full suite is parallelized; all other pytest arguments pass through serially."""
+    remaining, report = [], None
+    iterator = iter(args)
+    for arg in iterator:
+        if arg in ("--junitxml", "--junit-xml"):
+            report = next(iterator)  # pytest would reject a missing value too.
+        elif arg.startswith(("--junitxml=", "--junit-xml=")):
+            report = arg.split("=", 1)[1]
+        else:
+            remaining.append(arg)
+    return remaining, Path(report).resolve() if report else None
+
+
+def reap_orphans(processes, *, terminate=False):
+    if sys.platform != "linux":
+        return
+    # Only adopted children belong to this reaper. Popen must retain the exit
+    # statuses of registered collection/workers, including already-dead workers.
+    managed = {proc.pid for proc in processes if proc.returncode is None}
+    children = Path(f"/proc/self/task/{os.getpid()}/children")
+    while True:
+        try:
+            direct = None if terminate else set(map(int, children.read_text().split()))
+        except FileNotFoundError:  # CONFIG_PROC_CHILDREN is optional.
+            direct = None
+        # During final cleanup, children are exiting and the proc list can omit
+        # entries. Use the same fallback snapshot for that discovery too.
+        if direct is None:
+            direct = {pid for pid, ppid, _, _ in process_snapshot() if ppid == os.getpid()}
+        adopted, reaped = direct - managed, False
+        for pid in adopted:
+            try:
+                # The fallback snapshot includes its already-reaped ps child.
+                # Confirm wait ownership before signaling any remaining child.
+                waited, _ = os.waitpid(pid, os.WNOHANG)
+                if terminate and not waited:
+                    try:
+                        os.kill(pid, signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+                    waited, _ = os.waitpid(pid, 0)
+                reaped |= bool(waited)
+            except ChildProcessError:
+                pass
+        if not terminate or not reaped:
+            return
+
+
+def wait_for(proc, processes):
+    while proc.poll() is None:
+        reap_orphans(processes)
+        time.sleep(.02)
+    reap_orphans(processes)
+    return proc.returncode
+
+
+def process_snapshot():
+    # Readline can set native COLUMNS behind os.environ's back. Ownership paths
+    # must survive that terminal width in both pytest fixtures and final cleanup.
+    listing = subprocess.check_output(["ps", "-ww", "-eo", "pid=,ppid=,pgid=,args="], text=True)
+    return [(int(row[0]), int(row[1]), int(row[2]), row[3])
+            for row in (line.split(None, 3) for line in listing.splitlines()) if len(row) == 4]
+
+
+def cleanup(processes, state):
+    """Reap workers and kill their process groups even when pytest exited first."""
+    for proc in processes:
+        try:
+            os.killpg(proc.pid, signal.SIGKILL)
+        except PermissionError:
+            # If group signaling is refused, kill the leader directly; the
+            # ownership scan below still covers its group and detached children.
+            proc.kill()
+        except ProcessLookupError:
+            pass
+        proc.wait()
+    # Tests can detach a process from pytest's group. Reuse the fixture's
+    # disposable-path ownership rule to clean those up on crashes/cancellation.
+    rows = process_snapshot()
+    parents = {pid: ppid for pid, ppid, _, _ in rows}
+    ancestors, pid = set(), os.getpid()
+    while pid > 1 and pid not in ancestors:
+        ancestors.add(pid)
+        pid = parents.get(pid, 0)
+    owned = {os.getpid()}
+    while True:
+        expanded = owned | {pid for pid, parent in parents.items() if parent in owned}
+        if expanded == owned:
+            break
+        owned = expanded
+    groups = {proc.pid for proc in processes}
+    for pid, _, pgid, args in rows:
+        if (str(state) in args or pid in owned or pgid in groups) and pid not in ancestors:
+            try:
+                os.kill(pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+    reap_orphans(processes, terminate=True)
+
+
+def merge_reports(jobs, destination):
+    merged = ET.Element("testsuites")
+    durations, modules = [], defaultdict(float)
+    for job, expected in jobs:
+        manifest = json.loads((job / "manifest.json").read_text())
+        if Counter(manifest["collected"]) != Counter(expected) or Counter(manifest["started"]) != Counter(expected):
+            raise ValueError(f"incomplete or duplicate test coverage in {job}")
+        report = ET.parse(job / "junit.xml").getroot()
+        suites = [report] if report.tag == "testsuite" else list(report.findall("testsuite"))
+        if not suites:
+            raise ValueError(f"missing testsuites in {job}")
+        cases = [case for suite in suites for case in suite.findall("testcase")]
+        covered = [case.find("properties/property[@name='talaria_nodeid']") for case in cases]
+        if any(prop is None for prop in covered) or set(prop.get("value") for prop in covered) != set(expected):
+            raise ValueError(f"JUnit coverage differs from collection in {job}")
+        by_node = defaultdict(list)
+        for case, prop in zip(cases, covered):
+            by_node[prop.get("value")].append(case)
+        for records in by_node.values():
+            # Pytest emits two records only for a call failure plus a teardown
+            # error. Repeated successful records would inflate receipt counts.
+            if len(records) > 1 and (len(records) != 2 or
+                    sum(case.find("failure") is not None for case in records) != 1 or
+                    sum(case.find("error") is not None for case in records) != 1):
+                raise ValueError(f"duplicate JUnit testcase in {job}")
+        for suite in suites:
+            cases = suite.findall("testcase")
+            counts = {"tests": len(cases), **{key: sum(len(case.findall(tag)) for case in cases)
+                      for key, tag in (("failures", "failure"), ("errors", "error"), ("skipped", "skipped"))}}
+            if any(int(suite.get(key, "-1")) != count for key, count in counts.items()):
+                raise ValueError(f"JUnit counts differ from testcases in {job}")
+            merged.append(suite)
+            for case in cases:
+                duration = float(case.get("time", "0"))
+                nodeid = case.find("properties/property[@name='talaria_nodeid']").get("value")
+                durations.append((duration, nodeid))
+                modules[nodeid.split("::", 1)[0]] += duration
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    temporary = destination.with_suffix(".tmp")
+    ET.ElementTree(merged).write(temporary, encoding="utf-8", xml_declaration=True)
+    temporary.replace(destination)
+    for label, entries in (("Slow tests", durations), ("Slow modules", [(seconds, name) for name, seconds in modules.items()])):
+        print(label + ":", flush=True)
+        for seconds, name in sorted(entries, reverse=True)[:10]:
+            print(f"  {seconds:.3f}s {name}", flush=True)
+    return any(case.find(tag) is not None for case in merged.iter("testcase") for tag in ("failure", "error"))
+
+
+def run(args):
+    count = worker_count(os.environ.get(WORKERS))
+    remaining, report = report_argument(args)
+    selected = 1 if remaining else count
+    print(f"Sidecar pytest: workers={selected}, host budget={os.environ.get('TALARIA_SIDECAR_TOTAL_WORKERS', count)}", flush=True)
+    if report:
+        report.unlink(missing_ok=True)  # A stale report must never prove this run passed.
+    diagnostics = Path(str(report) + ".workers") if report else Path(__file__).resolve().parents[2] / ".codex-tmp/sidecar-pytest"
+    diagnostics.mkdir(parents=True, exist_ok=True)
+    # A unique invocation directory also prevents reusing evidence on retries.
+    output = Path(tempfile.mkdtemp(prefix="run-", dir=diagnostics))
+    processes, streams = [], []
+    with tempfile.TemporaryDirectory(prefix="talaria-pytest-state-") as directory:
+        state = Path(directory)
+
+        def launch(name, arguments):
+            job, home = output / name, state / name
+            job.mkdir()
+            for child in ("home", "tmp"):
+                (home / child).mkdir(parents=True)
+            env = {**os.environ, "HOME": str(home / "home"), "TMPDIR": str(home / "tmp"),
+                   "PYTHONPATH": str(Path(__file__).parent) + os.pathsep + os.environ.get("PYTHONPATH", ""),
+                   "TALARIA_PYTEST_MANIFEST": str(job / "manifest.json")}
+            log = (job / "output.log").open("w")
+            streams.append(log)
+            proc = subprocess.Popen([sys.executable, "-m", "pytest", "-p", "suite_runner", "-q",
+                                     f"--basetemp={home / 'pytest'}", *arguments], env=env,
+                                    stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
+            processes.append(proc)
+            return job, proc
+
+        try:
+            if remaining:
+                job, proc = launch("serial", args)
+                return wait_for(proc, processes)
+            job, collection = launch("collection", ["--collect-only"])
+            if wait_for(collection, processes):
+                raise ValueError("pytest collection failed")
+            nodeids = json.loads((job / "manifest.json").read_text())["collected"]
+            if not nodeids or len(nodeids) != len(set(nodeids)):
+                raise ValueError("empty or duplicate pytest collection")
+            groups = partition(nodeids, count)
+            print(f"Collected {len(nodeids)} cases in {len(groups)} module groups; diagnostics: {output}", flush=True)
+            jobs = []
+            for index, group in enumerate(groups):
+                name = f"worker-{index + 1}"
+                job, proc = launch(name, [f"--junitxml={output / name / 'junit.xml'}",
+                                         *dict.fromkeys(nodeid.split("::", 1)[0] for nodeid in group)])
+                jobs.append((job, group, proc))
+            failed = False
+            for job, _, proc in jobs:
+                code = wait_for(proc, processes)
+                print(f"{job.name}: exit={code}", flush=True)
+                failed |= code != 0
+            report_failed = merge_reports([(job, group) for job, group, _ in jobs], report or output / "junit.xml")
+            return int(failed or report_failed)
+        finally:
+            for sig in (signal.SIGINT, signal.SIGTERM):
+                signal.signal(sig, signal.SIG_IGN)
+            try:
+                cleanup(processes, state)
+            finally:
+                # Release checkouts may be disposable. Keep logs in the run
+                # output too, including cancellation and partial startup.
+                for stream in streams:
+                    stream.close()
+                    path = Path(stream.name)
+                    print(f"=== {path.parent.name}/output.log ===\n{path.read_text()}", flush=True)
+                print(f"Worker diagnostics retained: {output}", flush=True)
+
+
+def main():
+    if sys.platform == "linux":
+        import ctypes
+        if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0):  # PR_SET_CHILD_SUBREAPER
+            raise OSError("could not enable pytest child reaping")
+
+    def interrupt(signum, frame):
+        raise InterruptedError(f"sidecar suite interrupted by signal {signum}")
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        signal.signal(sig, interrupt)
+    try:
+        return run(sys.argv[1:])
+    except (Exception, KeyboardInterrupt) as error:
+        print(f"Sidecar suite failed: {error}", file=sys.stderr, flush=True)
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/sidecar/scripts/test.sh

@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 # Run the sidecar pytest suite. pytest runs on any Python >= 3.11 (a private
-# venv under sidecar/.venv); the sidecar itself is spawned on the pinned
-# Agent's venv interpreter discovered by tests/conftest.py.
+# venv under sidecar/.venv); the sidecar itself is spawned on the Agent venv in
+# HERMES_WEBUI_AGENT_DIR, else on the active pinned release: the script re-runs
+# itself under scripts/check-agent-compatibility.py, which provisions that
+# release once into a shared cache and holds it for the whole run.
 set -euo pipefail
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 runner="${HERMES_WEBUI_TEST_PYTHON:-python3}"
+if [[ -z "${HERMES_WEBUI_AGENT_DIR:-}" ]]; then
+  exec "$runner" "$here/../../scripts/check-agent-compatibility.py" --skip-docker --skip-sidecar-tests \
+    --command "$here/scripts/test.sh" "$@"
+fi
 venv="$here/.venv"
 if [[ ! -x "$venv/bin/python" ]]; then
   "$runner" -m venv "$venv"
@@ -16,7 +22,9 @@ trap 'rm -rf -- "$state"' EXIT
 cd "$here"
 # GitHub's relocated Linux Python needs LD_LIBRARY_PATH; nothing else from the caller's environment leaks in.
 env -i PATH="$PATH" HOME="$state" TMPDIR="$state" ${LD_LIBRARY_PATH:+LD_LIBRARY_PATH="$LD_LIBRARY_PATH"} \
-  HERMES_WEBUI_AGENT_DIR="${HERMES_WEBUI_AGENT_DIR:-$HOME/.hermes/hermes-agent}" \
+  HERMES_WEBUI_AGENT_DIR="$HERMES_WEBUI_AGENT_DIR" \
   HERMES_WEBUI_PYTHON="${HERMES_WEBUI_PYTHON:-}" \
+  ${TALARIA_SIDECAR_WORKERS+TALARIA_SIDECAR_WORKERS="$TALARIA_SIDECAR_WORKERS"} \
+  ${TALARIA_SIDECAR_TOTAL_WORKERS:+TALARIA_SIDECAR_TOTAL_WORKERS="$TALARIA_SIDECAR_TOTAL_WORKERS"} \
   PYTHONPATH="$here" \
-  "$venv/bin/python" -m pytest -q "$@"
+  "$venv/bin/python" "$here/scripts/suite_runner.py" "$@"

package/sidecar/talaria_sidecar/__init__.py

@@ -7,4 +7,9 @@ stdio (docs/architecture/sidecar-rpc.md). Standard library only.
 
 from __future__ import annotations
 
-SIDECAR_RPC_VERSION = 17
+import os
+
+# Also protect direct Python entry points before any method imports the Agent.
+os.environ["HERMES_DISABLE_LAZY_INSTALLS"] = "1"
+
+SIDECAR_RPC_VERSION = 44

package/sidecar/talaria_sidecar/__main__.py

@@ -7,6 +7,7 @@ import os
 import sys
 from pathlib import Path
 
+from .home import _PROCESS_HOME
 from .methods import build_methods
 from .rpc import DEFAULT_MAX_CALLS, RpcServer
 from .runtime import AgentRuntime
@@ -29,11 +30,10 @@ def _claim_stdout():
 def main() -> int:
     rpc_out = _claim_stdout()
     logging.basicConfig(stream=sys.stderr, level=os.environ.get("TALARIA_SIDECAR_LOG_LEVEL", "INFO"), format="[sidecar] %(levelname)s %(name)s: %(message)s")
-    hermes_home = Path(os.environ.get("HERMES_HOME") or Path.home() / ".hermes").expanduser()
     # Every turn here has a human behind the Web UI: flagged actions must ask (approval cards), never auto-approve.
     os.environ.setdefault("HERMES_EXEC_ASK", "1")
     agent_dir = os.environ.get("TALARIA_SIDECAR_AGENT_DIR") or os.environ.get("HERMES_WEBUI_AGENT_DIR")
-    runtime = AgentRuntime(hermes_home, Path(agent_dir) if agent_dir else None)
+    runtime = AgentRuntime(_PROCESS_HOME, Path(agent_dir) if agent_dir else None)
     if runtime.agent_dir is not None and str(runtime.agent_dir) not in sys.path:
         # Append, never prepend: the Agent venv's site-packages must win over
         # any same-named module in the checkout (api/config.py has the history).
@@ -44,7 +44,9 @@ def main() -> int:
     finally:
         # The server closes stdin to stop the sidecar: end every cached agent's memory session on the way out.
         from .methods.chat import drain_agents
+        from .agent_mutation import close_all
 
+        close_all()
         drain_agents()