@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/families/agent-eval-harness-py/files/tests/test_runner.py

@@ -0,0 +1,291 @@
+"""Runner tests.
+
+Regressions these protect against:
+
+- Scenario loader silently dropping a scenario file (regression: would emit
+  a scorecard with N-1 flows but no error).
+- Aggregate computation including ``unmeasured`` / ``gap`` in the denominator
+  (regression: drops aggregate any time a judge times out).
+- ``derive_status`` flipping for ``lower-better`` flows (regression: marked
+  a passing fast benchmark as fail because direction inversion was missed).
+- ``write_scorecard`` emitting non-deterministic key order (regression:
+  cross-language byte-equal compare would break).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from eval.judges.rubric import JudgeInput, JudgeResult
+from eval.runner import RunConfig, run_scenarios
+from eval.scorecard import (
+    Scorecard,
+    ScorecardFlow,
+    aggregate_score,
+    derive_status,
+    load_scorecard,
+    write_scorecard,
+)
+
+# ---------------------------------------------------------------------------
+# Scenario discovery + flow emission
+# ---------------------------------------------------------------------------
+
+
+def _write_scenario(scenarios_dir: Path, name: str, body: str) -> None:
+    path = scenarios_dir / f"{name}.py"
+    path.write_text(body, encoding="utf-8")
+
+
+_PASS_SCENARIO = """
+from eval.judges.rubric import JudgeInput, JudgeResult
+
+
+def _agent(prompt):
+    return "yes"
+
+
+def _judge(ji: JudgeInput) -> JudgeResult:
+    return JudgeResult(score=1.0, ok=True, rationale="exact")
+
+
+SCENARIO = {
+    "name": "always_pass",
+    "input": "anything",
+    "expected_output": "yes",
+    "target": 0.5,
+    "direction": "higher-better",
+    "productValueClaim": "smoke",
+    "agent_fn": _agent,
+    "judges": [_judge],
+}
+"""
+
+_FAIL_SCENARIO = """
+from eval.judges.rubric import JudgeInput, JudgeResult
+
+
+def _agent(prompt):
+    return "no"
+
+
+def _judge(ji: JudgeInput) -> JudgeResult:
+    return JudgeResult(score=0.1, ok=True, rationale="below target")
+
+
+SCENARIO = {
+    "name": "always_fail",
+    "input": "anything",
+    "expected_output": "yes",
+    "target": 0.85,
+    "direction": "higher-better",
+    "productValueClaim": "smoke",
+    "agent_fn": _agent,
+    "judges": [_judge],
+}
+"""
+
+_NO_JUDGE_SCENARIO = """
+def _agent(prompt):
+    return "x"
+
+
+SCENARIO = {
+    "name": "no_judge",
+    "input": "anything",
+    "target": 0.85,
+    "direction": "higher-better",
+    "productValueClaim": "gap-test",
+    "agent_fn": _agent,
+    "judges": [],
+}
+"""
+
+_LOWER_BETTER_SCENARIO = """
+from eval.judges.rubric import JudgeInput, JudgeResult
+
+
+def _agent(prompt):
+    return "x"
+
+
+def _judge(ji: JudgeInput) -> JudgeResult:
+    # Latency-style metric — lower is better. Score is the measured value (e.g. 0.2s).
+    return JudgeResult(score=0.2, ok=True)
+
+
+SCENARIO = {
+    "name": "fast_path",
+    "input": "anything",
+    "target": 0.5,
+    "direction": "lower-better",
+    "productValueClaim": "fast=good",
+    "agent_fn": _agent,
+    "judges": [_judge],
+}
+"""
+
+_BROKEN_JUDGE_SCENARIO = """
+from eval.judges.rubric import JudgeInput, JudgeResult
+
+
+def _agent(prompt):
+    return "x"
+
+
+def _judge(ji: JudgeInput) -> JudgeResult:
+    raise RuntimeError("judge boom")
+
+
+SCENARIO = {
+    "name": "judge_explodes",
+    "input": "anything",
+    "target": 0.85,
+    "direction": "higher-better",
+    "productValueClaim": "judge-error-test",
+    "agent_fn": _agent,
+    "judges": [_judge],
+}
+"""
+
+
+def test_runner_emits_one_flow_per_scenario(tmp_path: Path) -> None:
+    """Every scenario file produces exactly one flow — no silent drops."""
+    scen = tmp_path / "scenarios"
+    scen.mkdir()
+    _write_scenario(scen, "a", _PASS_SCENARIO)
+    _write_scenario(scen, "b", _FAIL_SCENARIO)
+    _write_scenario(scen, "c", _NO_JUDGE_SCENARIO)
+
+    cfg = RunConfig(scenarios_dir=scen, out_path=tmp_path / "out.json")
+    card = run_scenarios(cfg)
+
+    names = sorted(f.name for f in card.flows)
+    assert names == ["always_fail", "always_pass", "no_judge"], names
+    assert card.product == "agent-eval-harness-py"
+
+
+def test_status_pass_fail_gap_unmeasured(tmp_path: Path) -> None:
+    """Status assignment matches the contract for each scenario type."""
+    scen = tmp_path / "scenarios"
+    scen.mkdir()
+    _write_scenario(scen, "p", _PASS_SCENARIO)
+    _write_scenario(scen, "f", _FAIL_SCENARIO)
+    _write_scenario(scen, "n", _NO_JUDGE_SCENARIO)
+    _write_scenario(scen, "b", _BROKEN_JUDGE_SCENARIO)
+
+    cfg = RunConfig(scenarios_dir=scen, out_path=tmp_path / "out.json")
+    card = run_scenarios(cfg)
+
+    by_name = {f.name: f for f in card.flows}
+    assert by_name["always_pass"].status == "pass", by_name["always_pass"]
+    assert by_name["always_fail"].status == "fail", by_name["always_fail"]
+    assert by_name["no_judge"].status == "gap", by_name["no_judge"]
+    # Broken judge → all judges failed → unmeasured (NOT fail).
+    assert by_name["judge_explodes"].status == "unmeasured", by_name["judge_explodes"]
+    assert by_name["judge_explodes"].value is None
+    assert "judge raised" in (by_name["judge_explodes"].notes or "")
+
+
+def test_aggregate_excludes_unmeasured_and_gap(tmp_path: Path) -> None:
+    """Aggregate denominator is measurable flows only — gap/unmeasured don't move it."""
+    scen = tmp_path / "scenarios"
+    scen.mkdir()
+    _write_scenario(scen, "p", _PASS_SCENARIO)
+    _write_scenario(scen, "f", _FAIL_SCENARIO)
+    _write_scenario(scen, "n", _NO_JUDGE_SCENARIO)
+
+    cfg = RunConfig(scenarios_dir=scen, out_path=tmp_path / "out.json")
+    card = run_scenarios(cfg)
+    # 1 pass + 1 fail measurable; gap excluded → aggregate = 1/2 = 0.5
+    assert card.aggregate == 0.5, card.aggregate
+    assert "1/3 flows measured" in card.coverage or "2/3 flows measured" in card.coverage
+
+
+def test_lower_better_direction_inversion() -> None:
+    """``derive_status`` inverts comparison for lower-better flows."""
+    assert derive_status(0.2, 0.5, "lower-better") == "pass"
+    assert derive_status(0.7, 0.5, "lower-better") == "fail"
+    assert derive_status(0.7, 0.5, "higher-better") == "pass"
+    assert derive_status(0.2, 0.5, "higher-better") == "fail"
+    assert derive_status(None, 0.5, "higher-better") == "unmeasured"
+
+
+def test_write_scorecard_round_trip_and_byte_stable(tmp_path: Path) -> None:
+    """``write_scorecard`` produces a deterministic JSON we can re-load."""
+    flows = [
+        ScorecardFlow(
+            name="x",
+            value=0.91234567,
+            target=0.85,
+            status="pass",
+            productValueClaim="claim",
+            direction="higher-better",
+        ),
+        ScorecardFlow(
+            name="y",
+            value=None,
+            target=0.85,
+            status="gap",
+            productValueClaim="claim2",
+            direction="higher-better",
+        ),
+    ]
+    card = Scorecard.build(product="t", flows=flows)
+    out = write_scorecard(card, tmp_path / "sc.json")
+    raw = out.read_text(encoding="utf-8")
+    parsed = json.loads(raw)
+    # Rounding: 0.91234567 -> 0.9123 (4 decimals)
+    assert parsed["flows"][0]["value"] == 0.9123, parsed["flows"][0]
+    # ``value`` is required — gap flow keeps the key with null so round-trip works.
+    assert "value" in parsed["flows"][1] and parsed["flows"][1]["value"] is None
+    # ``notes`` is optional + None → omitted entirely (matches TS undefined-drop).
+    assert "notes" not in parsed["flows"][0]
+    # Round-trip
+    reloaded = load_scorecard(out)
+    assert reloaded.aggregate == card.aggregate
+    assert [f.name for f in reloaded.flows] == ["x", "y"]
+
+
+def test_aggregate_score_zero_when_no_measurable() -> None:
+    flows = [
+        ScorecardFlow(
+            name="g",
+            value=None,
+            target=0.85,
+            status="gap",
+            productValueClaim="x",
+            direction="higher-better",
+        ),
+        ScorecardFlow(
+            name="u",
+            value=None,
+            target=0.85,
+            status="unmeasured",
+            productValueClaim="x",
+            direction="higher-better",
+        ),
+    ]
+    assert aggregate_score(flows) == 0.0
+
+
+def test_runner_raises_when_no_scenarios(tmp_path: Path) -> None:
+    scen = tmp_path / "scenarios"
+    scen.mkdir()
+    cfg = RunConfig(scenarios_dir=scen, out_path=tmp_path / "out.json")
+    with pytest.raises(ValueError, match="no scenarios"):
+        run_scenarios(cfg)
+
+
+def test_runner_raises_on_missing_required_keys(tmp_path: Path) -> None:
+    scen = tmp_path / "scenarios"
+    scen.mkdir()
+    (scen / "bad.py").write_text(
+        "SCENARIO = {'name': 'x'}\n", encoding="utf-8"
+    )
+    cfg = RunConfig(scenarios_dir=scen, out_path=tmp_path / "out.json")
+    with pytest.raises(ValueError, match="missing required keys"):
+        run_scenarios(cfg)

package/registry/families/agent-eval-harness-py/files/tests/test_sandbox.py

@@ -0,0 +1,258 @@
+"""SandboxDriver lifecycle tests.
+
+The real ``tangle_sandbox`` SDK requires a live API key; we mock the SDK
+to keep these tests offline. The mock asserts the lifecycle contract
+(create → wait_for → files.write → task → delete) without making network
+calls. A live integration test belongs in CI with a real key — those
+should set ``EVAL_LIVE_SANDBOX=1`` and dispatch the runner directly.
+
+What this protects against:
+
+- Driver mutating its private state in a way that lets ``run()`` succeed
+  before ``open()`` (regression: silent NoneType during the run).
+- ``close()`` raising on a teardown error and corrupting the run summary
+  (regression: a flaky delete blew away the scorecard).
+- Bundle resolver requiring ``tangle_sandbox`` even in dry contexts (the
+  package should be importable without the SDK installed).
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+from typing import Any
+from unittest import mock
+
+import pytest
+
+
+# ---------------------------------------------------------------------------
+# Helpers — install fake `tangle_sandbox` + fake `starter_foundry.agent_bundle`
+# into ``sys.modules`` for the duration of a test.
+# ---------------------------------------------------------------------------
+
+
+class _FakeTaskResult:
+    def __init__(self, success: bool, response: str | None = None, error: str | None = None) -> None:
+        self.success = success
+        self.response = response
+        self.error = error
+
+
+class _FakeFiles:
+    def __init__(self) -> None:
+        self.writes: list[tuple[str, str]] = []
+
+    def write(self, path: str, content: str) -> None:
+        self.writes.append((path, content))
+
+
+class _FakeBox:
+    def __init__(self, task_response: str = "ok") -> None:
+        self.id = "sbx_fake_001"
+        self.files = _FakeFiles()
+        self.task_response = task_response
+        self.deleted = False
+        self.wait_for_calls: list[str] = []
+
+    def wait_for(self, status: str) -> None:
+        self.wait_for_calls.append(status)
+
+    def task(self, prompt: str) -> _FakeTaskResult:
+        return _FakeTaskResult(True, response=self.task_response)
+
+    def delete(self) -> None:
+        self.deleted = True
+
+
+class _FakeClient:
+    def __init__(self, *, api_key: str, base_url: str) -> None:
+        self.api_key = api_key
+        self.base_url = base_url
+        self.created_with: Any = None
+        self.boxes: list[_FakeBox] = []
+
+    def create(self, options: Any) -> _FakeBox:
+        self.created_with = options
+        box = _FakeBox()
+        self.boxes.append(box)
+        return box
+
+
+def _install_fake_sdk(monkeypatch: pytest.MonkeyPatch) -> _FakeClient | None:
+    """Inject fake ``tangle_sandbox`` module. Returns the holder for assertions."""
+    holder: dict[str, _FakeClient] = {}
+
+    def _make_client(*, api_key: str, base_url: str) -> _FakeClient:
+        c = _FakeClient(api_key=api_key, base_url=base_url)
+        holder["client"] = c
+        return c
+
+    fake_mod = types.ModuleType("tangle_sandbox")
+    fake_mod.Sandbox = _make_client  # type: ignore[attr-defined]
+
+    class _BackendConfig:
+        def __init__(self, *, profile: dict[str, Any] | None = None) -> None:
+            self.profile = profile
+
+    class _CreateSandboxOptions:
+        def __init__(
+            self,
+            *,
+            name: str | None = None,
+            image: str | None = None,
+            backend: Any = None,
+        ) -> None:
+            self.name = name
+            self.image = image
+            self.backend = backend
+
+    fake_mod.BackendConfig = _BackendConfig  # type: ignore[attr-defined]
+    fake_mod.CreateSandboxOptions = _CreateSandboxOptions  # type: ignore[attr-defined]
+    monkeypatch.setitem(sys.modules, "tangle_sandbox", fake_mod)
+
+    # Lazy accessor so callers can grab the client after open() runs.
+    return holder  # type: ignore[return-value]
+
+
+def _install_fake_bundle_loader(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Inject a fake ``starter_foundry.agent_bundle`` into sys.modules."""
+
+    sf_mod = types.ModuleType("starter_foundry")
+    bundle_mod = types.ModuleType("starter_foundry.agent_bundle")
+
+    def _load(_path: Path | str) -> dict[str, Any]:
+        return {"name": "fake", "version": "0.0.1"}
+
+    def _to_profile(_bundle: Any, _bundle_dir: Path | str) -> dict[str, Any]:
+        return {"name": "fake", "version": "0.0.1"}
+
+    class _WSFile:
+        def __init__(self, target: str, content: str) -> None:
+            self.targetPath = target
+            self.content = content
+
+    def _to_workspace_files(_bundle: Any, _bundle_dir: Path | str) -> list[_WSFile]:
+        return [_WSFile("/home/agent/AGENTS.md", "# fake bundle\n")]
+
+    def _resolve_workspace_root(_bundle: Any) -> str:
+        return "/home/agent"
+
+    bundle_mod.load_agent_bundle = _load  # type: ignore[attr-defined]
+    bundle_mod.to_agent_profile = _to_profile  # type: ignore[attr-defined]
+    bundle_mod.to_workspace_files = _to_workspace_files  # type: ignore[attr-defined]
+    bundle_mod.resolve_workspace_root = _resolve_workspace_root  # type: ignore[attr-defined]
+    sf_mod.agent_bundle = bundle_mod  # type: ignore[attr-defined]
+    monkeypatch.setitem(sys.modules, "starter_foundry", sf_mod)
+    monkeypatch.setitem(sys.modules, "starter_foundry.agent_bundle", bundle_mod)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_sandbox_module_imports_without_tangle_sandbox_installed() -> None:
+    """The harness should be importable even when the SDK isn't on the path."""
+    # We don't try to uninstall — instead, just confirm the import is lazy
+    # (no top-level `import tangle_sandbox`).
+    import importlib
+
+    mod = importlib.import_module("eval.sandbox_runner")
+    assert hasattr(mod, "SandboxDriver")
+    # The module must not have eagerly bound a Sandbox symbol at import time.
+    assert "Sandbox" not in dir(mod), "tangle_sandbox import must be lazy"
+
+
+def test_sandbox_driver_full_lifecycle(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    holder = _install_fake_sdk(monkeypatch)
+    _install_fake_bundle_loader(monkeypatch)
+    monkeypatch.setenv("TANGLE_SANDBOX_API_KEY", "sk-test")
+    monkeypatch.setenv("TANGLE_SANDBOX_BASE_URL", "https://test.example.com")
+
+    from eval.sandbox_runner import SandboxDriver
+
+    bundle_dir = tmp_path / "bundle"
+    bundle_dir.mkdir()
+    (bundle_dir / "agent.json").write_text("{}", encoding="utf-8")  # not actually parsed by fake
+
+    with SandboxDriver(bundle_dir=bundle_dir, name="t-eval", image="node:20") as drv:
+        out = drv.run({"input": "ping"})
+        assert out == "ok"
+
+    client = holder["client"]
+    assert client.api_key == "sk-test"
+    assert client.base_url == "https://test.example.com"
+    assert client.created_with.name == "t-eval"
+    assert client.created_with.image == "node:20"
+    box = client.boxes[0]
+    assert box.wait_for_calls == ["running"]
+    assert box.files.writes == [("/home/agent/AGENTS.md", "# fake bundle\n")]
+    assert box.deleted is True
+
+
+def test_sandbox_driver_raises_when_api_key_missing(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _install_fake_sdk(monkeypatch)
+    _install_fake_bundle_loader(monkeypatch)
+    monkeypatch.delenv("TANGLE_SANDBOX_API_KEY", raising=False)
+    monkeypatch.setenv("TANGLE_SANDBOX_BASE_URL", "https://test.example.com")
+
+    from eval.sandbox_runner import SandboxDriver
+
+    bundle_dir = tmp_path / "b"
+    bundle_dir.mkdir()
+    (bundle_dir / "agent.json").write_text("{}", encoding="utf-8")
+    with pytest.raises(RuntimeError, match="TANGLE_SANDBOX_API_KEY is unset"):
+        SandboxDriver(bundle_dir=bundle_dir).open()
+
+
+def test_sandbox_driver_run_before_open_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+    from eval.sandbox_runner import SandboxDriver
+
+    drv = SandboxDriver(profile={"name": "x", "version": "0"})
+    with pytest.raises(RuntimeError, match="before open"):
+        drv.run({"input": "x"})
+
+
+def test_sandbox_driver_close_swallows_teardown_errors(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Teardown failures must never propagate — they'd shadow the real run failure."""
+    from eval.sandbox_runner import SandboxDriver
+
+    drv = SandboxDriver(profile={"name": "x", "version": "0"})
+    bad_box = mock.MagicMock()
+    bad_box.delete.side_effect = RuntimeError("delete boom")
+    drv._box = bad_box  # private; intentional test seam
+    drv._client = mock.MagicMock()
+    # Should not raise
+    drv.close()
+    assert drv._box is None
+    assert drv._client is None
+
+
+def test_sandbox_driver_task_failure_raises_runtime(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    holder = _install_fake_sdk(monkeypatch)
+    _install_fake_bundle_loader(monkeypatch)
+    monkeypatch.setenv("TANGLE_SANDBOX_API_KEY", "sk-test")
+    monkeypatch.setenv("TANGLE_SANDBOX_BASE_URL", "https://test.example.com")
+
+    from eval.sandbox_runner import SandboxDriver
+
+    bundle_dir = tmp_path / "bundle"
+    bundle_dir.mkdir()
+    (bundle_dir / "agent.json").write_text("{}", encoding="utf-8")
+    with SandboxDriver(bundle_dir=bundle_dir) as drv:
+        # patch the box's task method to fail
+        client = holder["client"]
+        box = client.boxes[0]
+        box.task = lambda prompt: _FakeTaskResult(False, error="task explosion")  # type: ignore[assignment]
+        with pytest.raises(RuntimeError, match="sandbox task failed"):
+            drv.run({"input": "x"})