@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/families/agent-eval-harness-py/files/src/eval/cli.py

@@ -0,0 +1,281 @@
+"""argparse CLI: ``python -m eval [run|compare]``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+from .regression import (
+    DEFAULT_ALPHA,
+    DEFAULT_D_THRESHOLD,
+    DEFAULT_RESAMPLES,
+    DEFAULT_SEED,
+    GateReport,
+    regression_gate,
+)
+from .runner import RunConfig, run_scenarios
+from .sandbox_runner import SandboxDriver
+from .scorecard import load_scorecard, write_scorecard
+
+log = logging.getLogger("eval.cli")
+
+
+def _build_run_parser(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("run", help="run scenarios against an agent and emit a scorecard")
+    p.add_argument(
+        "--scenarios",
+        type=Path,
+        default=Path("scenarios"),
+        help="directory containing scenario *.py modules (default: scenarios/)",
+    )
+    p.add_argument(
+        "--out",
+        type=Path,
+        default=Path("scorecard.json"),
+        help="scorecard output path (default: scorecard.json)",
+    )
+    p.add_argument(
+        "--driver",
+        choices=["local", "http", "sandbox"],
+        default="local",
+        help="agent-under-test driver (default: local)",
+    )
+    p.add_argument("--agent-url", default=None, help="agent endpoint URL (driver=http)")
+    p.add_argument(
+        "--bundle",
+        type=Path,
+        default=None,
+        help="agent bundle directory (driver=sandbox)",
+    )
+    p.add_argument(
+        "--sandbox-image",
+        default="node:20",
+        help="sandbox image (driver=sandbox, default node:20)",
+    )
+    p.add_argument(
+        "--sandbox-name",
+        default="eval-harness",
+        help="sandbox name prefix (driver=sandbox)",
+    )
+    p.add_argument(
+        "--product",
+        default="agent-eval-harness-py",
+        help="scorecard.product field (default: agent-eval-harness-py)",
+    )
+
+
+def _build_compare_parser(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("compare", help="compare two scorecards via Welch + Cohen + bootstrap CI")
+    p.add_argument("baseline", type=Path, help="baseline scorecard JSON")
+    p.add_argument("head", type=Path, help="head scorecard JSON")
+    p.add_argument(
+        "--alpha",
+        type=float,
+        default=DEFAULT_ALPHA,
+        help=f"Welch p-value threshold (default {DEFAULT_ALPHA})",
+    )
+    p.add_argument(
+        "--d-threshold",
+        type=float,
+        default=DEFAULT_D_THRESHOLD,
+        help=f"Cohen d magnitude threshold (default {DEFAULT_D_THRESHOLD})",
+    )
+    p.add_argument(
+        "--resamples",
+        type=int,
+        default=DEFAULT_RESAMPLES,
+        help=f"bootstrap resamples (default {DEFAULT_RESAMPLES})",
+    )
+    p.add_argument(
+        "--seed",
+        type=int,
+        default=DEFAULT_SEED,
+        help=f"bootstrap RNG seed (default {DEFAULT_SEED:#x})",
+    )
+    p.add_argument(
+        "--samples",
+        type=Path,
+        default=None,
+        help=(
+            "optional JSON file mapping flow name -> {baseline:[...], head:[...]} "
+            "of repeated samples. When absent, compare uses the single value from "
+            "each scorecard (degenerate gate — emits a warning)."
+        ),
+    )
+    p.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="write the gate report JSON to this path (in addition to stdout)",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="eval",
+        description="Python eval harness — run scenarios + gate regressions.",
+    )
+    parser.add_argument("--log-level", default="INFO", help="logging level (default INFO)")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    _build_run_parser(sub)
+    _build_compare_parser(sub)
+    return parser
+
+
+# ----------------------------------------------------------------------
+# `run`
+# ----------------------------------------------------------------------
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    cfg = RunConfig(
+        scenarios_dir=args.scenarios,
+        out_path=args.out,
+        driver=args.driver,
+        agent_url=args.agent_url,
+        bundle_dir=args.bundle,
+        sandbox_image=args.sandbox_image,
+        sandbox_name_prefix=args.sandbox_name,
+        product=args.product,
+    )
+
+    if cfg.driver == "sandbox":
+        if cfg.bundle_dir is None:
+            sys.stderr.write("[blocked] --driver sandbox requires --bundle <dir>\n")
+            return 2
+        with SandboxDriver(
+            bundle_dir=cfg.bundle_dir,
+            image=cfg.sandbox_image,
+            name=cfg.sandbox_name_prefix,
+        ) as drv:
+            scorecard = run_scenarios(cfg, sandbox_runner=drv)
+    else:
+        scorecard = run_scenarios(cfg)
+
+    out_path = write_scorecard(scorecard, cfg.out_path)
+    sys.stdout.write(
+        f"[eval.run] product={scorecard.product} "
+        f"aggregate={scorecard.aggregate:.4f} "
+        f"coverage={scorecard.coverage} "
+        f"out={out_path}\n"
+    )
+    # Surface every flow status compactly
+    for f in scorecard.flows:
+        v = f"{f.value:.4f}" if f.value is not None else "—"
+        sys.stdout.write(
+            f"  [{f.status:>10}] {f.name:<40} value={v}  target={f.target:.4f}  ({f.direction})\n"
+        )
+    # Non-zero exit when any flow regressed (callers can gate CI on this).
+    failed = [f for f in scorecard.flows if f.status == "fail"]
+    return 1 if failed else 0
+
+
+# ----------------------------------------------------------------------
+# `compare`
+# ----------------------------------------------------------------------
+
+
+def _load_samples(path: Path | None) -> dict[str, dict[str, list[float]]] | None:
+    if path is None:
+        return None
+    with path.open("r", encoding="utf-8") as fh:
+        data = json.load(fh)
+    if not isinstance(data, dict):
+        raise ValueError(f"--samples file {path} must be a JSON object")
+    out: dict[str, dict[str, list[float]]] = {}
+    for k, v in data.items():
+        if not isinstance(v, dict) or "baseline" not in v or "head" not in v:
+            raise ValueError(
+                f"--samples[{k!r}] must be {{baseline:[...], head:[...]}}"
+            )
+        out[k] = {
+            "baseline": [float(x) for x in v["baseline"]],
+            "head": [float(x) for x in v["head"]],
+        }
+    return out
+
+
+def _cmd_compare(args: argparse.Namespace) -> int:
+    baseline = load_scorecard(args.baseline)
+    head = load_scorecard(args.head)
+    samples = _load_samples(args.samples)
+
+    head_by_name = {f.name: f for f in head.flows}
+    report = GateReport()
+
+    for bf in baseline.flows:
+        hf = head_by_name.get(bf.name)
+        if hf is None:
+            sys.stdout.write(f"  [missing-head] {bf.name}: present in baseline, absent in head\n")
+            continue
+        if bf.value is None or hf.value is None:
+            sys.stdout.write(
+                f"  [unmeasured] {bf.name}: skipping gate (baseline={bf.value} head={hf.value})\n"
+            )
+            continue
+        if samples and bf.name in samples:
+            base_arr = samples[bf.name]["baseline"]
+            head_arr = samples[bf.name]["head"]
+        else:
+            sys.stderr.write(
+                f"  [warn] {bf.name}: --samples not provided; gate degrades to single-value compare\n"
+            )
+            base_arr = [bf.value] * 8
+            head_arr = [hf.value] * 8
+        v = regression_gate(
+            flow=bf.name,
+            baseline=base_arr,
+            head=head_arr,
+            direction=bf.direction,
+            alpha=args.alpha,
+            d_threshold=args.d_threshold,
+            resamples=args.resamples,
+            seed=args.seed,
+        )
+        report.verdicts.append(v)
+        tag = "REGRESS" if v.regressed else "ok"
+        sys.stdout.write(f"  [{tag:>7}] {v.reason}\n")
+
+    payload = report.to_dict()
+    if args.out:
+        args.out.parent.mkdir(parents=True, exist_ok=True)
+        with args.out.open("w", encoding="utf-8") as fh:
+            json.dump(payload, fh, indent=2, ensure_ascii=False)
+            fh.write("\n")
+        sys.stdout.write(f"[eval.compare] wrote {args.out}\n")
+
+    if report.any_regressed:
+        sys.stdout.write(
+            f"[eval.compare] FAIL: {len(report.regressed_flows)} regressed: "
+            f"{', '.join(report.regressed_flows)}\n"
+        )
+        return 1
+    sys.stdout.write("[eval.compare] PASS: no flow regressed\n")
+    return 0
+
+
+# ----------------------------------------------------------------------
+# entrypoint
+# ----------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    logging.basicConfig(
+        level=getattr(logging, args.log_level.upper(), logging.INFO),
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+    if args.cmd == "run":
+        return _cmd_run(args)
+    if args.cmd == "compare":
+        return _cmd_compare(args)
+    parser.error(f"unknown command {args.cmd!r}")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/registry/families/agent-eval-harness-py/files/src/eval/judges/__init__.py

@@ -0,0 +1,27 @@
+"""LLM-as-judge primitives.
+
+A *judge* is anything callable matching::
+
+    Judge = Callable[[JudgeInput], JudgeResult | Awaitable[JudgeResult]]
+
+where ``JudgeInput`` carries ``input``, ``expected_output``, ``actual_output``
+and ``JudgeResult`` carries a per-dimension score (0..1) plus rationale.
+
+The default :class:`RubricJudge` POSTs to router.tangle.tools (OpenAI-shaped
+chat completions) with a structured rubric and parses the JSON response.
+
+Custom judges live in this package — see ``example_judge.py`` for the
+contract.
+"""
+
+from __future__ import annotations
+
+from .rubric import JudgeInput, JudgeResult, RubricDimension, RubricJudge, RubricResult
+
+__all__ = [
+    "JudgeInput",
+    "JudgeResult",
+    "RubricJudge",
+    "RubricResult",
+    "RubricDimension",
+]

package/registry/families/agent-eval-harness-py/files/src/eval/judges/example_judge.py

@@ -0,0 +1,52 @@
+"""Example judge — a 3-dimension rubric for code-generation tasks.
+
+Drop this in any scenario via::
+
+    from eval.judges.example_judge import code_quality_judge
+
+    SCENARIO = {
+        ...,
+        "judges": [code_quality_judge],
+    }
+
+The dimensions are illustrative — replace with whatever your domain needs.
+"""
+
+from __future__ import annotations
+
+from .rubric import RubricDimension, RubricJudge
+
+code_quality_judge = RubricJudge(
+    name="code-quality",
+    dimensions=[
+        RubricDimension(
+            name="correctness",
+            description=(
+                "Does the output produce the requested behaviour? Score 1 if "
+                "the actual_output, when run as described, would yield the "
+                "expected_output. Score 0 if it would error or produce a "
+                "wrong answer. Partial credit only for cases where logic is "
+                "correct but a specific edge case is missed."
+            ),
+            weight=2.0,
+        ),
+        RubricDimension(
+            name="completeness",
+            description=(
+                "Does the output cover the entire task, or does it stop "
+                "early / hand-wave a hard subproblem? Score 1 for a complete "
+                "deliverable, 0 for stub-only."
+            ),
+            weight=1.0,
+        ),
+        RubricDimension(
+            name="idiomatic",
+            description=(
+                "Is the output written in the idiomatic style of the target "
+                "language/framework? Score 1 for production-quality, 0 for "
+                "obvious anti-patterns or anti-idioms."
+            ),
+            weight=1.0,
+        ),
+    ],
+)

package/registry/families/agent-eval-harness-py/files/src/eval/judges/rubric.py

@@ -0,0 +1,299 @@
+"""LLM-as-judge over a multi-dimension rubric, via router.tangle.tools.
+
+Wire shape: OpenAI-compatible ``/v1/chat/completions``. The harness sends a
+single user message containing the input, expected_output, actual_output,
+and the rubric (as JSON), and asks the model to respond with a single JSON
+object scoring each dimension 0..1 plus rationale. We parse strict JSON and
+surface a structured error when parsing fails — never coerce.
+
+Failure modes are explicit:
+
+- HTTP non-2xx → ``RubricResult(ok=False, ...)`` with the body in ``error``.
+- JSON parse error → same.
+- Missing dimension in the response → same (the run records ``unmeasured``,
+  not a fabricated score).
+- ``TANGLE_API_KEY`` unset → raise ``RuntimeError``; the runner converts
+  to a top-level ``[blocked]``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field
+
+DEFAULT_BASE_URL = "https://router.tangle.tools/v1"
+DEFAULT_MODEL = "anthropic/claude-sonnet-4.5"
+DEFAULT_TIMEOUT_SEC = 60.0
+
+
+class JudgeInput(BaseModel):
+    """What the judge sees."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    scenario: str
+    input: str
+    expected_output: str | None = None
+    actual_output: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class RubricDimension(BaseModel):
+    """One scoring axis."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    description: str
+    weight: float = 1.0
+
+
+class JudgeResult(BaseModel):
+    """Generic judge result interface — RubricResult is the default impl."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    score: float  # 0..1, weighted mean across dimensions
+    ok: bool
+    rationale: str | None = None
+    error: str | None = None
+
+
+class RubricResult(JudgeResult):
+    """Multi-dim judge result — per-dimension scores plus weighted mean."""
+
+    dimensions: dict[str, float] = Field(default_factory=dict)
+    raw_response: str | None = None
+
+
+@dataclass
+class RubricJudge:
+    """LLM-as-judge over a fixed rubric.
+
+    Instantiate once per scenario suite (rubrics rarely vary per-scenario).
+    Calling the judge returns a :class:`RubricResult`.
+    """
+
+    name: str
+    dimensions: list[RubricDimension]
+    model: str = DEFAULT_MODEL
+    base_url: str = DEFAULT_BASE_URL
+    api_key_env: str = "TANGLE_API_KEY"
+    timeout_sec: float = DEFAULT_TIMEOUT_SEC
+    system_prompt: str = (
+        "You are a strict eval judge. Given an input, the expected output, "
+        "and the actual output, score each rubric dimension on a 0..1 scale "
+        "(0=fails dimension, 0.5=partial, 1=fully meets). Respond with a "
+        "single JSON object on one line: "
+        '{"dimensions":{"<name>":<score>,...},"rationale":"<one sentence>"}. '
+        "Do not include any text outside the JSON. Do not score dimensions "
+        "you weren't given."
+    )
+
+    def __post_init__(self) -> None:
+        if not self.dimensions:
+            raise ValueError("RubricJudge requires at least one dimension")
+        weights = [d.weight for d in self.dimensions]
+        if any(w <= 0 for w in weights):
+            raise ValueError("RubricJudge dimension weights must be positive")
+
+    # ------------------------------------------------------------------
+    # API key resolution
+    # ------------------------------------------------------------------
+
+    def _resolve_api_key(self) -> str:
+        key = os.environ.get(self.api_key_env)
+        if not key:
+            raise RuntimeError(
+                f"GAP: env var {self.api_key_env} is unset; RubricJudge cannot reach "
+                f"{self.base_url}. Set it or run with --no-judge."
+            )
+        return key
+
+    # ------------------------------------------------------------------
+    # Prompt construction
+    # ------------------------------------------------------------------
+
+    def _build_user_message(self, ji: JudgeInput) -> str:
+        rubric = [
+            {"name": d.name, "description": d.description, "weight": d.weight}
+            for d in self.dimensions
+        ]
+        payload = {
+            "scenario": ji.scenario,
+            "input": ji.input,
+            "expected_output": ji.expected_output,
+            "actual_output": ji.actual_output,
+            "rubric": rubric,
+        }
+        return json.dumps(payload, ensure_ascii=False)
+
+    # ------------------------------------------------------------------
+    # Response parsing
+    # ------------------------------------------------------------------
+
+    def _parse_response(self, raw: str) -> RubricResult:
+        # The model is instructed to emit a single JSON object on one line.
+        # Be tolerant of markdown code fences (some models add them despite
+        # instructions); strip them and retry.
+        candidate = raw.strip()
+        if candidate.startswith("```"):
+            # Remove leading fence (```json\n or ```\n) and trailing ```
+            candidate = candidate.strip("`")
+            # After stripping backticks both sides, optional language tag
+            # remains as the first token until newline.
+            if "\n" in candidate:
+                first, rest = candidate.split("\n", 1)
+                if first.strip().lower() in {"json", ""}:
+                    candidate = rest
+        try:
+            obj = json.loads(candidate)
+        except json.JSONDecodeError as err:
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error=f"judge response not JSON: {err}",
+                raw_response=raw,
+            )
+
+        dims = obj.get("dimensions")
+        if not isinstance(dims, dict):
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error="judge response missing 'dimensions' object",
+                raw_response=raw,
+            )
+
+        # Every declared dimension must be present and a float in [0, 1].
+        scored: dict[str, float] = {}
+        for d in self.dimensions:
+            if d.name not in dims:
+                return RubricResult(
+                    score=0.0,
+                    ok=False,
+                    error=f"judge response missing dimension {d.name!r}",
+                    raw_response=raw,
+                )
+            try:
+                v = float(dims[d.name])
+            except (TypeError, ValueError):
+                return RubricResult(
+                    score=0.0,
+                    ok=False,
+                    error=f"dimension {d.name!r} not numeric: {dims[d.name]!r}",
+                    raw_response=raw,
+                )
+            if not 0.0 <= v <= 1.0:
+                return RubricResult(
+                    score=0.0,
+                    ok=False,
+                    error=f"dimension {d.name!r} out of [0,1]: {v}",
+                    raw_response=raw,
+                )
+            scored[d.name] = v
+
+        total_w = sum(d.weight for d in self.dimensions)
+        weighted = sum(scored[d.name] * d.weight for d in self.dimensions) / total_w
+
+        return RubricResult(
+            score=round(weighted, 4),
+            ok=True,
+            rationale=obj.get("rationale"),
+            dimensions=scored,
+            raw_response=raw,
+        )
+
+    # ------------------------------------------------------------------
+    # Public call surfaces
+    # ------------------------------------------------------------------
+
+    def __call__(self, ji: JudgeInput) -> RubricResult:
+        return self.score(ji)
+
+    def score(self, ji: JudgeInput) -> RubricResult:
+        api_key = self._resolve_api_key()
+        body = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": self._build_user_message(ji)},
+            ],
+            "temperature": 0.0,
+            "max_tokens": 512,
+        }
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+        url = f"{self.base_url.rstrip('/')}/chat/completions"
+        try:
+            with httpx.Client(timeout=self.timeout_sec) as client:
+                resp = client.post(url, json=body, headers=headers)
+        except httpx.HTTPError as err:
+            return RubricResult(
+                score=0.0, ok=False, error=f"judge transport error: {err}"
+            )
+        if resp.status_code >= 400:
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error=f"judge HTTP {resp.status_code}: {resp.text[:500]}",
+            )
+        try:
+            data = resp.json()
+            content = data["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, ValueError) as err:
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error=f"judge response missing choices[0].message.content: {err}",
+                raw_response=resp.text,
+            )
+        return self._parse_response(content)
+
+    async def score_async(self, ji: JudgeInput) -> RubricResult:
+        api_key = self._resolve_api_key()
+        body = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": self._build_user_message(ji)},
+            ],
+            "temperature": 0.0,
+            "max_tokens": 512,
+        }
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+        url = f"{self.base_url.rstrip('/')}/chat/completions"
+        try:
+            async with httpx.AsyncClient(timeout=self.timeout_sec) as client:
+                resp = await client.post(url, json=body, headers=headers)
+        except httpx.HTTPError as err:
+            return RubricResult(
+                score=0.0, ok=False, error=f"judge transport error: {err}"
+            )
+        if resp.status_code >= 400:
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error=f"judge HTTP {resp.status_code}: {resp.text[:500]}",
+            )
+        try:
+            data = resp.json()
+            content = data["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, ValueError) as err:
+            return RubricResult(
+                score=0.0,
+                ok=False,
+                error=f"judge response missing choices[0].message.content: {err}",
+                raw_response=resp.text,
+            )
+        return self._parse_response(content)