@tangle-network/agent-bench 0.1.0 → 0.3.5

package/scripts/dabstep_judge.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""DABStep judge bridge.
+
+Reads {"prediction": str, "golds": list} from stdin and delegates scoring to
+the official DABStep grade.py module. This script owns no grading semantics.
+"""
+
+import argparse
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+
+def load_grade(grade_file: Path):
+    spec = importlib.util.spec_from_file_location("dabstep_grade", grade_file)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"could not import DABStep grade file: {grade_file}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.grade
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Score one DABStep answer")
+    parser.add_argument("--grade-file", required=True)
+    args = parser.parse_args()
+
+    try:
+        payload = json.loads(sys.stdin.read())
+        prediction = payload["prediction"]
+        golds = payload["golds"]
+        correct = bool(load_grade(Path(args.grade_file))(prediction, golds))
+        print(json.dumps({"correct": correct, "score": 1.0 if correct else 0.0}))
+        return 0
+    except Exception as exc:
+        print(json.dumps({"error": str(exc)}))
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/enterpriseops_gym_judge.py

@@ -0,0 +1,281 @@
+# EnterpriseOps-Gym judge driver.
+#
+# Scores a worker's tool-call transcript against a task's verifiers, deterministically,
+# by REPLAYING the transcript into a FRESHLY-SEEDED gym database and then reading the
+# verifier SQL back. The full deployable-checker loop (matches the official benchmark/
+# executor.py protocol):
+#
+#   1. SEED   per gym: create an isolated database (its own database_id) from the task's
+#             seed_database_file SQL snapshot via POST /api/seed-database. Every later call
+#             targets that database via the `x-database-id` header, so concurrent judges on
+#             one container never collide.
+#   2. REPLAY each transcript call as an MCP JSON-RPC `tools/call` to /mcp (NOT a bare
+#             {tool,arguments} POST, which the server accepts as a no-op 204 and never
+#             executes). A tool-call HTTP error is the AGENT's mistake (wrong args / policy
+#             violation) — counted + surfaced, never fatal; the verifiers score the result.
+#             A transport failure (server down) IS fatal (fail loud, never a fake score).
+#   3. VERIFY each database_state verifier's SQL via /api/sql-runner (parsing the server's
+#             {"data":[{...}]} shape), compared to expected_value under comparison_type.
+#   4. CLEAN  delete each seeded database (best-effort).
+#
+# Seed snapshots are resolved against EOPS_GYM_DBS_DIR (the unzipped gym_dbs.zip). A task
+# whose server has no seed_database_file, or an unset EOPS_GYM_DBS_DIR, fails loud — an
+# unseeded run would score every solve against the same default state (no signal).
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import uuid
+
+
+def fail(msg: str) -> None:
+    print(json.dumps({"error": msg}))
+    sys.exit(1)
+
+
+def request(url: str, payload: dict, headers: dict, method: str = "POST", timeout: float = 60.0) -> str:
+    body = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(url, data=body, method=method)
+    req.add_header("Content-Type", "application/json")
+    req.add_header("Accept", "application/json, text/event-stream")
+    for k, v in (headers or {}).items():
+        req.add_header(k, str(v))
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return resp.read().decode("utf-8")
+
+
+def parse_body(raw: str) -> object:
+    """Parse a JSON body OR an SSE `data: {...}` stream (the /mcp endpoint streams SSE)."""
+    raw = raw.strip()
+    if not raw:
+        return {}
+    if raw.startswith("event:") or raw.startswith("data:") or "\ndata:" in raw:
+        matches = re.findall(r"data:\s*(\{.*\})", raw)
+        if matches:
+            return json.loads(matches[-1])
+    return json.loads(raw)
+
+
+def auth_headers(server: dict) -> dict:
+    """Context auth headers (e.g. x-itsm-user-token) plus the per-run x-database-id."""
+    headers = dict(server.get("context") or {})
+    db_id = server.get("_database_id")
+    if db_id:
+        headers["x-database-id"] = db_id
+    return headers
+
+
+def server_for_gym(servers: list, gym_name: str) -> dict:
+    by_name = {s.get("mcp_server_name"): s for s in servers}
+    if gym_name in by_name:
+        return by_name[gym_name]
+    if len(servers) == 1:
+        return servers[0]
+    fail(f"verifier gym_name {gym_name!r} not in gym_servers_config ({sorted(by_name)})")
+
+
+def seed_databases(servers: list, gym_dbs_dir: str) -> None:
+    for s in servers:
+        seed_file = s.get("seed_database_file")
+        if not seed_file:
+            fail(
+                f"server {s.get('mcp_server_name')!r} has no seed_database_file — cannot seed a "
+                "deterministic state (an unseeded run scores every solve against the same default DB)"
+            )
+        if not gym_dbs_dir:
+            fail(
+                "EOPS_GYM_DBS_DIR is unset — cannot resolve seed_database_file. Unzip gym_dbs.zip "
+                "(from ServiceNow/EnterpriseOps-Gym) and set EOPS_GYM_DBS_DIR to its directory."
+            )
+        path = seed_file if os.path.isabs(seed_file) else os.path.join(gym_dbs_dir, seed_file)
+        try:
+            sql = open(path, encoding="utf-8").read()
+        except Exception as e:  # noqa: BLE001
+            fail(f"reading seed file {path} failed: {e}")
+        db_id = "gate_" + uuid.uuid4().hex[:12]
+        url = s["mcp_server_url"].rstrip("/") + "/api/seed-database"
+        payload = {
+            "database_id": db_id,
+            "name": f"gate_{db_id}",
+            "description": f"gate seed from {os.path.basename(seed_file)}",
+            "sql_content": sql,
+        }
+        try:
+            request(url, payload, {}, timeout=max(300.0, len(sql) / 1024))
+        except Exception as e:  # noqa: BLE001
+            fail(f"seed-database POST to {url} failed: {e}")
+        s["_database_id"] = db_id
+
+
+def delete_databases(servers: list) -> None:
+    for s in servers:
+        db_id = s.get("_database_id")
+        if not db_id:
+            continue
+        url = s["mcp_server_url"].rstrip("/") + "/api/delete-database"
+        try:
+            request(url, {"database_id": db_id}, {}, method="DELETE", timeout=30)
+        except Exception:  # noqa: BLE001
+            pass  # best-effort cleanup; a leaked db never corrupts a future run (unique id)
+
+
+def replay_transcript(servers: list, transcript: list) -> int:
+    """Replay each call as an MCP tools/call. Returns the count of agent-side call errors
+    (surfaced, never fatal). A transport failure fails loud."""
+    errors = 0
+    for i, call in enumerate(transcript):
+        tool = call.get("tool")
+        if not isinstance(tool, str) or not tool:
+            errors += 1
+            continue
+        gym_name = call.get("gym_name")
+        server = server_for_gym(servers, gym_name) if gym_name else servers[0]
+        url = server["mcp_server_url"].rstrip("/") + "/mcp"
+        payload = {
+            "jsonrpc": "2.0",
+            "id": i + 1,
+            "method": "tools/call",
+            "params": {"name": tool, "arguments": call.get("arguments") or {}},
+        }
+        try:
+            raw = request(url, payload, auth_headers(server), timeout=60)
+            result = parse_body(raw)
+            # A JSON-RPC error or an isError tool result is the agent's mistake, not a judge
+            # failure — count it so a wholesale replay failure is visible, but keep going.
+            if isinstance(result, dict) and (result.get("error") or (result.get("result") or {}).get("isError")):
+                errors += 1
+        except urllib.error.HTTPError:
+            errors += 1  # the agent issued a bad call (wrong args / policy) — verifiers score it
+        except urllib.error.URLError as e:
+            fail(f"gym server unreachable at {url} replaying tool {tool!r}: {e}")
+        except Exception as e:  # noqa: BLE001
+            fail(f"tool call {tool!r} against {url} failed unexpectedly: {e}")
+    return errors
+
+
+def run_sql(server: dict, query: str) -> object:
+    url = server["mcp_server_url"].rstrip("/") + "/api/sql-runner"
+    payload = {"query": query, "database_id": server.get("_database_id")}
+    try:
+        out = parse_body(request(url, payload, auth_headers(server), timeout=60))
+    except urllib.error.URLError as e:
+        fail(f"gym server unreachable at {url}: {e}")
+    except Exception as e:  # noqa: BLE001
+        fail(f"sql-runner POST to {url} failed: {e}")
+    if isinstance(out, dict):
+        if out.get("error"):
+            fail(f"sql-runner error for query {query!r}: {out['error']}")
+        # The gym server returns {"data":[{col:val}], ...}; older/other shapes use result/rows.
+        for key in ("data", "rows"):
+            rows = out.get(key)
+            if rows:
+                first = rows[0]
+                if isinstance(first, dict) and first:
+                    return next(iter(first.values()))
+                if isinstance(first, list) and first:
+                    return first[0]
+                return first
+        if "result" in out:
+            return out["result"]
+    return out
+
+
+def compare(actual: object, expected: object, comparison_type: str) -> bool:
+    if comparison_type == "equals":
+        try:
+            return float(actual) == float(expected)
+        except (TypeError, ValueError):
+            return str(actual) == str(expected)
+    if comparison_type == "greater_than":
+        return float(actual) > float(expected)
+    if comparison_type == "less_than":
+        return float(actual) < float(expected)
+    if comparison_type == "contains":
+        return str(expected) in str(actual)
+    fail(f"unsupported comparison_type {comparison_type!r}")
+
+
+def cmd_judge(args) -> None:
+    raw_transcript = sys.stdin.read()
+    try:
+        task = json.load(open(args.task_json, encoding="utf-8"))
+    except Exception as e:  # noqa: BLE001
+        fail(f"reading task json {args.task_json} failed: {e}")
+
+    servers = task.get("gym_servers_config")
+    if isinstance(servers, str):
+        servers = json.loads(servers)
+    if not isinstance(servers, list) or not servers:
+        fail("task has no gym_servers_config list")
+
+    verifiers = task.get("verifiers")
+    if isinstance(verifiers, str):
+        verifiers = json.loads(verifiers)
+    if not isinstance(verifiers, list) or not verifiers:
+        fail("task has no verifiers list")
+
+    transcript: list = []
+    if raw_transcript.strip():
+        try:
+            parsed = json.loads(raw_transcript)
+        except Exception as e:  # noqa: BLE001
+            fail(f"transcript is not valid JSON: {e}")
+        transcript = parsed.get("calls", []) if isinstance(parsed, dict) else parsed
+        if not isinstance(transcript, list):
+            fail('transcript must be a JSON list of tool calls (or {"calls":[...]})')
+
+    gym_dbs_dir = os.environ.get("EOPS_GYM_DBS_DIR", "")
+    seed_databases(servers, gym_dbs_dir)
+    try:
+        replay_errors = replay_transcript(servers, transcript)
+
+        results = []
+        passes = 0
+        for v in verifiers:
+            vtype = v.get("verifier_type")
+            if vtype != "database_state":
+                fail(f"verifier {v.get('name')!r} has non-deterministic verifier_type {vtype!r}")
+            cfg = v.get("validation_config") or {}
+            query = cfg.get("query")
+            if not isinstance(query, str) or not query.strip():
+                fail(f"verifier {v.get('name')!r} has no SQL query")
+            server = server_for_gym(servers, v.get("gym_name"))
+            actual = run_sql(server, query)
+            ok = compare(actual, cfg.get("expected_value"), cfg.get("comparison_type", "equals"))
+            if ok:
+                passes += 1
+            results.append({"name": v.get("name"), "passed": bool(ok)})
+
+        total = len(verifiers)
+        print(
+            json.dumps(
+                {
+                    "success": passes == total,
+                    "passes": passes,
+                    "total": total,
+                    "replay_calls": len(transcript),
+                    "replay_errors": replay_errors,
+                    "verifiers": results,
+                }
+            )
+        )
+    finally:
+        delete_databases(servers)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="EnterpriseOps-Gym judge driver")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("judge")
+    p.add_argument("--task-json", required=True)
+    args = ap.parse_args()
+    if args.cmd == "judge":
+        cmd_judge(args)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/programbench_judge.py

@@ -0,0 +1,120 @@
+# ProgramBench judge driver: materialize the worker's file-manifest submission
+# (read from stdin, ===FILE:<path>=== envelopes) into a run-dir
+# (<run>/<instance>/submission.tar.gz + <instance>.traj.json), pull the hidden
+# test blobs with `programbench blob sync`, run `programbench eval`, then parse
+# <instance>.eval.json applying the tests.json ignore mask. Emit
+# {"passed","total","resolved"} as the LAST stdout line. Fail loud: any harness or
+# Docker error prints {"error": "..."} and exits nonzero — never a fabricated score.
+
+import argparse
+import io
+import json
+import os
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+
+
+def fail(msg: str) -> None:
+    print(json.dumps({"error": msg}))
+    sys.exit(1)
+
+
+def parse_manifest(text: str) -> dict[str, str]:
+    """Split the ===FILE:<path>=== envelope stream into {path: contents}."""
+    files: dict[str, str] = {}
+    parts = text.split("===FILE:")
+    for part in parts[1:]:
+        header, _, body = part.partition("\n")
+        path = header.replace("===", "").strip()
+        if path:
+            files[path] = body
+    return files
+
+
+def run(cmd: list[str], cwd: str | None = None) -> subprocess.CompletedProcess:
+    return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="programbench single-instance judge")
+    ap.add_argument("--instance", required=True)
+    args = ap.parse_args()
+
+    submission_text = sys.stdin.read()
+    files = parse_manifest(submission_text)
+    # A submission with no compile.sh cannot build; the harness scores it 0, which
+    # is the honest fail-closed outcome — we still stage it so `total` is real.
+
+    pb = os.environ.get("PROGRAMBENCH_BIN", "programbench")
+
+    run_dir = tempfile.mkdtemp(prefix="programbench-run-")
+    try:
+        inst_dir = Path(run_dir) / args.instance
+        inst_dir.mkdir(parents=True, exist_ok=True)
+
+        # submission.tar.gz = gzipped tar of the agent's workspace files.
+        tar_path = inst_dir / "submission.tar.gz"
+        with tarfile.open(tar_path, "w:gz") as tar:
+            for path, content in files.items():
+                data = content.encode("utf-8")
+                info = tarfile.TarInfo(name=path)
+                info.size = len(data)
+                # compile.sh must be executable for the harness to run it.
+                info.mode = 0o755 if path.endswith(".sh") else 0o644
+                tar.addfile(info, io.BytesIO(data))
+
+        # Minimal trajectory file the harness expects alongside the submission.
+        (inst_dir / f"{args.instance}.traj.json").write_text(json.dumps({"instance_id": args.instance, "steps": []}))
+
+        # Pull the hidden behavioral test blobs for this instance.
+        sync = run([pb, "blob", "sync", args.instance], cwd=run_dir)
+        if sync.returncode != 0:
+            fail(f"programbench blob sync failed: {sync.stderr.strip()[:600]}")
+
+        # Run the cleanroom Docker image, build via compile.sh, run hidden tests.
+        ev = run([pb, "eval", run_dir], cwd=run_dir)
+        if ev.returncode != 0:
+            fail(f"programbench eval failed: {ev.stderr.strip()[:800]}")
+
+        eval_path = inst_dir / f"{args.instance}.eval.json"
+        if not eval_path.exists():
+            fail(f"programbench wrote no eval.json at {eval_path}: {ev.stdout.strip()[:400]}")
+
+        report = json.loads(eval_path.read_text())
+        if report.get("error_code"):
+            fail(f"programbench eval error_code={report['error_code']}: {report.get('error_details')}")
+
+        results = report.get("test_results", [])
+
+        # Apply the tests.json ignore mask (the same logic `programbench info`
+        # applies). The mask ships in the package data; load it if present.
+        ignored: set[str] = set()
+        try:
+            import programbench  # noqa: F401
+            from importlib import resources
+
+            data = resources.files("programbench").joinpath("data/tests.json")
+            if data.is_file():
+                mask = json.loads(data.read_text())
+                entry = mask.get(args.instance, {})
+                ignored = set(entry.get("ignored_tests", []))
+        except Exception:  # noqa: BLE001
+            ignored = set()
+
+        scored = [r for r in results if r.get("name") not in ignored]
+        passed = sum(1 for r in scored if r.get("status") == "passed")
+        total = len(scored)
+        resolved = total > 0 and passed == total
+
+        print(json.dumps({"passed": passed, "total": total, "resolved": resolved}))
+    finally:
+        import shutil
+
+        shutil.rmtree(run_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/render-gate-chart.mjs

@@ -0,0 +1,176 @@
+/**
+ * Render the gate results as clean, dependency-free SVG bar charts.
+ *
+ * One honest chart per benchmark: bars for each arm (baseline greys + the
+ * verifier-grounded method in the accent), an optional dashed oracle-ceiling line,
+ * and a single annotated delta. No chartjunk — value labels, one accent, lots of
+ * whitespace. Data is passed in (no hidden numbers) so the charts regenerate from
+ * the real run outputs.
+ *
+ *   node bench/scripts/render-gate-chart.mjs            # writes docs/assets/*.svg
+ *   (then: cairosvg docs/assets/<f>.svg -o /tmp/<f>.png  to preview)
+ */
+
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const assetsDir = join(here, '..', '..', 'docs', 'assets')
+
+const palette = {
+  bg: '#ffffff',
+  ink: '#0f172a', // slate-900 — titles, values
+  muted: '#64748b', // slate-500 — subtitles, labels
+  faint: '#e2e8f0', // slate-200 — baseline / axis
+  baseline: '#cbd5e1', // slate-300 — "blind" / weaker baselines
+  baseline2: '#94a3b8', // slate-400 — the competing selector
+  method: '#4f46e5', // indigo-600 — the verifier-grounded method (the hero)
+  methodSoft: '#eef2ff', // indigo-50 — method bar wash
+  ceiling: '#475569', // slate-600 — oracle ceiling line
+  good: '#059669', // emerald-600 — positive delta
+}
+
+const esc = (s) => String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+const f = 'system-ui, -apple-system, "Segoe UI", Inter, Roboto, sans-serif'
+
+/**
+ * @param {{
+ *   title: string, subtitle: string, footnote?: string,
+ *   bars: {label: string, value: number, kind: 'blind'|'rival'|'method'}[],
+ *   ceiling?: {value: number, label: string},
+ *   delta?: {fromIdx: number, toIdx: number, label: string},
+ *   yMax?: number, unit?: string,
+ * }} cfg
+ */
+function chart(cfg) {
+  const W = 820
+  const H = 524
+  const m = { top: 152, right: 136, bottom: 120, left: 66 }
+  const plotW = W - m.left - m.right
+  const plotH = H - m.top - m.bottom
+  const yMax = cfg.yMax ?? 100
+  const unit = cfg.unit ?? '%'
+  const y = (v) => m.top + plotH * (1 - v / yMax)
+
+  const n = cfg.bars.length
+  const bandW = plotW / n
+  const barW = Math.min(112, bandW * 0.56)
+  const cx = (i) => m.left + bandW * (i + 0.5)
+  const fill = { blind: palette.baseline, rival: palette.baseline2, method: palette.method }
+
+  const parts = []
+  parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}" viewBox="0 0 ${W} ${H}" font-family='${f}'>`)
+  parts.push(`<rect width="${W}" height="${H}" fill="${palette.bg}"/>`)
+
+  // title + subtitle
+  parts.push(`<text x="${m.left}" y="46" font-size="22" font-weight="700" fill="${palette.ink}">${esc(cfg.title)}</text>`)
+  parts.push(`<text x="${m.left}" y="72" font-size="13.5" fill="${palette.muted}">${esc(cfg.subtitle)}</text>`)
+
+  // baseline axis
+  parts.push(`<line x1="${m.left}" y1="${y(0)}" x2="${m.left + plotW}" y2="${y(0)}" stroke="${palette.faint}" stroke-width="1.5"/>`)
+  // light gridlines at 25/50/75/100
+  for (const g of [25, 50, 75, 100]) {
+    if (g > yMax) continue
+    parts.push(`<line x1="${m.left}" y1="${y(g)}" x2="${m.left + plotW}" y2="${y(g)}" stroke="${palette.faint}" stroke-width="1" stroke-dasharray="2 5"/>`)
+    parts.push(`<text x="${m.left - 12}" y="${y(g) + 4}" font-size="11" text-anchor="end" fill="${palette.muted}">${g}${unit}</text>`)
+  }
+
+  // oracle ceiling line
+  if (cfg.ceiling) {
+    const yc = y(cfg.ceiling.value)
+    parts.push(`<line x1="${m.left}" y1="${yc}" x2="${m.left + plotW}" y2="${yc}" stroke="${palette.ceiling}" stroke-width="1.5" stroke-dasharray="6 4"/>`)
+    parts.push(`<text x="${m.left + plotW + 8}" y="${yc - 5}" font-size="12" font-weight="600" fill="${palette.ceiling}">${esc(cfg.ceiling.label)}</text>`)
+    parts.push(`<text x="${m.left + plotW + 8}" y="${yc + 12}" font-size="12" fill="${palette.ceiling}">${cfg.ceiling.value}${unit}</text>`)
+  }
+
+  // bars
+  cfg.bars.forEach((b, i) => {
+    const x = cx(i) - barW / 2
+    const top = y(b.value)
+    const h = y(0) - top
+    const isMethod = b.kind === 'method'
+    if (isMethod) parts.push(`<rect x="${x - 3}" y="${top - 3}" width="${barW + 6}" height="${h + 3}" rx="7" fill="${palette.methodSoft}"/>`)
+    parts.push(`<rect x="${x}" y="${top}" width="${barW}" height="${h}" rx="5" fill="${fill[b.kind]}"/>`)
+    // value label
+    parts.push(`<text x="${cx(i)}" y="${top - 12}" font-size="16" font-weight="700" text-anchor="middle" fill="${isMethod ? palette.method : palette.ink}">${b.value}${unit}</text>`)
+    // x label (two lines if it contains a newline)
+    const lines = b.label.split('\n')
+    lines.forEach((ln, li) => {
+      parts.push(`<text x="${cx(i)}" y="${y(0) + 24 + li * 16}" font-size="12.5" font-weight="${isMethod ? 600 : 400}" text-anchor="middle" fill="${isMethod ? palette.method : palette.muted}">${esc(ln)}</text>`)
+    })
+  })
+
+  // delta annotation: a bracket from one bar's top to another's, labelled
+  if (cfg.delta) {
+    const a = cfg.delta.fromIdx
+    const b = cfg.delta.toIdx
+    const xa = cx(a)
+    const xb = cx(b)
+    const ya = y(cfg.bars[a].value)
+    const yb = y(cfg.bars[b].value)
+    const yBar = Math.min(ya, yb) - 34
+    const midx = (xa + xb) / 2
+    parts.push(`<line x1="${xa}" y1="${ya - 26}" x2="${xa}" y2="${yBar}" stroke="${palette.good}" stroke-width="1.25"/>`)
+    parts.push(`<line x1="${xb}" y1="${yb - 26}" x2="${xb}" y2="${yBar}" stroke="${palette.good}" stroke-width="1.25"/>`)
+    parts.push(`<line x1="${xa}" y1="${yBar}" x2="${xb}" y2="${yBar}" stroke="${palette.good}" stroke-width="1.25"/>`)
+    const lw = cfg.delta.label.length * 7.2 + 16
+    parts.push(`<rect x="${midx - lw / 2}" y="${yBar - 22}" width="${lw}" height="20" rx="10" fill="${palette.good}"/>`)
+    parts.push(`<text x="${midx}" y="${yBar - 8}" font-size="12.5" font-weight="700" text-anchor="middle" fill="#ffffff">${esc(cfg.delta.label)}</text>`)
+  }
+
+  if (cfg.footnote) {
+    // wrap to the plot width (~92 chars at 11.5px over the full canvas)
+    const words = cfg.footnote.split(' ')
+    const lines = []
+    let cur = ''
+    for (const w of words) {
+      if ((cur + ' ' + w).trim().length > 104) {
+        lines.push(cur.trim())
+        cur = w
+      } else cur = `${cur} ${w}`
+    }
+    if (cur.trim()) lines.push(cur.trim())
+    const startY = H - 20 - (lines.length - 1) * 15
+    lines.forEach((ln, i) => {
+      parts.push(`<text x="${m.left}" y="${startY + i * 15}" font-size="11.5" fill="${palette.muted}">${esc(ln)}</text>`)
+    })
+  }
+  parts.push('</svg>')
+  return parts.join('\n')
+}
+
+// ---- the charts (numbers are the real, verified run outputs) -------------------
+
+const humaneval = chart({
+  title: 'Verifier-grounded selection recovers the oracle ceiling',
+  subtitle: 'HumanEval · gpt-3.5-turbo · k=4 · n=50 · paired bootstrap (B=10000) + Benjamini–Hochberg',
+  bars: [
+    { label: 'Blind\n(1 shot)', value: 80, kind: 'blind' },
+    { label: 'Self-consistency\n(pick the consensus)', value: 82, kind: 'rival' },
+    { label: 'Verifier-grounded\n(re-run the tests)', value: 92, kind: 'method' },
+  ],
+  ceiling: { value: 92, label: 'oracle ceiling', },
+  delta: { fromIdx: 1, toIdx: 2, label: '+10.0pp  CI[+2, +18]' },
+  footnote: 'Deployable, non-oracle selector: ranks k attempts by re-running the task’s own unit tests (gold never shown). It captures the FULL ceiling (gap 0); self-consistency leaves 10pp on the table. Reproduced across two independent n=50 runs (+10pp / +12pp, both BH-positive).',
+})
+
+// commit0 — single-task Layer-1 VALIDATION (wcwidth, k=3). Regenerate with the powered
+// n=10 cross-task gate when it lands. Honestly labelled as a per-task validation.
+const commit0Validation = chart({
+  title: 'The selector signal survives real stateful coding rollouts',
+  subtitle: 'commit0 · gpt-4.1 · k=3 · single-task validation (wcwidth) — official pytest harness, graded pass-rate',
+  bars: [
+    { label: 'Random pick\n(blind, mean-of-k)', value: 67.5, kind: 'blind' },
+    { label: 'Verifier-grounded\n(highest test-pass)', value: 78.9, kind: 'method' },
+  ],
+  ceiling: { value: 78.9, label: 'best-of-k', },
+  delta: { fromIdx: 0, toIdx: 1, label: '+11.4pp' },
+  footnote: 'Layer-1: agent clones the repo, implements, runs pytest, iterates, emits a diff. Attempts score 60.5 / 78.9 / 63.2% — within-task variance the selector exploits. Powered n=10 cross-task gate in progress.',
+})
+
+mkdirSync(assetsDir, { recursive: true })
+writeFileSync(join(assetsDir, 'gate-humaneval.svg'), humaneval)
+writeFileSync(join(assetsDir, 'gate-commit0-validation.svg'), commit0Validation)
+console.log(`wrote ${join(assetsDir, 'gate-humaneval.svg')}`)
+console.log(`wrote ${join(assetsDir, 'gate-commit0-validation.svg')}`)

package/scripts/run-package-tests.mjs

@@ -0,0 +1,56 @@
+import { execFile } from 'node:child_process'
+import { access, readdir } from 'node:fs/promises'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+
+const execFileAsync = promisify(execFile)
+const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const sourceDir = path.join(benchDir, 'src')
+
+async function collectTests(dir) {
+  const files = []
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const absolute = path.join(dir, entry.name)
+    if (entry.isDirectory()) files.push(...(await collectTests(absolute)))
+    else if (entry.isFile() && /\.test\.(?:mts|ts)$/.test(entry.name)) files.push(absolute)
+  }
+  return files.sort()
+}
+
+async function run(command, args, env = process.env) {
+  try {
+    await execFileAsync(command, args, {
+      cwd: benchDir,
+      env,
+      maxBuffer: 10 * 1024 * 1024,
+      timeout: 120_000,
+    })
+  } catch (error) {
+    if (error?.stdout) process.stdout.write(error.stdout)
+    if (error?.stderr) process.stderr.write(error.stderr)
+    const invocation = [command, ...args].join(' ')
+    const message = error instanceof Error ? error.message : String(error)
+    throw new Error(`${invocation} failed: ${message}`, { cause: error })
+  }
+}
+
+const python = path.join(benchDir, '.venv', 'bin', 'python')
+try {
+  await access(python)
+} catch {
+  await run('python3', ['-m', 'venv', '.venv'])
+}
+
+const tests = await collectTests(sourceDir)
+const relativeTests = tests.map((file) => path.relative(benchDir, file))
+if (relativeTests.length === 0) throw new Error('no package tests found under src/')
+
+await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], {
+  ...process.env,
+  TSX_TSCONFIG_PATH: 'tsconfig.public.json',
+})
+
+await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
+
+console.log(`package tests passed: ${tests.length}/${tests.length} TypeScript files + Pier bridge`)