@tangle-network/agent-bench 0.1.0 → 0.3.5

package/scripts/appworld_driver.py

@@ -0,0 +1,359 @@
+# AppWorld engine driver. Two subcommands the TS adapter shells out to:
+#   load     --split S [--limit N] [--ids a,b]   -> {"tasks":[{task_id,instruction}]}
+#   evaluate --task-id T --split S  (solution code on stdin)
+#                                                -> {"success":bool,"passes":int,"fails":int}
+# evaluate runs the worker's solution in a fresh AppWorld(task_id=T) world, then
+# AppWorld's OWN programmatic evaluator (world.evaluate().to_dict()) reports the
+# verdict. JSON is emitted as the LAST stdout line. Fail loud: any engine error is
+# printed as {"error": "..."} and exits nonzero — never a fabricated verdict.
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+
+
+def fail(msg: str) -> None:
+    print(json.dumps({"error": msg}))
+    sys.exit(1)
+
+
+_CODE_RE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL)
+
+
+def _extract_code(text: str) -> str:
+    blocks = _CODE_RE.findall(text or "")
+    return (blocks[-1] if blocks else "").strip()
+
+
+def _router_chat(base: str, key: str, model: str, messages: list, timeout: float = 180.0):
+    """One router chat-completion with retry on transient/429/5xx. Returns
+    (content, input_tokens, output_tokens). Raises on exhausted retries."""
+    import httpx
+
+    url = base.rstrip("/") + "/chat/completions"
+    last = None
+    for attempt in range(4):
+        try:
+            r = httpx.post(
+                url,
+                headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
+                json={"model": model, "messages": messages},
+                timeout=timeout,
+            )
+            if r.status_code in (429, 500, 502, 503, 504):
+                last = f"{r.status_code}: {r.text[:160]}"
+                time.sleep(2**attempt)
+                continue
+            r.raise_for_status()
+            d = r.json()
+            content = (d["choices"][0]["message"].get("content") or "")
+            usage = d.get("usage") or {}
+            return content, int(usage.get("prompt_tokens", 0) or 0), int(usage.get("completion_tokens", 0) or 0)
+        except Exception as e:  # noqa: BLE001
+            last = str(e)
+            if attempt < 3:
+                time.sleep(2**attempt)
+                continue
+            raise RuntimeError(f"router_chat failed after retries: {last}")
+    raise RuntimeError(f"router_chat exhausted: {last}")
+
+
+def _build_system(directive: str, world) -> str:
+    sup = world.task.supervisor
+    apps = list(getattr(world.task, "allowed_apps", []) or [])
+    descs = getattr(world.task, "app_descriptions", "")
+    desc_str = json.dumps(descs) if isinstance(descs, (dict, list)) else str(descs)
+    return (
+        f"You are an AI agent completing a digital task for your supervisor "
+        f"{getattr(sup, 'first_name', '')} {getattr(sup, 'last_name', '')} "
+        f"(email {getattr(sup, 'email', '')}, phone {getattr(sup, 'phone_number', '')}) "
+        "by WRITING PYTHON that calls app APIs (the apis.<app>.<function>(...) surface).\n\n"
+        f"Available apps: {', '.join(apps)}.\n"
+        f"App descriptions: {desc_str[:1500]}\n\n"
+        "How to work, one step per turn:\n"
+        "- Discover APIs with apis.api_docs.show_api_descriptions(app_name='<app>') and "
+        "apis.api_docs.show_api_doc(app_name='<app>', api_name='<api>') BEFORE calling them.\n"
+        "- Get the supervisor's app passwords with apis.supervisor.show_account_passwords(), then log in "
+        "to each app you use to obtain its access_token.\n"
+        "- Write ONE short Python code block per turn. After it runs you SEE its OUTPUT (or error "
+        "traceback) — use that to decide the next step. Print intermediate values you need.\n"
+        "- Iterate: inspect -> authenticate -> act -> verify. Do not guess API names or arguments.\n"
+        "- When the task is fully done call apis.supervisor.complete_task(answer=<answer>) (include the "
+        "answer if the task asks a question, otherwise apis.supervisor.complete_task()).\n"
+        "- Reply with EXACTLY ONE fenced ```python block per turn and nothing else.\n\n"
+        f"{directive}"
+    )
+
+
+def cmd_react(args) -> None:
+    """Multi-turn REPL agent: the model writes a python block, the engine executes
+    it in the PERSISTENT world, the output is fed back, and it iterates until it
+    completes the task or hits max-turns. Then AppWorld's own evaluator scores it.
+    Config (directive, model, router creds, max_turns) arrives as JSON on stdin so
+    the candidate directive can be arbitrarily long. The directive is the optimized
+    surface; the loop + contract are fixed."""
+    cfg = {}
+    raw = sys.stdin.read()
+    if raw.strip():
+        try:
+            cfg = json.loads(raw)
+        except Exception as e:  # noqa: BLE001
+            fail(f"react config JSON parse failed: {e}")
+    directive = str(cfg.get("directive", ""))
+    model = str(cfg.get("model", "gpt-4o"))
+    max_turns = int(cfg.get("max_turns", 8))
+    router_base = str(cfg.get("router_base", "https://router.tangle.tools/v1"))
+    router_key = str(cfg.get("router_key") or os.environ.get("TANGLE_API_KEY", ""))
+    if not router_key:
+        fail("react: router_key/TANGLE_API_KEY required")
+
+    try:
+        from appworld import AppWorld
+    except Exception as e:  # noqa: BLE001
+        fail(f"appworld import failed: {e}")
+
+    in_tok = 0
+    out_tok = 0
+    turns = 0
+    turns_log: list = []
+    try:
+        with AppWorld(
+            task_id=args.task_id,
+            experiment_name="bench-react",
+            raise_on_failure=False,
+        ) as world:
+            messages = [
+                {"role": "system", "content": _build_system(directive, world)},
+                {"role": "user", "content": f"Task: {world.task.instruction}"},
+            ]
+            for turn in range(max_turns):
+                turns = turn + 1
+                content, ui, uo = _router_chat(router_base, router_key, model, messages)
+                in_tok += ui
+                out_tok += uo
+                code = _extract_code(content)
+                messages.append({"role": "assistant", "content": content})
+                if not code:
+                    messages.append({
+                        "role": "user",
+                        "content": "Reply with exactly one ```python block that makes progress, "
+                        "or call apis.supervisor.complete_task().",
+                    })
+                    continue
+                output = world.execute(code)
+                turns_log.append({"code": code[:600], "output": str(output)[:600]})
+                messages.append({"role": "user", "content": "OUTPUT:\n" + str(output)[:4000]})
+                if world.task_completed():
+                    break
+            evaluation = world.evaluate().to_dict()
+    except Exception as e:  # noqa: BLE001
+        fail(f"react of {args.task_id} failed: {e}")
+
+    if "success" not in evaluation or "num_tests" not in evaluation:
+        fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}")
+    passes = evaluation.get("passes", [])
+    failures = evaluation.get("failures", [])
+    n_pass = len(passes) if isinstance(passes, list) else int(passes or 0)
+    n_fail = len(failures) if isinstance(failures, list) else int(failures or 0)
+    print(
+        json.dumps(
+            {
+                "success": bool(evaluation["success"]),
+                "passes": n_pass,
+                "fails": n_fail,
+                "num_tests": int(evaluation["num_tests"]),
+                # Failed sub-test names — the evidence a trace analyst steers on.
+                "failure_names": [str(f)[:160] for f in failures][:8]
+                if isinstance(failures, list)
+                else [],
+                "turns": turns,
+                "input_tokens": in_tok,
+                "output_tokens": out_tok,
+                "transcript": "\n---\n".join(
+                    f"CODE:\n{t['code']}\nOUTPUT:\n{t['output']}" for t in turns_log[-3:]
+                )[:1600],
+            }
+        )
+    )
+
+
+def cmd_session(args) -> None:
+    """Dumb world shim: a persistent AppWorld session driven over stdin JSONL.
+    NO LLM calls here — the agent loop lives in the runtime (routerToolLoop);
+    this process only owns world state. One JSON object per line, both ways:
+      {"op":"execute","code":"..."} -> {"output":"...","task_completed":bool}
+      {"op":"evaluate"}             -> the evaluate verdict JSON (+failure_names)
+    Emits {"ready":true,"instruction":...} on start; exits on stdin EOF."""
+    try:
+        from appworld import AppWorld
+    except Exception as e:  # noqa: BLE001
+        fail(f"appworld import failed: {e}")
+    try:
+        with AppWorld(
+            task_id=args.task_id,
+            experiment_name="bench-session",
+            raise_on_failure=False,
+        ) as world:
+            print(json.dumps({"ready": True, "instruction": world.task.instruction}), flush=True)
+            for line in sys.stdin:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    cmd = json.loads(line)
+                except Exception as e:  # noqa: BLE001
+                    print(json.dumps({"error": f"bad command JSON: {e}"}), flush=True)
+                    continue
+                op = cmd.get("op")
+                if op == "execute":
+                    # An exception here is the AGENT's outcome (bad code), not an
+                    # infra fault — feed it back, keep the world alive.
+                    try:
+                        output = str(world.execute(str(cmd.get("code", ""))))
+                    except Exception as e:  # noqa: BLE001
+                        output = f"EXECUTION ERROR: {e}"
+                    print(
+                        json.dumps(
+                            {"output": output[:4000], "task_completed": bool(world.task_completed())}
+                        ),
+                        flush=True,
+                    )
+                elif op == "evaluate":
+                    ev = world.evaluate().to_dict()
+                    passes = ev.get("passes", [])
+                    failures = ev.get("failures", [])
+                    print(
+                        json.dumps(
+                            {
+                                "success": bool(ev.get("success")),
+                                "passes": len(passes) if isinstance(passes, list) else int(passes or 0),
+                                "fails": len(failures) if isinstance(failures, list) else int(failures or 0),
+                                "num_tests": int(ev.get("num_tests", 0)),
+                                "failure_names": [str(f)[:160] for f in failures][:8]
+                                if isinstance(failures, list)
+                                else [],
+                            }
+                        ),
+                        flush=True,
+                    )
+                else:
+                    print(json.dumps({"error": f"unknown op: {op}"}), flush=True)
+    except Exception as e:  # noqa: BLE001
+        fail(f"session of {args.task_id} failed: {e}")
+
+
+def cmd_load(args) -> None:
+    try:
+        from appworld import load_task_ids
+    except Exception as e:  # noqa: BLE001
+        fail(f"appworld import failed: {e}")
+
+    try:
+        ids = list(load_task_ids(args.split))
+    except Exception as e:  # noqa: BLE001
+        fail(f"load_task_ids({args.split}) failed: {e}")
+
+    if args.ids:
+        want = set(args.ids.split(","))
+        ids = [i for i in ids if i in want]
+    elif args.limit is not None:
+        ids = ids[: args.limit]
+
+    from appworld import AppWorld
+
+    tasks = []
+    for task_id in ids:
+        # Open each world read-only just to read the instruction; close immediately.
+        try:
+            with AppWorld(task_id=task_id, experiment_name="bench-load") as world:
+                tasks.append({"task_id": task_id, "instruction": world.task.instruction})
+        except Exception as e:  # noqa: BLE001
+            fail(f"opening task {task_id} failed: {e}")
+    print(json.dumps({"tasks": tasks}))
+
+
+def cmd_evaluate(args) -> None:
+    code = sys.stdin.read()
+    try:
+        from appworld import AppWorld
+    except Exception as e:  # noqa: BLE001
+        fail(f"appworld import failed: {e}")
+
+    try:
+        with AppWorld(
+            task_id=args.task_id,
+            experiment_name="bench-eval",
+            # An API error should surface to the agent's output, not crash the world.
+            raise_on_failure=False,
+        ) as world:
+            if code.strip():
+                world.execute(code)
+            evaluation = world.evaluate().to_dict()
+    except Exception as e:  # noqa: BLE001
+        fail(f"evaluate of {args.task_id} failed: {e}")
+
+    # TestTracker.to_dict() carries `success` (binary task-goal-completion),
+    # `num_tests` (the authoritative per-requirement total), and the
+    # `passes`/`failures` lists. `failures` (not `fails`) is the real key — read it
+    # directly and never substitute a default count.
+    if "success" not in evaluation or "num_tests" not in evaluation:
+        fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}")
+    success = bool(evaluation["success"])
+    passes = evaluation.get("passes", [])
+    failures = evaluation.get("failures", [])
+    n_pass = len(passes) if isinstance(passes, list) else int(passes or 0)
+    n_fail = len(failures) if isinstance(failures, list) else int(failures or 0)
+    print(
+        json.dumps(
+            {
+                "success": success,
+                "passes": n_pass,
+                "fails": n_fail,
+                "num_tests": int(evaluation["num_tests"]),
+                # The failed sub-test names are the diagnosable evidence a trace
+                # analyst steers on — bounded so the JSON line stays small.
+                "failure_names": [str(f)[:160] for f in failures][:8]
+                if isinstance(failures, list)
+                else [],
+            }
+        )
+    )
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="appworld engine driver")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p_load = sub.add_parser("load")
+    p_load.add_argument("--split", required=True)
+    p_load.add_argument("--limit", type=int, default=None)
+    p_load.add_argument("--ids", default=None)
+
+    p_eval = sub.add_parser("evaluate")
+    p_eval.add_argument("--task-id", required=True)
+    p_eval.add_argument("--split", required=True)
+
+    p_react = sub.add_parser("react")
+    p_react.add_argument("--task-id", required=True)
+    p_react.add_argument("--split", required=True)
+
+    p_session = sub.add_parser("session")
+    p_session.add_argument("--task-id", required=True)
+    p_session.add_argument("--split", required=True)
+
+    args = ap.parse_args()
+    if args.cmd == "load":
+        cmd_load(args)
+    elif args.cmd == "evaluate":
+        cmd_evaluate(args)
+    elif args.cmd == "react":
+        cmd_react(args)
+    elif args.cmd == "session":
+        cmd_session(args)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/cadbench_prepare.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Fetch FreedomIntelligence/CADBench + flatten its nested `criteria` into a flat
+bullet list, writing the cleaned JSONL the CADBench adapter reads via CADBENCH_PATH.
+Usage: python3 scripts/cadbench_prepare.py [out.jsonl]  (clones the HF dataset to /tmp)"""
+import json, subprocess, sys, os
+OUT = sys.argv[1] if len(sys.argv) > 1 else '/tmp/cadbench_clean.jsonl'
+SRC = '/tmp/cadbench-ds'
+if not os.path.exists(SRC):
+    subprocess.run(['git', 'clone', '--depth', '1', 'https://huggingface.co/datasets/FreedomIntelligence/CADBench', SRC], check=True)
+def flatten(x, out):
+    if isinstance(x, dict):
+        for v in x.values(): flatten(v, out)
+    elif isinstance(x, list):
+        for v in x: flatten(v, out)
+    elif isinstance(x, str): out.append(x)
+rows = [json.loads(l) for l in open(f'{SRC}/CADBench.jsonl', encoding='utf-8-sig') if l.strip()]
+clean = []
+for r in rows:
+    b = []; flatten(r['criteria'], b)
+    clean.append({'id': r['id'], 'name': r.get('name', ''), 'instruction': r['instruction'], 'type': r.get('type', ''), 'criteria': b})
+open(OUT, 'w').write('\n'.join(json.dumps(c) for c in clean))
+print(f'wrote {len(clean)} tasks -> {OUT}')

package/scripts/cadgenbench_hard_parts.py

@@ -0,0 +1,48 @@
+from build123d import *
+import os, json
+OUT = "/tmp/cgb-hard"; os.makedirs(OUT, exist_ok=True)
+tasks = []
+def save(pid, part, desc):
+    p = f"{OUT}/{pid}.step"; export_step(part, p)
+    tasks.append({"id": pid, "prompt": desc, "gtStep": p}); print(pid, "OK", round(part.volume,1))
+
+# 1. L-bracket: base plate + vertical leg + 4 holes through base
+base = Pos(0,0,4) * Box(60,40,8)
+leg  = Pos(-26,0,30) * Box(8,40,52)
+lb = base + leg
+for (x,y) in [(-10,-10),(-10,10),(20,-10),(20,10)]:
+    lb -= Pos(x,y,4) * Cylinder(3,12)
+save("l-bracket", lb, "An L-shaped steel mounting bracket. A base plate 60 (X) by 40 (Y) by 8 (Z) units sits flat with its bottom at z=0, centered on X and Y. A vertical leg 8 (X) by 40 (Y) by 52 (Z) units rises from the base's -X edge, forming a right angle (the whole part is ~60 wide, 40 deep, 60 tall). Four vertical through-holes of radius 3 pass through the base plate at X,Y offsets (-10,-10), (-10,10), (20,-10), (20,10) from center.")
+
+# 2. Washer: flat annulus
+w = Cylinder(20,4) - Cylinder(10,4)
+save("washer", w, "A flat circular washer: an annular disk of outer radius 20, inner radius 10, and thickness 4 units, centered at the origin.")
+
+# 3. Flanged pipe: pipe + flange disk at base + 4 bolt holes in flange
+pipe = Cylinder(12,50, align=(Align.CENTER,Align.CENTER,Align.MIN)) - Cylinder(8,50, align=(Align.CENTER,Align.CENTER,Align.MIN))
+flange = Cylinder(24,6, align=(Align.CENTER,Align.CENTER,Align.MIN)) - Cylinder(8,6, align=(Align.CENTER,Align.CENTER,Align.MIN))
+fp = pipe + flange
+import math
+for i in range(4):
+    a = math.radians(45 + i*90); fp -= Pos(18*math.cos(a),18*math.sin(a),3) * Cylinder(2.5,8)
+save("flanged-pipe", fp, "A flanged pipe. A hollow cylindrical pipe of outer radius 12, inner radius 8 (bore), height 50, standing with its base at z=0. At the base is a circular flange disk of radius 24 and thickness 6, sharing the same central bore (radius 8). Four bolt holes of radius 2.5 pass vertically through the flange on a circle of radius 18 from the axis, at 45, 135, 225, 315 degrees.")
+
+# 4. Stepped shaft: 3 stacked cylinders, decreasing radius
+s = Cylinder(15,20, align=(Align.CENTER,Align.CENTER,Align.MIN))
+s += Pos(0,0,20) * Cylinder(10,20, align=(Align.CENTER,Align.CENTER,Align.MIN))
+s += Pos(0,0,40) * Cylinder(6,20, align=(Align.CENTER,Align.CENTER,Align.MIN))
+save("stepped-shaft", s, "A stepped cylindrical shaft, coaxial along Z, base at z=0: a bottom section radius 15 height 20, a middle section radius 10 height 20, and a top section radius 6 height 20 (total height 60).")
+
+# 5. Mounting plate: plate + 4 corner holes + center hole
+mp = Box(80,60,6)
+for (x,y) in [(-32,-22),(-32,22),(32,-22),(32,22)]:
+    mp -= Pos(x,y,0) * Cylinder(3,10)
+mp -= Cylinder(8,10)
+save("mounting-plate", mp, "A rectangular mounting plate 80 (X) by 60 (Y) by 6 (Z) units, centered at the origin. A central through-hole of radius 8, plus four through-holes of radius 3 at the corners, offset (+/-32, +/-22) from center.")
+
+# 6. Hex standoff: hexagonal prism with axial hole
+hexp = extrude(RegularPolygon(radius=10, side_count=6), 30) - Cylinder(5,30, align=(Align.CENTER,Align.CENTER,Align.MIN))
+save("hex-standoff", hexp, "A hexagonal standoff: a regular hexagonal prism (circumradius 10, i.e. ~17.3 across flats... actually 20 across corners) extruded 30 units tall from z=0, with a coaxial cylindrical through-hole of radius 5 along its full height.")
+
+json.dump(tasks, open(f"{OUT}/tasks.json","w"), indent=2)
+print("WROTE", len(tasks), "tasks")

package/scripts/clbench_codebase_judge.py

@@ -0,0 +1,73 @@
+"""Deployable-checker bridge for CL-Bench (Continual) Codebase Adaptation.
+
+CL-Bench's `codebase_adaptation` is the ONE domain whose scorer is a deployable
+checker (not an oracle): it applies the instance's provided `test_patch` and runs
+the project's pytest suite inside the instance's Docker image, keying off the exit
+code — exactly the SWE-bench / commit0 regime. This bridge exposes that scorer as a
+standalone (instance_id, patch) -> {success,status} call so our TypeScript gate can
+rank K candidate patches by a verifier the agent could legitimately run itself.
+
+Run it with CL-Bench's OWN venv + repo root on the path (its `src.tasks...` package):
+
+    <clbench>/.venv/bin/python clbench_codebase_judge.py \
+        --dataset <clbench>/data/codebase_adaptation/final-dataset.jsonl \
+        --instance-id jazzband__tablib-534 --patch-file /tmp/candidate.patch
+    # invoked with cwd=<clbench> so `import src.tasks...` resolves
+
+Prints one JSON line: {"instance_id","success","status","error"}. Fail loud — a
+Docker/import failure exits non-zero with the message on stderr, never a silent 0.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from src.tasks.codebase_adaptation.evaluator import evaluate_submission
+
+
+def load_instance(dataset_path: str, instance_id: str) -> dict:
+    with open(dataset_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            row = json.loads(line)
+            if row.get("instance_id") == instance_id:
+                return row
+    raise SystemExit(f"instance_id not found in {dataset_path}: {instance_id}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="CL-Bench codebase_adaptation deployable judge")
+    ap.add_argument("--dataset", required=True, help="path to final-dataset.jsonl")
+    ap.add_argument("--instance-id", required=True)
+    ap.add_argument("--patch-file", required=True, help="file holding the candidate unified git diff")
+    args = ap.parse_args()
+
+    instance = load_instance(args.dataset, args.instance_id)
+    with open(args.patch_file, encoding="utf-8") as f:
+        patch = f.read()
+
+    # evaluate_submission spins the instance's Docker image, applies test_patch + the
+    # candidate, runs pytest, and reports success on a clean exit. A genuine infra
+    # failure raises — let it propagate (non-zero exit) so the caller never reads a
+    # transport fault as a failed test.
+    result = evaluate_submission(patch, instance)
+    print(json.dumps({
+        "instance_id": args.instance_id,
+        "success": bool(result.success),
+        "status": result.status,
+        "error": (result.error or "")[:500],
+    }))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except SystemExit:
+        raise
+    except Exception as exc:  # infra/import failure — fail loud, do not emit a fake verdict
+        print(f"clbench_codebase_judge: {type(exc).__name__}: {exc}", file=sys.stderr)
+        sys.exit(2)

package/scripts/commit0_judge.py

@@ -0,0 +1,170 @@
+# Commit0 judge driver: stage one repo at base_commit, apply the worker's diff
+# (read from stdin) into src_dir, run the official commit0 pytest harness on a
+# local Docker backend, and emit {"passed": int, "total": int} as the LAST stdout
+# line. The TS adapter wraps this; scoring (passed+xfail)/total mirrors
+# commit0.harness.evaluate. Fail loud: any harness/Docker error is printed as
+# {"error": "..."} and a nonzero exit — never a fabricated pass count.
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def fail(msg: str) -> None:
+    print(json.dumps({"error": msg}))
+    sys.exit(1)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="commit0 single-instance judge")
+    ap.add_argument("--dataset", required=True)
+    ap.add_argument("--split", required=True)
+    ap.add_argument("--instance", required=True)  # e.g. commit-0/wcwidth
+    ap.add_argument("--src-dir", required=True)
+    ap.add_argument("--backend", default=os.environ.get("COMMIT0_BACKEND", "local"))
+    ap.add_argument("--timeout", type=int, default=int(os.environ.get("COMMIT0_TIMEOUT", "1800")))
+    args = ap.parse_args()
+
+    diff = sys.stdin.read()
+    if not diff.strip():
+        # Empty implementation: nothing to apply → 0 of the suite passes. Stage the
+        # repo anyway so `total` reflects the real test count (fail-closed, real).
+        diff = ""
+
+    try:
+        from datasets import load_dataset
+        from commit0.harness.setup import main as setup_main
+        from commit0.harness.run_pytest_ids import main as run_tests
+        from commit0.harness.get_pytest_ids import main as get_tests
+        from commit0.harness.constants import BASE_BRANCH, RUN_PYTEST_LOG_DIR
+        from commit0.harness.utils import get_hash_string
+    except Exception as e:  # noqa: BLE001
+        fail(f"commit0 import failed: {e}")
+
+    repo_name = args.instance.split("/")[-1]
+    ds = load_dataset(args.dataset, split=args.split)
+    example = next((x for x in ds if x["instance_id"] == args.instance), None)
+    if example is None:
+        fail(f"instance not found in {args.dataset}: {args.instance}")
+
+    # run_pytest_ids wants the SPACE-joined test-id STRING (e.g. "tests/a.py::t1 ...")
+    # — NOT the dataset's test_dir — and keys its report dir on get_hash_string(that
+    # same string). Passing test_dir ran zero tests and wrote the report under a
+    # different hash, so the judge silently found nothing. get_tests reads the test-ids.
+    test_ids = [t for t in get_tests(repo_name, 0) if t.strip()]
+    test_ids_str = " ".join(test_ids)
+    # setup.main checks out the stubbed repo onto BASE_BRANCH ("commit0") and the
+    # worker's diff is committed there; run_pytest_ids resolves `branch` -> a git
+    # commit (NOT a checkout) to diff against base_commit, so it must be the
+    # branch the work actually lives on. The dataset short name is not a git ref.
+    branch = BASE_BRANCH
+
+    base_dir = tempfile.mkdtemp(prefix="commit0-base-")
+    try:
+        # Clone + checkout the stubbed base branch for just this repo.
+        setup_main(args.dataset, args.split, repo_name, base_dir)
+
+        repo_dir = Path(base_dir) / repo_name
+        if not repo_dir.exists():
+            fail(f"commit0 setup did not clone {repo_name} into {base_dir}")
+
+        # Apply the worker's unified diff into the cloned repo (src only). git apply
+        # tolerates a/ b/ prefixes; --reject is avoided so a bad hunk fails loud.
+        if diff:
+            patch_path = Path(base_dir) / "worker.diff"
+            patch_path.write_text(diff)
+            r = subprocess.run(
+                ["git", "apply", "--whitespace=nowarn", str(patch_path)],
+                cwd=str(repo_dir),
+                capture_output=True,
+                text=True,
+            )
+            if r.returncode != 0:
+                # Retry with -p0 (diffs sometimes lack the a/ b/ prefix).
+                r2 = subprocess.run(
+                    ["git", "apply", "-p0", "--whitespace=nowarn", str(patch_path)],
+                    cwd=str(repo_dir),
+                    capture_output=True,
+                    text=True,
+                )
+                if r2.returncode != 0:
+                    fail(f"git apply failed: {r.stderr.strip()[:500]} / {r2.stderr.strip()[:500]}")
+            subprocess.run(["git", "add", "-A"], cwd=str(repo_dir), check=False)
+            subprocess.run(
+                ["git", "-c", "user.email=bench@local", "-c", "user.name=bench", "commit", "-m", "worker"],
+                cwd=str(repo_dir),
+                capture_output=True,
+                text=True,
+            )
+
+        # Run the official per-repo pytest harness on the requested backend.
+        try:
+            run_tests(
+                args.dataset,
+                args.split,
+                base_dir,
+                repo_name,
+                branch,
+                test_ids_str,
+                False,  # coverage
+                args.backend,
+                args.timeout,
+                1,  # num_cpus
+                # rebuild_image is honored by the MODAL backend only (force_build).
+                # The local Docker execution context just creates a container from
+                # the existing wentingzhao/<repo>:v0 image — pre-pull/build it via
+                # bench/src/commit0-prereqs.sh or this run dies with ImageNotFound.
+                rebuild_image=os.environ.get("COMMIT0_REBUILD_IMAGE") == "1",
+                verbose=0,
+            )
+        except SystemExit:
+            # run_pytest_ids ends with sys.exit(pytest_exit_code): 0 = all pass, nonzero
+            # = some tests failed — the NORMAL graded/partial-credit case, NOT a harness
+            # error. report.json is already written; fall through and read it.
+            pass
+        except BaseException as e:  # noqa: BLE001 — never exit silently on a real backend fault
+            if isinstance(e, KeyboardInterrupt):
+                raise
+            import traceback
+
+            hint = ""
+            if "ImageNotFound" in repr(e) or "No such image" in repr(e):
+                hint = f" | missing per-repo Docker image — run: bench/src/commit0-prereqs.sh {repo_name}"
+            fail(f"commit0 run_pytest_ids failed for {repo_name}: {e!r}{hint} | {traceback.format_exc()[-1200:]}")
+
+        hashed = get_hash_string(test_ids_str)
+        report_file = RUN_PYTEST_LOG_DIR / repo_name / branch / hashed / "report.json"
+        if not report_file.exists():
+            fail(f"commit0 wrote no report.json at {report_file}")
+
+        report = json.loads(report_file.read_text())
+        # pytest-json: new form has "created"+"tests"; old form is a flat list.
+        if isinstance(report, dict) and "created" in report:
+            calls = {x["nodeid"]: x.get("call", {}) for x in report.get("tests", []) if "call" in x}
+            outcomes = [c.get("outcome") for c in calls.values()]
+        else:
+            outcomes = [x["outcome"] for x in report if x.get("when") == "call"]
+
+        passed = sum(1 for o in outcomes if o in ("passed", "xfailed"))
+        total = len(outcomes)
+        # Fall back to the declared test-id count when the harness produced no call
+        # records (e.g. collection error) so `total` is never a phantom 0.
+        if total == 0:
+            # COUNT of test ids, not their character lengths (get_tests returns a flat
+            # list of node-id strings). Only fires on a collection error, where passed
+            # is already 0 — but a phantom inflated total would still mislead a reader.
+            total = len(test_ids)
+
+        print(json.dumps({"passed": passed, "total": total}))
+    finally:
+        import shutil
+
+        shutil.rmtree(base_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()