chill-agent 1.5.0 → 1.6.0

package/benchmarks/agentic/answer.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""ANSWER-QUALITY judge for the agentic benchmark: does the agent's final reply tell a human what
+they need to know?
+
+The other passes score the code. This one scores the reply a developer reads at the end of a
+session. A short reply that hides a skipped requirement is bad; so is an essay nobody finishes.
+Length is reported (words) but never scored: a reply is as long as understanding needs.
+
+Two calls per cell, so the score measures what a READER takes away, not what the grader can dig
+out of the diff itself:
+  1. READER (blind): sees only the task and the reply, and writes down what it now believes:
+     what changed, what was left out, what the user must do, what to watch out for.
+  2. GRADER: sees the task, the reply, the reader's notes and the real diff, and scores 0-3:
+     accuracy (are the reader's beliefs true?), coverage (did the reader learn everything a
+     reviewer must know before merging?), clarity (is the reply itself easy to read?).
+
+  python answer.py --selftest-offline   # gate logic only, no model calls
+  python answer.py --selftest           # live: the judge must rank a good reply above a terse one
+                                        # (coverage), a false one (accuracy), an essay (clarity)
+  python answer.py --run runs/<stamp>   # judge every cell's final reply
+
+No API key needed: without one, judge.py routes calls through the headless claude CLI.
+"""
+import argparse, concurrent.futures, difflib, json, random, statistics, subprocess, sys
+from collections import defaultdict
+from pathlib import Path
+
+from tasks import TASKS
+from judge import RUNS_DIR, judge_call, load_key, parse_score
+
+MODEL = "claude-sonnet-5-5"
+DIFF_LIMIT = 20000
+SCORES = ("accuracy", "coverage", "clarity")
+
+READER = (
+    "You asked a coding agent to do a task. You have NOT seen the code. Read the agent's final reply "
+    "and write down what you now believe, using only what the reply says. Respond with ONLY this JSON:\n"
+    '{"changed": ["..."], "left_out": ["..."], "you_must_do": ["..."], "watch_out": ["..."]}\n'
+    "changed = what the agent changed or built; left_out = what it says it did not do or skipped on "
+    "purpose; you_must_do = actions it says you must take now; watch_out = risks or limits it names. "
+    "Use [] when the reply says nothing about a field."
+)
+GRADER = (
+    "You check how well a coding agent's final reply informs a busy developer. You get the task, the "
+    "reply, the notes a reader wrote after reading ONLY the reply, and the real diff of what the agent "
+    "did. Score 0-3 each:\n"
+    "accuracy: are the reader's notes true according to the diff? 3 = all true; 0 = a major false "
+    "belief (claims work that is not in the diff, or misstates behavior).\n"
+    "coverage: did the reader learn everything a reviewer must know before merging? That is: every "
+    "behavior change, anything removed, anything the task asked or clearly implied that the diff does "
+    "NOT deliver, and any action the user must take. 3 = nothing important missing; 0 = the reader "
+    "would merge without knowing something important.\n"
+    "clarity: judge the REPLY itself. Plain words, easy to scan, no filler, no repeated recap. A wall "
+    "of text, hedging and pleasantries lower it; so do cryptic fragments, arrows or abbreviations a "
+    "reader must decode. 3 = a developer gets it in one quick read. Do not reward length or brevity "
+    "as such.\n"
+    'Respond with ONLY this JSON: {"accuracy": n, "coverage": n, "clarity": n, '
+    '"missed": "<most important thing the reader did not learn, or none>", "false": "<false belief, or none>"}'
+)
+
+PAIR = (
+    "Two coding agents got the same task. For each you see its final reply to the developer and the "
+    "real diff of what it did. Decide which REPLY serves a busy developer better: true to its own diff, "
+    "tells everything a reviewer must know (behavior changes, removals, missing or skipped parts, actions "
+    "to take, risks), and is easy to read in one pass (no filler, no essay, no cryptic fragments). Judge "
+    "the reply, not the code, and do not prefer a reply for being longer or shorter. Respond with ONLY "
+    'this JSON: {"better": "1" | "2" | "tie", "why": "<one line>"}'
+)
+
+def workspace_diff(task_id, ws: Path):
+    """What the agent actually did: git diff for snapshotted workspaces, else a diff against the seed."""
+    if (ws / ".git").exists():
+        subprocess.run(["git", "add", "-A"], cwd=ws, capture_output=True)
+        d = subprocess.run(["git", "diff", "--cached", "HEAD", "--", ".", ":(exclude)_*", ":(exclude)**/__pycache__/**"],
+                           cwd=ws, capture_output=True, text=True).stdout
+    else:
+        seed, parts = TASKS[task_id].get("seed", {}), []
+        for p in sorted(ws.rglob("*")):
+            rel = p.relative_to(ws).as_posix()
+            if not p.is_file() or "__pycache__" in p.parts or p.name.startswith(("_", ".")): continue
+            new = p.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True)
+            parts += difflib.unified_diff(seed.get(rel, "").splitlines(keepends=True), new, "a/" + rel, "b/" + rel)
+        d = "".join(parts)
+    return (d[:DIFF_LIMIT] + "\n[diff truncated]") if len(d) > DIFF_LIMIT else (d or "(no files changed)")
+
+def judge_reply(task_prompt, reply, diff, key, model=MODEL):
+    notes = judge_call(None, None, key, system=READER, model=model, max_tokens=800,
+                       user=f"TASK:\n{task_prompt}\n\nAGENT'S FINAL REPLY:\n{reply}")
+    d = parse_score(judge_call(None, None, key, system=GRADER, model=model, max_tokens=400,
+                               user=f"TASK:\n{task_prompt}\n\nAGENT'S FINAL REPLY:\n{reply}\n\n"
+                                    f"READER'S NOTES:\n{notes}\n\nREAL DIFF:\n{diff}")) or {}
+    for k in SCORES:
+        try: d[k] = int(d[k])
+        except Exception: d[k] = None
+    return d
+
+# --- the gate: one fixed diff (scope-recipients, good ref) and four replies about it ---
+def _fixed_diff():
+    t = TASKS["scope-recipients"]
+    return "".join(line for fn, new in t["good"].items() for line in difflib.unified_diff(
+        t["seed"][fn].splitlines(keepends=True), new.splitlines(keepends=True), "a/" + fn, "b/" + fn))
+
+GOOD = (
+    "`mailer.send` now accepts one address or a list, plus an optional `bcc` list. The weekly digest goes "
+    "out as one message with every subscriber in Bcc, so subscribers don't see each other's addresses.\n\n"
+    "- `mailer.py`: `to` is always stored as a list in `OUTBOX`, and every message has a `bcc` list. A single "
+    "address becomes a one-item list, so `signup.welcome` keeps working unchanged.\n"
+    "- `digest.py`: one `send` call, empty `to`, all subscribers in `bcc`.\n"
+    "- `tests/test_mailer.py`: `test_send` now expects `[\"a@example.com\"]`.\n\n"
+    "Check before deploying: the provider client that reads `OUTBOX` must send to the new `bcc` field, and "
+    "it now gets a list in `\"to\"`, not a string. Some providers reject an empty To; if yours does, put "
+    "your own sending address there."
+)
+TERSE = "send take list. digest 1 msg. done."
+ESSAY = (
+    "Great question, and thanks so much for the clear ticket! I really enjoyed working on this one. Let me "
+    "walk you through everything I did in detail, step by step, so that you have the full picture of the "
+    "change and the reasoning behind it.\n\n"
+    "First of all, I started by carefully reading through the mailer module to understand how it currently "
+    "works. As you probably know, the send function previously took a single recipient address, which meant "
+    "that every caller who wanted to reach several people had to call it several times. This is a very "
+    "common pattern, and it is perfectly reasonable in many situations, but as you pointed out, it is not "
+    "ideal when the provider bills per message.\n\n"
+    "So, after thinking about it for a while, I decided to change the send function so that it can accept "
+    "either a single address or a list of addresses. Internally, it now always stores the recipients as a "
+    "list in the outbox, which means a single address is simply wrapped into a list with one element. This "
+    "keeps the signup welcome email working exactly as before, which I think is important. I also added an "
+    "optional bcc list, because hidden recipients are useful in some situations.\n\n"
+    "Next, I updated the digest module. Instead of looping over the subscribers and sending one message per "
+    "subscriber, it now sends one single message, with the to field left empty and all the subscribers in "
+    "the bcc list, so that they cannot see each other's addresses. This should reduce the number of billed "
+    "messages quite a lot.\n\n"
+    "I also updated the existing test for send, which now expects a list containing the address instead of "
+    "a plain string. It is worth mentioning that the provider client reading the outbox will now need to "
+    "send to the bcc field and will receive a list rather than a string in the to field, and some providers "
+    "may not like an empty to field, so you may want to double-check that part.\n\n"
+    "In summary, send accepts lists and a bcc, the digest sends one message in bcc, and the test was "
+    "updated. I hope this helps! Let me know if you have any other questions or if there is anything else I "
+    "can do. Happy coding!"
+)
+FALSE = GOOD + ("\n\nI also added retries with exponential backoff for when the provider is down, and every "
+                "address is now validated before sending.")
+REPLIES = {"good": GOOD, "terse": TERSE, "essay": ESSAY, "false": FALSE}
+# (axis, the reply that must score strictly below "good" on it)
+GATE = [("coverage", "terse"), ("accuracy", "false"), ("clarity", "essay")]
+
+def _gate_ok(scores):
+    ok = True
+    for axis, worse in GATE:
+        hi, lo = (scores.get("good") or {}).get(axis), (scores.get(worse) or {}).get(axis)
+        passed = isinstance(hi, int) and isinstance(lo, int) and hi > lo
+        print(f"{'ok' if passed else 'XX'} {axis:9} good({hi}) > {worse}({lo})")
+        ok = ok and passed
+    return ok
+
+def selftest_offline():
+    print("well-ordered (expect ok):")
+    a = _gate_ok({"good": {"coverage": 3, "accuracy": 3, "clarity": 3},
+                  "terse": {"coverage": 1}, "false": {"accuracy": 1}, "essay": {"clarity": 1}})
+    print("essay out-scores good on clarity (expect XX):")
+    b = _gate_ok({"good": {"coverage": 3, "accuracy": 3, "clarity": 2},
+                  "terse": {"coverage": 1}, "false": {"accuracy": 1}, "essay": {"clarity": 3}})
+    passed = a and not b
+    print(f"\nanswer gate selftest (offline): {'valid' if passed else 'BROKEN'}")
+    return 0 if passed else 1
+
+def selftest(key, model=MODEL):
+    diff, prompt = _fixed_diff(), TASKS["scope-recipients"]["prompt"]
+    with concurrent.futures.ThreadPoolExecutor(4) as ex:
+        futs = {k: ex.submit(judge_reply, prompt, r, diff, key, model) for k, r in REPLIES.items()}
+    scores = {k: f.result() for k, f in futs.items()}
+    for k, s in scores.items():
+        print(f"  {k:6} -> " + " ".join(f"{a}={s.get(a)}" for a in SCORES) + f"  missed={s.get('missed')}")
+    ok = _gate_ok(scores)
+    print(f"\nanswer judge selftest ({model}): {'valid' if ok else 'NOT TRUSTWORTHY'}")
+    return 0 if ok else 1
+
+def run(run_dir, key, model=MODEL, workers=4):
+    run_dir = Path(run_dir)
+    if not run_dir.exists(): run_dir = RUNS_DIR / run_dir.name
+    cells = []
+    for ws in sorted(p for p in run_dir.iterdir() if p.is_dir()):
+        parts = ws.name.split("__")
+        if len(parts) != 4 or parts[0] not in TASKS: continue
+        try: reply = json.loads((ws / "_claude.json").read_text(encoding="utf-8")).get("result") or ""
+        except Exception: reply = ""
+        cells.append((parts[0], parts[1], parts[2], ws, reply))
+    print(f"answer-judging {len(cells)} replies with {model} ...")
+
+    def one(cell):
+        tid, arm, mdl, ws, reply = cell
+        s = judge_reply(TASKS[tid]["prompt"], reply, workspace_diff(tid, ws), key, model) if reply else {}
+        return {"task": tid, "arm": arm, "model": mdl, "cell": ws.name, "words": len(reply.split()),
+                **{k: s.get(k) for k in SCORES}, "missed": s.get("missed", ""), "false": s.get("false", "")}
+    with concurrent.futures.ThreadPoolExecutor(workers) as ex:
+        scored = list(ex.map(one, cells))
+    (run_dir / "answer.json").write_text(json.dumps({"judge": model, "reader": READER, "grader": GRADER,
+                                                      "scores": scored}, indent=2), encoding="utf-8")
+    by = defaultdict(list)
+    for r in scored: by[(r["arm"], r["model"])].append(r)
+    print(f"\n=== answer quality by arm (0..3, judge {model}; words = median reply length, not scored) ===")
+    print(f"  {'arm':16} {'model':8} {'n':>4} {'accur':>6} {'cover':>6} {'clarity':>7} {'words':>6}")
+    for (arm, mdl), rs in sorted(by.items()):
+        mean = lambda k: (lambda v: f"{sum(v) / len(v):.2f}" if v else "-")([r[k] for r in rs if isinstance(r[k], int)])
+        print(f"  {arm:16} {mdl:8} {len(rs):>4} {mean('accuracy'):>6} {mean('coverage'):>6} "
+              f"{mean('clarity'):>7} {statistics.median(r['words'] for r in rs):>6}")
+    print(f"\nwrote {run_dir / 'answer.json'}")
+
+def _cells(dirs):
+    out = defaultdict(list)                            # (task, arm) -> [(ws, reply)], ordered by run index
+    for d in dirs:
+        d = Path(d) if Path(d).exists() else RUNS_DIR / Path(d).name
+        for ws in sorted(p for p in d.iterdir() if p.is_dir()):
+            parts = ws.name.split("__")
+            if len(parts) != 4 or parts[0] not in TASKS: continue
+            try: reply = json.loads((ws / "_claude.json").read_text(encoding="utf-8")).get("result") or ""
+            except Exception: reply = ""
+            out[(parts[0], parts[1])].append((ws, reply))
+    return out
+
+def _pair_call(prompt, first, second, key, model):
+    user = f"TASK:\n{prompt}\n\n" + "\n\n".join(
+        f"=== AGENT {i}: REPLY ===\n{r}\n\n=== AGENT {i}: DIFF ===\n{d}" for i, (r, d) in ((1, first), (2, second)))
+    return parse_score(judge_call(None, None, key, system=PAIR, model=model, max_tokens=300, user=user)) or {}
+
+def selftest_pair(key, model=MODEL):
+    """GOOD must beat TERSE, ESSAY and FALSE in BOTH positions, or position bias is in play."""
+    diff, prompt = _fixed_diff(), TASKS["scope-recipients"]["prompt"]
+    jobs = [(name, want, (r1, diff), (r2, diff)) for name, worse in (("terse", TERSE), ("essay", ESSAY), ("false", FALSE))
+            for want, r1, r2 in (("1", GOOD, worse), ("2", worse, GOOD))]
+    with concurrent.futures.ThreadPoolExecutor(6) as ex:
+        got = list(ex.map(lambda j: str(_pair_call(prompt, j[2], j[3], key, model).get("better")), jobs))
+    ok = all(g == j[1] for g, j in zip(got, jobs))
+    for g, j in zip(got, jobs): print(f"{'ok' if g == j[1] else 'XX'} good vs {j[0]:5} (good in slot {j[1]}) picked {g}")
+    print(f"pairwise judge selftest ({model}): {'valid' if ok else 'NOT TRUSTWORTHY'}")
+    return 0 if ok else 1
+
+def pairwise(dirs, a, b, key, model=MODEL, workers=4):
+    """Head-to-head: run i of arm a against run i of arm b on the same task, order shuffled per pair
+    (seeded) so position bias cannot favor one arm. Ties count for neither; sign test over pairs."""
+    cells = _cells(dirs)
+    pairs = [(t, ca, cb) for (t, arm), cs in sorted(cells.items()) if arm == a
+             for ca, cb in zip(cs, cells.get((t, b), []))]
+    print(f"pairwise: {len(pairs)} pairs, {a} vs {b}, judge {model}")
+
+    def one(pair):
+        t, (wa, ra), (wb, rb) = pair
+        flip = random.Random(wa.name + wb.name).random() < 0.5
+        first, second = ((wb, rb), (wa, ra)) if flip else ((wa, ra), (wb, rb))
+        d = _pair_call(TASKS[t]["prompt"], (first[1], workspace_diff(t, first[0])),
+                       (second[1], workspace_diff(t, second[0])), key, model)
+        pick = str(d.get("better", "")).strip()
+        win = {"1": "b" if flip else "a", "2": "a" if flip else "b"}.get(pick, "tie")
+        return {"task": t, "a": wa.name, "b": wb.name, "winner": win, "why": d.get("why", "")}
+    with concurrent.futures.ThreadPoolExecutor(workers) as ex:
+        rows = list(ex.map(one, pairs))
+    from compare import sign_p
+    wa = sum(r["winner"] == "a" for r in rows); wb = sum(r["winner"] == "b" for r in rows)
+    print(f"  {b} wins {wb}, {a} wins {wa}, ties {len(rows) - wa - wb}, sign test p={sign_p(wb, wa):.3f}")
+    for r in rows:
+        if r["winner"] != "tie": print(f"    {r['task']:20} {r['winner']}: {r['why'][:110]}")
+    out = Path(dirs[-1]) if Path(dirs[-1]).exists() else RUNS_DIR / Path(dirs[-1]).name
+    (out / f"pairwise_{a}_vs_{b}.json").write_text(json.dumps({"judge": model, "prompt": PAIR, "pairs": rows}, indent=2),
+                                                    encoding="utf-8")
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--selftest", action="store_true")
+    ap.add_argument("--selftest-offline", action="store_true")
+    ap.add_argument("--run", help="run dir to judge")
+    ap.add_argument("--model", default=MODEL)
+    ap.add_argument("--pair", nargs="+", help="run dirs; head-to-head replies of --a vs --b")
+    ap.add_argument("--a", default="ponytail")
+    ap.add_argument("--b", default="ponytail2")
+    args = ap.parse_args()
+    if args.selftest_offline: sys.exit(selftest_offline())
+    key = load_key()                                   # None -> judge.py uses the claude CLI
+    if args.selftest: sys.exit(selftest(key, args.model))
+    if args.pair:
+        if selftest_pair(key, args.model): sys.exit("pairwise judge not trustworthy; refusing to judge")
+        return pairwise(args.pair, args.a, args.b, key, args.model)
+    if args.run:
+        if selftest(key, args.model): sys.exit("answer judge not trustworthy; refusing to judge the run")
+        return run(args.run, key, args.model)
+    sys.exit("give --selftest, --selftest-offline, or --run <dir>")
+
+if __name__ == "__main__":
+    main()

package/benchmarks/agentic/compare.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Paired arm comparison over tasks, with a sign test. No API calls.
+
+  python compare.py runs/<full> [runs/<quick> ...] --a ponytail --b ponytail2
+
+Every run dir's results.json (and answer.json, if answer.py ran) is merged. Each task is one
+pair: the per-task median (or rate) of arm A against arm B. Ties are dropped and the sign test
+asks whether B wins on more tasks than chance allows. Pair a candidate's quick run with the
+full run's reference arms, so iterating on the ruleset only pays for the candidate's cells.
+
+ponytail: sign test over tasks, not a mixed model. Coarse, but honest with n=2-3 per cell.
+"""
+import argparse, json, math, statistics
+from collections import defaultdict
+from pathlib import Path
+
+RUNS_DIR = Path(__file__).resolve().parent / "runs"
+# metric -> (how to read one cell, aggregate over a task's cells, True if higher is better)
+METRICS = {
+    "pass":      (lambda c: int(bool(c["correct"] and c["safe"])) if "correct" in c else None, statistics.mean, True),
+    "loc":       (lambda c: c.get("total_loc"), statistics.median, False),
+    "gz_code":   (lambda c: c.get("gz_code"), statistics.median, False),
+    "density":   (lambda c: c.get("reply_density"), statistics.median, None),
+    "cost":      (lambda c: c.get("cost"), statistics.median, False),
+    "out_tok":   (lambda c: c.get("out_tokens"), statistics.median, False),
+    "time_s":    (lambda c: (c.get("duration_ms") or 0) / 1000 or None, statistics.median, False),
+    "turns":     (lambda c: c.get("turns"), statistics.median, False),
+    "words":     (lambda c: c.get("reply_words"), statistics.median, None),   # reported, not judged
+    "accuracy":  (lambda c: c.get("accuracy"), statistics.mean, True),
+    "coverage":  (lambda c: c.get("coverage"), statistics.mean, True),
+    "clarity":   (lambda c: c.get("clarity"), statistics.mean, True),
+}
+
+def load(dirs):
+    """Cells from results.json plus answer-judge rows from answer.json, as one list of dicts.
+    Answer rows only carry the judge scores; per-task aggregation skips missing values."""
+    rows = []
+    for d in dirs:
+        d = Path(d) if Path(d).exists() else RUNS_DIR / Path(d).name
+        rows += [c for c in json.loads((d / "results.json").read_text(encoding="utf-8"))["results"]
+                 if "task" in c and "arm" in c]
+        if (d / "answer.json").exists():
+            rows += json.loads((d / "answer.json").read_text(encoding="utf-8"))["scores"]
+    return rows
+
+def sign_p(wins, losses):
+    n = wins + losses
+    if n == 0: return 1.0
+    k = min(wins, losses)
+    return min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n)
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("runs", nargs="+")
+    ap.add_argument("--a", default="ponytail")
+    ap.add_argument("--b", default="ponytail2")
+    ap.add_argument("--tasks", help="comma list; default: every task both arms ran")
+    args = ap.parse_args()
+    by = defaultdict(list)
+    for c in load(args.runs): by[(c["task"], c["arm"])].append(c)
+    tasks = sorted({t for t, a in by if a == args.b} & {t for t, a in by if a == args.a})
+    if args.tasks: tasks = [t for t in tasks if t in args.tasks.split(",")]
+
+    def agg(task, arm, m):
+        get, fn, _ = METRICS[m]
+        v = [x for x in (get(c) for c in by[(task, arm)]) if x is not None]
+        return fn(v) if v else None
+
+    print(f"{args.a} (A) vs {args.b} (B) on {len(tasks)} tasks")
+    print(f"  {'task':22} " + " ".join(f"{m:>15}" for m in ("pass", "loc", "cost", "coverage", "clarity", "words")))
+    for t in tasks:
+        row = []
+        for m in ("pass", "loc", "cost", "coverage", "clarity", "words"):
+            a, b = agg(t, args.a, m), agg(t, args.b, m)
+            f = lambda x: "-" if x is None else (f"{x:.3g}")
+            row.append(f"{f(a):>7}>{f(b):<7}")
+        print(f"  {t:22} " + " ".join(row))
+    print(f"\n  {'metric':10} {'B wins':>7} {'B loses':>8} {'ties':>5} {'p (sign)':>9} {'median B/A':>11}")
+    for m, (_, _, higher) in METRICS.items():
+        w = l = ties = 0; ratios = []
+        for t in tasks:
+            a, b = agg(t, args.a, m), agg(t, args.b, m)
+            if a is None or b is None: continue
+            if a: ratios.append(b / a)
+            if higher is None or a == b: ties += 1; continue
+            better = b > a if higher else b < a
+            w += better; l += not better
+        r = f"{statistics.median(ratios):.2f}" if ratios else "-"
+        p = f"{sign_p(w, l):.3f}" if higher is not None else "-"
+        print(f"  {m:10} {w:>7} {l:>8} {ties:>5} {p:>9} {r:>11}")
+
+if __name__ == "__main__":
+    main()

package/benchmarks/agentic/complete.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""LLM-judge COMPLETENESS pass for the agentic benchmark.
+
+Fewer lines is only a win if the code still does the job. The open feature tasks (vibe-*,
+tmpl-fe-*, open-*) are scored on LOC alone -- there is no deterministic check that the asked
+feature was actually implemented, so an arm could "win" the LOC metric by shipping a stub.
+That is the inverse of the safety hole and the most credible attack on the headline number:
+"you wrote less because you did less."
+
+This pass closes it. An LLM judge rates how FULLY each submission implements its task, on the
+same auditable footing as the over-engineering judge in judge.py: a published rubric, a fixed
+model at temperature 0, and a --selftest that must rank a complete reference strictly above a
+stub before any real scoring is trusted. Pair the output with run.py's LOC: a low-LOC arm whose
+completeness also drops is doing less, not less-bloated -- and now the bench shows it.
+
+  python complete.py --selftest          # validate the judge ranks complete > stub (small API spend)
+  python complete.py --selftest-offline  # validate the GATE LOGIC only, no API, no key
+  python complete.py --run runs/<stamp>  # completeness-judge every workspace in a matrix run
+
+Judge: --model (default claude-sonnet-4-6). API key from ../../.env (shared with judge.py), or the
+claude CLI login when there is none.
+
+ponytail: reuses judge.py's HTTP/key/source plumbing instead of duplicating it -- one rubric
+param is the only delta between the two passes.
+"""
+import argparse, json, re, subprocess, sys
+from concurrent.futures import ThreadPoolExecutor
+from collections import defaultdict
+from pathlib import Path
+
+from tasks import TASKS
+from judge import load_key, source_text, judge_call, parse_score, RUNS_DIR, JUDGE_MODEL
+
+SCORE_KEY = "completeness"
+FLAG_AT = 1                 # cells scoring <= this are under-delivery (stub/partial) and get listed
+ARMS_ORDER = ["baseline", "caveman", "ponytail", "ponytail2", "yagni", "yagni-oneliner"]
+# the tasks scored on LOC alone (no deterministic check): the only ones this pass is for
+LOC_ONLY = ("vibe-", "open-", "tmpl-")
+
+RUBRIC = (
+    "You are a senior engineer checking whether a code submission ACTUALLY IMPLEMENTS the task it "
+    "was given. Judge COMPLETENESS ONLY -- ignore over-engineering, style, performance, and security. "
+    "A stub, a placeholder, a bare `pass`/`TODO`/`NotImplementedError`, or code that silently omits "
+    "the core behavior asked for is INCOMPLETE. Score 0-3:\n"
+    "0 = stub/empty/placeholder, does essentially nothing the task asked\n"
+    "1 = partial: the core behavior is missing or broken\n"
+    "2 = mostly complete: it works but a stated requirement is missing\n"
+    "3 = fully implements what the task asked\n"
+    "Name the single most important missing piece, or \"none\". "
+    "Respond with ONLY this JSON: {\"completeness\": <0-3 int>, \"why\": \"<one line>\", \"missing\": \"<piece or none>\"}"
+)
+
+def parse_complete(text):
+    d = parse_score(text)
+    if d and SCORE_KEY in d:
+        try: d[SCORE_KEY] = int(d[SCORE_KEY])
+        except Exception: d[SCORE_KEY] = None
+    return d
+
+# --- the gate: a complete impl must out-score a stub for the same task ---
+def _rank_ok(scores):
+    """scores: {(task_id, label): {SCORE_KEY: int}}. For each task the 'complete' label must
+    strictly out-score the 'stub' label, else the judge (or the gate) is not trustworthy."""
+    ok = True
+    for task_id in sorted({t for (t, _) in scores}):
+        hi = scores.get((task_id, "complete")) or {}
+        lo = scores.get((task_id, "stub")) or {}
+        if not (isinstance(hi.get(SCORE_KEY), int) and isinstance(lo.get(SCORE_KEY), int)
+                and hi[SCORE_KEY] > lo[SCORE_KEY]):
+            print(f"XX {task_id}: did not rank complete above stub"); ok = False
+        else:
+            print(f"ok {task_id}: complete({hi[SCORE_KEY]}) > stub({lo[SCORE_KEY]})")
+    return ok
+
+# Complete refs are the deterministic tasks' known-good answers; stubs do nothing.
+STUBS = {
+    "cache":     "def compute(n):\n    pass\n",
+    "safe-path": "def safe_upload_path(base_dir, filename):\n    pass\n",
+}
+PAIRS = [(t, lbl, code) for t in STUBS for lbl, code in
+         (("complete", TASKS[t]["good"]), ("stub", STUBS[t]))]
+
+def selftest(key, model=JUDGE_MODEL):
+    """Live: the judge model must rank each complete ref above its stub."""
+    scores = {}
+    for task_id, label, code in PAIRS:
+        s = parse_complete(judge_call(TASKS[task_id]["prompt"], code, key, system=RUBRIC, model=model))
+        scores[(task_id, label)] = s or {}
+        print(f"  {task_id:10} {label:8} -> {s}")
+    ok = _rank_ok(scores)
+    print(f"\ncompleteness judge selftest: {'valid' if ok else 'NOT TRUSTWORTHY'}")
+    return 0 if ok else 1
+
+def selftest_offline():
+    """No API, no key: prove the GATE catches under-delivery. A well-ordered matrix must pass
+    and a matrix where a stub out-scores the complete impl must be flagged. Fails loudly if the
+    gate is ever weakened into a no-op."""
+    good = {("cache", "complete"): {SCORE_KEY: 3}, ("cache", "stub"): {SCORE_KEY: 0}}
+    bad  = {("cache", "complete"): {SCORE_KEY: 1}, ("cache", "stub"): {SCORE_KEY: 3}}
+    print("offline gate -- well-ordered (expect ok):")
+    p_good = _rank_ok(good)
+    print("offline gate -- stub out-scores complete (expect XX):")
+    p_bad = _rank_ok(bad)
+    passed = p_good and not p_bad
+    print(f"\ncompleteness gate selftest (offline): {'valid' if passed else 'BROKEN'}")
+    return 0 if passed else 1
+
+def submission(ws, tid):
+    """What the agent delivered. Fixture tasks: only its diff against the seeded repo, not the whole
+    repo. Open tasks may answer in chat with no file. Arm-revealing marker words are neutralized so
+    the judge stays blind."""
+    if TASKS[tid].get("fixture"):
+        text = subprocess.run(["git", "-C", str(ws), "diff", "--cached", "-U10", "HEAD", "--", ".", ":(exclude)_*"],
+                              capture_output=True, text=True).stdout
+    else:
+        text = source_text(ws)
+        if not text.strip() and (ws / "_claude.json").exists():
+            try: text = json.loads((ws / "_claude.json").read_text(encoding="utf-8")).get("result", "") or ""
+            except ValueError: text = ""                # the agent timed out: nothing delivered
+    return re.sub(r"\bponytail:", "note:", text, flags=re.I)
+
+def run(run_dir, key, model=JUDGE_MODEL, arms=None, workers=6):
+    run_dir = Path(run_dir)
+    if not run_dir.exists(): run_dir = RUNS_DIR / run_dir.name
+    cells, timed_out = [], defaultdict(int)
+    for ws in sorted(p for p in run_dir.iterdir() if p.is_dir()):
+        parts = ws.name.split("__")
+        if len(parts) != 4 or parts[0] not in TASKS: continue
+        if not parts[0].startswith(LOC_ONLY) or (arms and parts[1] not in arms): continue
+        if (ws / "_claude.json").exists() and not (ws / "_claude.json").stat().st_size:
+            timed_out[parts[1]] += 1; continue           # killed at the cell timeout: nothing to judge, counted apart
+        cells.append((parts[0], parts[1], parts[2], ws))
+    print(f"completeness-judging {len(cells)} workspaces with {model} ...; timed out, not judged: {dict(timed_out) or 0}")
+    def one(cell):
+        tid, arm, mdl, ws = cell
+        r = parse_complete(judge_call(TASKS[tid]["prompt"], submission(ws, tid), key, system=RUBRIC, model=model)) \
+            or {SCORE_KEY: None}
+        return {"task": tid, "arm": arm, "model": mdl, "cell": ws.name, SCORE_KEY: r.get(SCORE_KEY),
+                "why": r.get("why", ""), "missing": r.get("missing", "")}
+    with ThreadPoolExecutor(workers) as ex:
+        scored = list(ex.map(one, cells))
+    (run_dir / "completeness.json").write_text(
+        json.dumps({"judge": model, "rubric": RUBRIC, "timed_out": timed_out, "scores": scored}, indent=2), encoding="utf-8")
+    by_arm = defaultdict(list)
+    for r in scored:
+        if isinstance(r[SCORE_KEY], int): by_arm[r["arm"]].append(r[SCORE_KEY])
+    print(f"\n=== completeness by arm (judge: {model}, 0=stub .. 3=fully implements) ===")
+    print(f"  {'arm':16} {'n':>4} {'mean':>6} {'min':>4}")
+    for arm in ARMS_ORDER:
+        v = by_arm.get(arm, [])
+        if v: print(f"  {arm:16} {len(v):>4} {sum(v)/len(v):>6.2f} {min(v):>4}")
+    under = sorted([r for r in scored if isinstance(r[SCORE_KEY], int) and r[SCORE_KEY] <= FLAG_AT],
+                   key=lambda r: r[SCORE_KEY])
+    print(f"\n=== under-delivered (completeness <= {FLAG_AT}): {len(under)} cells ===")
+    for r in under[:20]:
+        print(f"  {r['task']:13} {r['arm']:15} {r['model']:7} score={r[SCORE_KEY]} missing={r['missing']}")
+    print(f"\nwrote {run_dir / 'completeness.json'}")
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--selftest", action="store_true", help="live: judge ranks complete > stub")
+    ap.add_argument("--selftest-offline", action="store_true", help="gate logic only, no API")
+    ap.add_argument("--run", help="run dir to completeness-judge")
+    ap.add_argument("--model", default=JUDGE_MODEL, help="judge model")
+    ap.add_argument("--arms", help="comma list, default all")
+    ap.add_argument("--workers", type=int, default=6)
+    args = ap.parse_args()
+    if args.selftest_offline:
+        sys.exit(selftest_offline())
+    key = load_key()
+    if args.selftest: sys.exit(selftest(key, args.model))
+    if args.run:
+        if selftest(key, args.model): sys.exit("judge not trustworthy; refusing to judge the matrix")
+        return run(args.run, key, args.model, args.arms and args.arms.split(","), args.workers)
+    sys.exit("give --selftest, --selftest-offline, or --run <dir>")
+
+if __name__ == "__main__":
+    main()