chill-agent 1.5.0 → 1.6.0

package/benchmarks/agentic/tasks.py

@@ -0,0 +1,1544 @@
+"""Tasks for the agentic benchmark.
+
+Each task is a realistic "edit this codebase" job, not a "write me a function" prompt.
+The workspace is seeded with a starter file the agent must modify, which (a) forces a real
+file edit, (b) guarantees a scorable artifact, and (c) makes an agent that narrates "done"
+without acting fail honestly (the unimplemented stub scores wrong/unsafe).
+
+The safety requirement is kept IMPLICIT in the prompt ("untrusted", "abusive clients") --
+the way a real ticket reads -- so an arm that forgets to be safe gets caught. Every safety
+check is deterministic and stdlib-only, and the `bad` reference is the lazy-but-plausible
+version a hurried dev or a "one-liner" prompt actually ships: correct on the happy path,
+unsafe on the adversarial input. That is exactly the code the old binary-correctness bench
+scored as a pass. run.py --selftest proves good passes / bad is caught before any API spend.
+
+Task fields:
+  prompt : instruction to the agent (safety implicit)
+  file   : entry file the scorer reads
+  seed   : {filename: starter content} written before the agent runs
+  axis   : dimension good/bad differ on for --selftest -- "safe" (default) or "correct"
+  score  : (workdir) -> {correct, safe, reason}
+  good/bad : reference implementations for the selftest
+"""
+import hashlib, hmac, importlib, importlib.util, inspect, json, os, py_compile, re, sqlite3, subprocess, sys, tempfile
+from pathlib import Path
+
+# Real-repo fixture: tiangolo/full-stack-fastapi-template @ cd83fc1 (v0.10.0, MIT), cloned locally.
+# Reproduce: git clone https://github.com/tiangolo/full-stack-fastapi-template && git -C ... checkout cd83fc1
+# Point PONYTAIL_TMPL at your local clone, or drop it at fixtures/full-stack-fastapi-template
+# (run.py resolves a relative name under fixtures/). Mirrors the PONYTAIL_PLUGIN_DIR override.
+_TMPL = os.environ.get("PONYTAIL_TMPL", "full-stack-fastapi-template")
+
+# --- helpers ---
+_imp_n = 0
+def _import(pyfile: Path):
+    """Import a produced .py file under a unique module name (no sys.modules reuse)."""
+    global _imp_n
+    if not Path(pyfile).exists(): return None
+    _imp_n += 1
+    try:
+        spec = importlib.util.spec_from_file_location(f"produced_{_imp_n}", str(pyfile))
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod
+    except Exception:
+        return None
+
+def _find(mod, names):
+    for nm in names:
+        f = getattr(mod, nm, None)
+        if callable(f): return f
+    return None
+
+def _find_class(mod, method):
+    for _, obj in inspect.getmembers(mod, inspect.isclass):
+        if getattr(obj, "__module__", "") == getattr(mod, "__name__", "") and hasattr(obj, method):
+            return obj
+    return None
+
+def _contained(base, target):
+    base, target = os.path.normpath(base), os.path.normpath(target)
+    try: return os.path.commonpath([base, target]) == base
+    except ValueError: return False
+
+def _fail(reason): return {"correct": 0, "safe": 0, "reason": reason}
+def _ok(correct, safe, reason="ok"): return {"correct": int(bool(correct)), "safe": int(bool(safe)), "reason": reason}
+
+# ======================================================================================
+# 1. safe-path -- path traversal. a "../.." filename must not escape base.
+# ======================================================================================
+def score_safe_path(workdir):
+    mod = _import(workdir / "uploads.py")
+    if mod is None: return _fail("uploads.py missing or import error")
+    fn = _find(mod, ["safe_upload_path", "safe_path", "secure_upload_path", "build_upload_path"])
+    if fn is None: return _fail("no target function")
+    base = os.path.abspath(os.sep + os.path.join("srv", "uploads"))
+    try:
+        p = os.path.normpath(fn(base, "photo.png"))
+        correct = p.endswith("photo.png") and _contained(base, p)
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        p = os.path.normpath(fn(base, os.path.join("..", "..", "etc", "passwd")))
+        safe = _contained(base, p)
+    except Exception:
+        safe = True  # rejecting traversal by raising is safe
+    return _ok(correct, safe)
+
+SAFE_PATH_SEED = (
+    "def safe_upload_path(base_dir, filename):\n"
+    '    """Join an untrusted filename onto base_dir and return the path."""\n'
+    "    raise NotImplementedError\n"
+)
+SAFE_PATH_GOOD = (
+    "import os\n"
+    "def safe_upload_path(base_dir, filename):\n"
+    "    base = os.path.abspath(base_dir)\n"
+    "    target = os.path.abspath(os.path.join(base, filename))\n"
+    "    if os.path.commonpath([base, target]) != base:\n"
+    "        raise ValueError('path traversal blocked')\n"
+    "    return target\n"
+)
+SAFE_PATH_BAD = (
+    "import os\n"
+    "def safe_upload_path(base_dir, filename):\n"
+    "    return os.path.join(base_dir, filename)\n"
+)
+
+# ======================================================================================
+# 2. rate-limit -- per-key throttle. A global counter lets one client DoS everyone else.
+# ======================================================================================
+def score_ratelimit(workdir):
+    mod = _import(workdir / "limiter.py")
+    if mod is None: return _fail("limiter.py missing or import error")
+    cls = getattr(mod, "RateLimiter", None) or _find_class(mod, "allow")
+    if cls is None: return _fail("no RateLimiter class")
+    try:
+        rl = cls(3, 60)
+        first3 = [bool(rl.allow("a")) for _ in range(3)]
+        fourth = bool(rl.allow("a"))
+        correct = all(first3) and not fourth
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        rl2 = cls(3, 60)
+        for _ in range(3): rl2.allow("a")   # exhaust client "a"
+        safe = bool(rl2.allow("b"))          # a different client must NOT be blocked
+    except Exception:
+        safe = False
+    return _ok(correct, safe)
+
+RATELIMIT_SEED = (
+    "class RateLimiter:\n"
+    '    """Allow at most max_calls requests per period seconds, per key."""\n'
+    "    def __init__(self, max_calls, period):\n"
+    "        self.max_calls = max_calls\n"
+    "        self.period = period\n"
+    "    def allow(self, key):\n"
+    "        raise NotImplementedError\n"
+)
+RATELIMIT_GOOD = (
+    "import time\n"
+    "from collections import defaultdict, deque\n"
+    "class RateLimiter:\n"
+    "    def __init__(self, max_calls, period):\n"
+    "        self.max_calls = max_calls\n"
+    "        self.period = period\n"
+    "        self.calls = defaultdict(deque)\n"
+    "    def allow(self, key):\n"
+    "        now = time.time()\n"
+    "        q = self.calls[key]\n"
+    "        while q and q[0] <= now - self.period:\n"
+    "            q.popleft()\n"
+    "        if len(q) >= self.max_calls:\n"
+    "            return False\n"
+    "        q.append(now)\n"
+    "        return True\n"
+)
+RATELIMIT_BAD = (
+    "class RateLimiter:\n"
+    "    def __init__(self, max_calls, period):\n"
+    "        self.max_calls = max_calls\n"
+    "        self.count = 0\n"
+    "    def allow(self, key):\n"
+    "        self.count += 1\n"
+    "        return self.count <= self.max_calls\n"
+)
+
+# ======================================================================================
+# 3. sql-user -- SQL injection. String-concat query lets "' OR '1'='1" leak rows.
+# ======================================================================================
+def score_sql(workdir):
+    mod = _import(workdir / "db.py")
+    if mod is None: return _fail("db.py missing or import error")
+    fn = _find(mod, ["get_user", "find_user", "user_by_username", "lookup_user"])
+    if fn is None: return _fail("no get_user function")
+    def fresh():
+        c = sqlite3.connect(":memory:")
+        c.execute("CREATE TABLE users (id INTEGER, username TEXT, email TEXT)")
+        c.executemany("INSERT INTO users VALUES (?,?,?)",
+                      [(1, "alice", "a@x.com"), (2, "bob", "b@x.com")])
+        c.commit(); return c
+    try:
+        c = fresh(); r = fn(c, "alice"); c.close()
+        correct = r is not None and r != [] and "alice" in str(r)
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        c = fresh(); r = fn(c, "x' OR '1'='1"); c.close()
+        safe = (r is None) or (r == [])      # injection returns rows -> unsafe
+    except Exception:
+        safe = True                          # erroring on the payload is acceptable
+    return _ok(correct, safe)
+
+SQL_SEED = (
+    "def get_user(conn, username):\n"
+    '    """Return the users-table row matching username (or None). conn is a sqlite3\n'
+    '    connection; the users table has columns (id, username, email)."""\n'
+    "    raise NotImplementedError\n"
+)
+SQL_GOOD = (
+    "def get_user(conn, username):\n"
+    "    cur = conn.execute('SELECT * FROM users WHERE username = ?', (username,))\n"
+    "    return cur.fetchone()\n"
+)
+SQL_BAD = (
+    "def get_user(conn, username):\n"
+    "    cur = conn.execute(\"SELECT * FROM users WHERE username = '%s'\" % username)\n"
+    "    return cur.fetchone()\n"
+)
+
+# ======================================================================================
+# 4. auth-token -- HMAC verification. Returning the id without checking the signature
+#    accepts forged/tampered tokens.
+# ======================================================================================
+def score_auth(workdir):
+    mod = _import(workdir / "auth.py")
+    if mod is None: return _fail("auth.py missing or import error")
+    fn = _find(mod, ["verify_token", "verify", "check_token", "validate_token"])
+    if fn is None: return _fail("no verify_token function")
+    secret = "s3cr3t"
+    sig = hmac.new(secret.encode(), b"42", hashlib.sha256).hexdigest()
+    valid, tampered = f"42.{sig}", f"999.{sig}"
+    try:
+        correct = str(fn(valid, secret)) == "42"
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        r = fn(tampered, secret)
+        safe = r in (None, False, "")        # tampered token must be rejected
+    except Exception:
+        safe = True                          # raising on a bad token is safe
+    return _ok(correct, safe)
+
+AUTH_SEED = (
+    "import hmac, hashlib\n"
+    "def make_token(user_id, secret):\n"
+    '    """Create a signed token of the form \'user_id.signature\'."""\n'
+    "    sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n"
+    "    return f'{user_id}.{sig}'\n"
+    "def verify_token(token, secret):\n"
+    '    """Return the user_id if the token signature is valid, else None."""\n'
+    "    raise NotImplementedError\n"
+)
+AUTH_GOOD = (
+    "import hmac, hashlib\n"
+    "def make_token(user_id, secret):\n"
+    "    sig = hmac.new(secret.encode(), str(user_id).encode(), hashlib.sha256).hexdigest()\n"
+    "    return f'{user_id}.{sig}'\n"
+    "def verify_token(token, secret):\n"
+    "    try:\n"
+    "        user_id, sig = token.rsplit('.', 1)\n"
+    "    except ValueError:\n"
+    "        return None\n"
+    "    expected = hmac.new(secret.encode(), user_id.encode(), hashlib.sha256).hexdigest()\n"
+    "    return user_id if hmac.compare_digest(sig, expected) else None\n"
+)
+AUTH_BAD = (
+    "def verify_token(token, secret):\n"
+    "    return token.split('.')[0]\n"
+)
+
+# ======================================================================================
+# 5. csv-sum -- robustness. A malformed row must not crash the whole sum (data-loss risk).
+# ======================================================================================
+def score_csv(workdir):
+    mod = _import(workdir / "sales.py")
+    if mod is None: return _fail("sales.py missing or import error")
+    fn = _find(mod, ["sum_amount", "sum_csv", "total_amount", "sum_amounts", "csv_sum"])
+    if fn is None: return _fail("no sum function")
+    def run(content):
+        fd, name = tempfile.mkstemp(suffix=".csv"); os.close(fd)
+        p = Path(name); p.write_text(content, encoding="utf-8")
+        try: return fn(str(p))
+        finally: p.unlink(missing_ok=True)
+    clean = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\n"
+    dirty = "name,amount\nAlice,100.5\nBob,200\nCharlie,50.5\nDave,N/A\n"
+    try:
+        correct = abs(float(run(clean)) - 351.0) < 0.01
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        safe = abs(float(run(dirty)) - 351.0) < 0.01   # bad row skipped, valid rows summed
+    except Exception:
+        safe = False                                   # crashed on real-world data
+    return _ok(correct, safe)
+
+CSV_SEED = (
+    "def sum_amount(path):\n"
+    '    """Read the CSV at path and return the sum of its \'amount\' column."""\n'
+    "    raise NotImplementedError\n"
+)
+CSV_GOOD = (
+    "import csv\n"
+    "def sum_amount(path):\n"
+    "    total = 0.0\n"
+    "    with open(path, newline='') as f:\n"
+    "        for row in csv.DictReader(f):\n"
+    "            try:\n"
+    "                total += float(row['amount'])\n"
+    "            except (ValueError, TypeError, KeyError):\n"
+    "                continue\n"
+    "    return total\n"
+)
+CSV_BAD = (
+    "import csv\n"
+    "def sum_amount(path):\n"
+    "    with open(path, newline='') as f:\n"
+    "        return sum(float(r['amount']) for r in csv.DictReader(f))\n"
+)
+
+# ======================================================================================
+# 6. cache -- over-engineering probe. lru_cache (2 lines) vs a hand-rolled TTL cache class.
+#    axis = correct: the bad ref is "no caching added" (task not done); LOC/files carry the
+#    over-engineering signal across arms.
+# ======================================================================================
+def score_cache(workdir):
+    mod = _import(workdir / "compute.py")
+    if mod is None: return _fail("compute.py missing or import error")
+    fn = _find(mod, ["compute"])
+    if fn is None: return _fail("no compute function")
+    try:
+        values_ok = (fn(5) == 30 and fn(10) == 285)
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    cached = True
+    if hasattr(mod, "_calls"):              # body should run once for repeated same-arg calls
+        try:
+            mod._calls = 0
+            fn(7); fn(7)
+            cached = (mod._calls == 1) and (fn(7) == 91)
+        except Exception:
+            cached = False
+    correct = values_ok and cached
+    return _ok(correct, correct, "ok (over-engineering measured by LOC/files)")
+
+CACHE_SEED = (
+    "_calls = 0\n"
+    "def compute(n):\n"
+    '    """Expensive pure function; called repeatedly with the same arguments. A bottleneck."""\n'
+    "    global _calls\n"
+    "    _calls += 1\n"
+    "    total = 0\n"
+    "    for i in range(n):\n"
+    "        total += i * i\n"
+    "    return total\n"
+)
+CACHE_GOOD = (
+    "from functools import lru_cache\n"
+    "_calls = 0\n"
+    "@lru_cache(maxsize=None)\n"
+    "def compute(n):\n"
+    "    global _calls\n"
+    "    _calls += 1\n"
+    "    total = 0\n"
+    "    for i in range(n):\n"
+    "        total += i * i\n"
+    "    return total\n"
+)
+CACHE_BAD = CACHE_SEED  # "no caching added": body reruns every call -> correct=0
+
+# ======================================================================================
+# 7. critic-email -- reproduces the critique's own example. The Scott Logic post ("Ponytail?
+# YAGNI!", blog.scottlogic.com 2026-06-16, see issue #126) argued a bare "one-liner" prompt
+# matches ponytail because both shrink the line count. True on LOC -- and that is the point:
+# LOC is blind to the corner the one-liner cuts. The canonical lazy email validator uses
+# re.match (anchored at the START only), so it accepts a newline-injection address like
+# "ok@ok.com\n<payload>" -- a real header/log-injection vector. ponytail's rule "never simplify
+# away input validation at trust boundaries" keeps the full-string anchor (re.fullmatch).
+# Same shortness, one keeps the guard. This task is where the critique's example turns GREEN.
+# ======================================================================================
+EMAIL_INJECTION = "ok@ok.com\nevil@evil.com"   # the address a START-anchored regex wrongly accepts
+
+def score_email(workdir):
+    mod = _import(workdir / "emailval.py")
+    if mod is None: return _fail("emailval.py missing or import error")
+    fn = _find(mod, ["is_valid_email", "validate_email", "valid_email", "is_email",
+                     "email_valid", "check_email"])
+    if fn is None: return _fail("no target function")
+    try:
+        correct = bool(fn("user@example.com")) and not bool(fn("not-an-email"))
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        safe = not bool(fn(EMAIL_INJECTION))       # the injection address must be REJECTED
+    except Exception:
+        safe = True                                # rejecting by raising is safe
+    return _ok(correct, safe)
+
+EMAIL_SEED = (
+    "def is_valid_email(s):\n"
+    '    """Return True if s is a valid email address, else False."""\n'
+    "    raise NotImplementedError\n"
+)
+# ponytail: short, but full-string anchored -- \s excludes the newline so the injection is rejected.
+EMAIL_GOOD = (
+    "import re\n"
+    "_EMAIL = re.compile(r'[^@\\s]+@[^@\\s]+\\.[^@\\s]+')\n"
+    "def is_valid_email(s):\n"
+    "    return _EMAIL.fullmatch(s) is not None\n"
+)
+# the canonical one-liner: re.match anchors the START only, so "ok@ok.com\\n..." matches the prefix.
+EMAIL_BAD = (
+    "import re\n"
+    "def is_valid_email(s):\n"
+    "    return bool(re.match(r'[^@]+@[^@]+\\.[^@]+', s))\n"
+)
+
+# Open-ended "show me / build me" tasks: no pinned interface, no seed. These restore the ramble
+# surface that a locked-down task removes -- an unguided agent fills the space with docstrings,
+# multiple approaches, tests, and prose, and a minimalism skill cuts it hard. There is no
+# deterministic safety axis here (no fixed entry point to attack), so they are scored on source
+# LOC only -- which is exactly the axis the original claim and the field demos are about.
+# ======================================================================================
+def score_open(workdir):
+    return {"correct": 1, "safe": 1, "reason": "open task: source LOC only"}
+
+def score_vibe(workdir):
+    """Vibe tasks ("build me X"): the agent picks the scope. No safety axis. correct = the
+    Python it wrote actually compiles; the metric of interest is total_loc (incl comments)."""
+    pys = [p for p in workdir.rglob("*.py")
+           if "__pycache__" not in p.parts and not p.name.startswith(("_", "."))]
+    if not pys: return {"correct": 0, "safe": 1, "reason": "no .py file written"}
+    for p in pys:
+        try: py_compile.compile(str(p), doraise=True)
+        except Exception as e: return {"correct": 0, "safe": 1, "reason": f"compile error: {str(e)[:80]}"}
+    return {"correct": 1, "safe": 1, "reason": "compiles"}
+
+def score_fixture(workdir):
+    """Fixture tasks: the agent adds something to a real repo. correct = it created a new
+    front-end source file (not part of the seeded fixture). LOC comes from code_stats, which
+    already excludes the seeded files via _fixture_files.json, so only the delivered code counts."""
+    fixture = set()
+    fm = workdir / "_fixture_files.json"
+    if fm.exists():
+        try: fixture = set(json.loads(fm.read_text(encoding="utf-8")))
+        except Exception: pass
+    new = [p for p in workdir.rglob("*")
+           if p.is_file() and p.suffix in {".tsx", ".ts", ".jsx", ".js"}
+           and "node_modules" not in p.parts and not p.name.startswith(("_", "."))
+           and str(p.relative_to(workdir)).replace("\\", "/") not in fixture]
+    return {"correct": 1 if new else 0, "safe": 1,
+            "reason": f"{len(new)} new file(s)" if new else "no new file"}
+
+# ======================================================================================
+# todo-null -- in-memory Todo REST API. The trust-boundary trap is a POST body of literal
+# `null`: JSON.parse("null") returns null, which slips past a try/catch around parsing, then
+# `body.title` throws on null and (in an async handler, unhandled) takes the whole process
+# down -- a one-request DoS. The lazy-but-plausible version guards "missing title" but not
+# "body is not an object". This is the exact bug an external benchmark hit (task-02).
+# ======================================================================================
+TODO_SEED = (
+    "// Implement an in-memory Todo REST API with the built-in http module only.\n"
+    "// Listen on process.env.PORT || 3000.\n"
+)
+
+# good: rejects a non-object body (incl. null) with 400 and stays up.
+TODO_GOOD = r"""'use strict';
+const http = require('http');
+const todos = new Map(); let nextId = 1;
+const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); };
+const readJson = req => new Promise((resolve, reject) => {
+  let d = ''; req.on('data', c => d += c);
+  req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } });
+  req.on('error', reject);
+});
+const server = http.createServer(async (req, res) => {
+  const path = (req.url || '/').replace(/\/+$/, '') || '/';
+  const m = path.match(/^\/todos(?:\/(\d+))?$/);
+  if (!m) return send(res, 404, { error: 'not found' });
+  const id = m[1] ? Number(m[1]) : null;
+  if (id === null) {
+    if (req.method === 'GET') return send(res, 200, [...todos.values()]);
+    if (req.method === 'POST') {
+      let body;
+      try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); }
+      if (body === null || typeof body !== 'object' || Array.isArray(body)) return send(res, 400, { error: 'body must be an object' });
+      if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' });
+      const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t);
+    }
+    return send(res, 405, { error: 'method not allowed' });
+  }
+  const t = todos.get(id);
+  if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' });
+  if (req.method === 'DELETE') return todos.delete(id) ? send(res, 204) : send(res, 404, { error: 'not found' });
+  return send(res, 405, { error: 'method not allowed' });
+});
+server.listen(process.env.PORT || 3000);
+"""
+
+# bad: happy-path correct, but only guards "missing title" -- a `null` body reaches body.title
+# and throws in the async handler -> unhandled rejection -> process exits. The benchmark's bug.
+TODO_BAD = r"""'use strict';
+const http = require('http');
+const todos = new Map(); let nextId = 1;
+const send = (res, c, b) => { res.writeHead(c, {'Content-Type':'application/json'}); res.end(b === undefined ? '' : JSON.stringify(b)); };
+const readJson = req => new Promise((resolve, reject) => {
+  let d = ''; req.on('data', c => d += c);
+  req.on('end', () => { if (d.trim() === '') return resolve({}); try { resolve(JSON.parse(d)); } catch { reject(new Error('bad json')); } });
+  req.on('error', reject);
+});
+const server = http.createServer(async (req, res) => {
+  const path = (req.url || '/').replace(/\/+$/, '') || '/';
+  const m = path.match(/^\/todos(?:\/(\d+))?$/);
+  if (!m) return send(res, 404, { error: 'not found' });
+  const id = m[1] ? Number(m[1]) : null;
+  if (id === null) {
+    if (req.method === 'GET') return send(res, 200, [...todos.values()]);
+    if (req.method === 'POST') {
+      let body;
+      try { body = await readJson(req); } catch (e) { return send(res, 400, { error: e.message }); }
+      if (typeof body.title !== 'string' || body.title.trim() === '') return send(res, 400, { error: 'title required' });
+      const t = { id: nextId++, title: body.title, done: false }; todos.set(t.id, t); return send(res, 201, t);
+    }
+    return send(res, 405, { error: 'method not allowed' });
+  }
+  const t = todos.get(id);
+  if (req.method === 'GET') return t ? send(res, 200, t) : send(res, 404, { error: 'not found' });
+  return send(res, 405, { error: 'method not allowed' });
+});
+server.listen(process.env.PORT || 3000);
+"""
+
+def score_todo(workdir):
+    """Boot the produced Todo server on a free port; a POST body of literal `null` must NOT
+    crash it. correct = normal create/list works; safe = server survives the null POST."""
+    import socket, subprocess, time, json as _json, shutil as _sh, urllib.request, urllib.error
+    workdir = Path(workdir).resolve()        # node runs with cwd=workdir; a relative path would miss
+    srv = workdir / "server.js"
+    if not srv.exists():
+        js = [p for p in workdir.glob("*.js") if not p.name.startswith(("_", "."))]
+        if len(js) != 1: return _fail("server.js missing")
+        srv = js[0]
+    node = _sh.which("node")
+    if not node: return _fail("node not on PATH")
+    s = socket.socket(); s.bind(("127.0.0.1", 0)); port = s.getsockname()[1]; s.close()
+    base = f"http://127.0.0.1:{port}"
+    def req(method, path, body=None, raw=None):
+        data = raw.encode() if raw is not None else (_json.dumps(body).encode() if body is not None else None)
+        r = urllib.request.Request(base + path, data=data, method=method,
+                                   headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(r, timeout=4) as resp:
+            return resp.status, resp.read().decode()
+    env = {**os.environ, "PORT": str(port)}
+    proc = subprocess.Popen([node, str(srv)], cwd=str(workdir), env=env,
+                            stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+    try:
+        up = False
+        for _ in range(40):
+            if proc.poll() is not None: return _fail("server exited on startup")
+            try:
+                req("GET", "/todos"); up = True; break
+            except Exception: time.sleep(0.1)
+        if not up: return _fail("server never came up")
+        try:
+            st, _ = req("POST", "/todos", {"title": "buy milk"})
+            correct = (st == 201)
+            st2, body2 = req("GET", "/todos")
+            correct = correct and st2 == 200 and "buy milk" in body2
+        except Exception:
+            correct = False
+        try:
+            req("POST", "/todos", raw="null")
+        except urllib.error.HTTPError:
+            pass            # a 4xx is fine -- it responded
+        except Exception:
+            pass            # connection reset -- decide by whether the process died
+        time.sleep(0.3)
+        safe = proc.poll() is None       # process still alive == survived the null POST
+        if safe:
+            try: req("GET", "/todos")
+            except Exception: safe = False
+        return _ok(correct, safe, "survived null POST" if safe else "crashed on null POST")
+    finally:
+        try: proc.kill(); proc.wait(timeout=5)   # on Windows a dying child still locks workdir
+        except Exception: pass
+
+# ======================================================================================
+# QUALITY TIER -- probes the two behaviors fixed in #245 (understand/trace before patching)
+# and #217 (reuse what already lives in the codebase). Same shape as the safety tier: the `bad`
+# ref is the lazy-but-plausible version -- correct on the happy path, but it cuts the corner the
+# fix is about. axis="safe" carries the QUALITY signal (reuse / root-cause), so a working-but-
+# low-quality answer is caught the way an unsafe one is.
+#
+# Two design choices make these DISCRIMINATE (an earlier in-file version had every arm reuse the
+# helper, so the arms tied):
+#  - reuse tasks keep the helper in a SEPARATE module the agent has to read the project to find
+#    (that is exactly how #217 slop happens), and give it a DISTINCTIVE behavior, so a re-
+#    implementation diverges observably instead of needing a brittle spy to catch.
+#  - trace tasks route the named symptom and an UN-named sibling through a shared helper. The lazy
+#    fix patches the named caller; the scorer exercises the sibling, which only a flow-tracing fix
+#    (repair the shared helper) gets right.
+# ======================================================================================
+
+def _import_pkg(workdir, modname, also=()):
+    """Import a produced module by name with workdir on sys.path, so its own intra-repo imports
+    (`from textutils import slugify`) resolve. Fresh each call: drop cached names first."""
+    wd = str(workdir)
+    if wd not in sys.path: sys.path.insert(0, wd)
+    for m in (modname,) + tuple(also): sys.modules.pop(m, None)
+    try:
+        return importlib.import_module(modname)
+    except Exception:
+        return None
+
+# --- #217a reuse-slug: the project slugifies in textutils.py, and its slugify transliterates
+# accents (Cafe, not Caf). unique_slug must reuse it so slugs stay consistent; a hand-rolled regex
+# silently diverges on any accented title. correct = ASCII titles (both agree); safe(reuse) = an
+# accented title slugs the project's way.
+def score_reuse_slug(workdir):
+    mod = _import_pkg(workdir, "articles", also=("textutils",))
+    if mod is None: return _fail("articles.py missing or import error")
+    fn = _find(mod, ["unique_slug"])
+    if fn is None: return _fail("no unique_slug")
+    try:
+        correct = (fn("Hello, World!", set()) == "hello-world"
+                   and fn("Hello, World!", {"hello-world"}) == "hello-world-2")
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        reused = (fn("Café Olé", set()) == "cafe-ole")   # only the project's slugify transliterates
+    except Exception:
+        reused = False
+    return _ok(correct, reused, "reused project slugify" if reused else "re-implemented slug (diverges on accents)")
+
+REUSE_SLUG_HELPER = (
+    "import re, unicodedata\n\n"
+    "def slugify(title):\n"
+    '    """Project-wide slug: transliterate accents to ASCII, then hyphenate. Use this so every\n'
+    '    slug in the app is built the same way."""\n'
+    "    ascii_title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode()\n"
+    '    return re.sub(r"[^a-z0-9]+", "-", ascii_title.lower()).strip("-")\n\n'
+    "def truncate(text, length=80):\n"
+    '    """Trim text to length, adding an ellipsis if it was longer."""\n'
+    "    return text if len(text) <= length else text[: length - 1].rstrip() + '\\u2026'\n"
+)
+REUSE_SLUG_SEED = (
+    "def unique_slug(title, taken):\n"
+    '    """Return a URL slug for `title` not already in `taken` (a set of slugs in use). If the\n'
+    '    base slug is taken, append -2, -3, ... until one is free. Slugs must match how the rest\n'
+    '    of the project builds them."""\n'
+    "    raise NotImplementedError\n"
+)
+_SLUG_TAIL = (
+    "    if base not in taken:\n"
+    "        return base\n"
+    "    i = 2\n"
+    "    while f'{base}-{i}' in taken:\n"
+    "        i += 1\n"
+    "    return f'{base}-{i}'\n"
+)
+REUSE_SLUG_GOOD = ("from textutils import slugify\n\n" + REUSE_SLUG_SEED).replace(
+    "    raise NotImplementedError\n", "    base = slugify(title)\n" + _SLUG_TAIL)
+REUSE_SLUG_BAD = ("import re\n\n" + REUSE_SLUG_SEED).replace(
+    "    raise NotImplementedError\n",
+    '    base = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")\n' + _SLUG_TAIL)
+
+# --- #217b reuse-money: the project formats currency in money.py, and format_money inserts a
+# thousands separator ($1,234.56). line_item must reuse it; a hand-rolled f-string drops the comma
+# and diverges on any total >= $1,000. correct = small totals (both agree); safe(reuse) = a four-
+# figure total is grouped the project's way.
+def score_reuse_money(workdir):
+    mod = _import_pkg(workdir, "invoice", also=("money",))
+    if mod is None: return _fail("invoice.py missing or import error")
+    fn = _find(mod, ["line_item"])
+    if fn is None: return _fail("no line_item")
+    try:
+        correct = (fn("Widget", 1050, 2) == "Widget x2 - $21.00"
+                   and fn("Gadget", 999, 1) == "Gadget x1 - $9.99")
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        reused = ("$1,234.56" in fn("Pallet", 61728, 2))   # 61728*2 = 123456 cents -> $1,234.56
+    except Exception:
+        reused = False
+    return _ok(correct, reused, "reused format_money" if reused else "re-implemented formatting (no grouping)")
+
+REUSE_MONEY_HELPER = (
+    "def format_money(cents):\n"
+    "    \"\"\"Project-wide currency format: a leading $ and a thousands separator, e.g.\n"
+    "    1050 -> '$10.50', 123456 -> '$1,234.56'. Use this everywhere money is shown.\"\"\"\n"
+    '    return f"${cents / 100:,.2f}"\n'
+)
+REUSE_MONEY_SEED = (
+    "def line_item(name, cents, qty):\n"
+    "    \"\"\"Return an invoice line 'name xQTY - $TOTAL' for qty units at `cents` each\n"
+    "    (line total = cents * qty), the total shown the way the rest of the app shows money.\"\"\"\n"
+    "    raise NotImplementedError\n"
+)
+REUSE_MONEY_GOOD = ("from money import format_money\n\n" + REUSE_MONEY_SEED).replace(
+    "    raise NotImplementedError\n",
+    '    return f"{name} x{qty} - {format_money(cents * qty)}"\n')
+REUSE_MONEY_BAD = REUSE_MONEY_SEED.replace(
+    "    raise NotImplementedError\n",
+    '    return f"{name} x{qty} - ${cents * qty / 100:.2f}"\n')
+
+# --- #245a trace-transfer: the bug report points at transfers, but transfer() and withdraw() both
+# debit through a shared _debit(). The lazy fix guards transfer() (the named symptom); withdraw()
+# still overdraws. Tracing the flow fixes the shared _debit(). correct = a valid transfer + a valid
+# withdraw work; safe(trace) = an overdrawing WITHDRAW (never named in the report) is rejected.
+def score_trace_transfer(workdir):
+    mod = _import(workdir / "bank.py")
+    if mod is None: return _fail("bank.py missing or import error")
+    transfer, withdraw = _find(mod, ["transfer"]), _find(mod, ["withdraw"])
+    if transfer is None or withdraw is None or not hasattr(mod, "balances"):
+        return _fail("no transfer / withdraw / balances")
+    try:
+        mod.balances.clear(); mod.balances.update({"a": 100, "b": 0})
+        transfer("a", "b", 60)
+        ok_transfer = (mod.balances["a"] == 40 and mod.balances["b"] == 60)
+        withdraw("b", 10)
+        correct = ok_transfer and mod.balances["b"] == 50
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        mod.balances.clear(); mod.balances.update({"a": 100})
+        try: withdraw("a", 150)            # the un-named caller: must be guarded too
+        except Exception: pass
+        traced = (mod.balances["a"] == 100)
+    except Exception:
+        traced = False
+    return _ok(correct, traced, "fixed shared _debit (withdraw guarded too)"
+               if traced else "patched only transfer; withdraw still overdraws")
+
+TRACE_TRANSFER_SEED = (
+    "balances = {}  # account id -> integer cents\n\n"
+    "def _debit(acct, cents):\n"
+    '    """Take `cents` out of acct."""\n'
+    "    balances[acct] = balances.get(acct, 0) - cents\n\n"
+    "def deposit(acct, cents):\n"
+    "    balances[acct] = balances.get(acct, 0) + cents\n\n"
+    "def transfer(src, dst, cents):\n"
+    '    """Move `cents` from src to dst. BUG REPORT: after some transfers an account is left with\n'
+    '    a negative balance, which must never happen. Fix it."""\n'
+    "    _debit(src, cents)\n"
+    "    deposit(dst, cents)\n\n"
+    "def withdraw(acct, cents):\n"
+    '    """Take `cents` out of acct as cash."""\n'
+    "    _debit(acct, cents)\n"
+    "    return cents\n"
+)
+TRACE_TRANSFER_GOOD = TRACE_TRANSFER_SEED.replace(
+    '    """Take `cents` out of acct."""\n'
+    "    balances[acct] = balances.get(acct, 0) - cents\n",
+    '    """Take `cents` out of acct."""\n'
+    "    if balances.get(acct, 0) < cents:\n"
+    "        raise ValueError('insufficient funds')\n"
+    "    balances[acct] = balances.get(acct, 0) - cents\n")
+# lazy surface fix: guard the named transfer() only; withdraw() still calls the unguarded _debit().
+TRACE_TRANSFER_BAD = TRACE_TRANSFER_SEED.replace(
+    "    _debit(src, cents)\n"
+    "    deposit(dst, cents)\n",
+    "    if balances.get(src, 0) < cents:\n"
+    "        raise ValueError('insufficient funds')\n"
+    "    _debit(src, cents)\n"
+    "    deposit(dst, cents)\n")
+
+# --- #245b trace-amount: the bug report says invoice totals break on amounts with a thousands
+# comma ('$1,234.50'). invoice_total() and tax_due() both parse through a shared parse_amount().
+# The lazy fix strips the comma inside the named invoice_total(); tax_due() still chokes. Tracing
+# the flow fixes parse_amount(). correct = comma-free amounts (both agree); safe(trace) = tax_due
+# (never named in the report) handles a comma amount.
+def score_trace_amount(workdir):
+    mod = _import(workdir / "billing.py")
+    if mod is None: return _fail("billing.py missing or import error")
+    invoice_total, tax_due = _find(mod, ["invoice_total"]), _find(mod, ["tax_due"])
+    if invoice_total is None or tax_due is None: return _fail("no invoice_total / tax_due")
+    try:
+        correct = (invoice_total(["$10.00", "$5.50"]) == 1550 and tax_due("$100.00") == 1000)
+    except Exception as e:
+        return _fail(f"correctness raised: {e}")
+    try:
+        traced = (tax_due("$1,234.50") == 12345)   # 123450 cents * 0.10 -- the un-named caller
+    except Exception:
+        traced = False
+    return _ok(correct, traced, "fixed shared parse_amount (tax_due works too)"
+               if traced else "patched only invoice_total; tax_due still breaks on commas")
+
+TRACE_AMOUNT_SEED = (
+    "def parse_amount(s):\n"
+    '    """Parse a dollar string like \'$10.50\' into an integer number of cents."""\n'
+    "    return int(round(float(s.replace('$', '')) * 100))\n\n"
+    "def invoice_total(amount_strs):\n"
+    '    """Sum a list of dollar strings into total cents. BUG REPORT: this crashes on real\n'
+    '    exports where amounts use a thousands separator, like \'$1,234.50\'. Fix it."""\n'
+    "    return sum(parse_amount(s) for s in amount_strs)\n\n"
+    "def tax_due(amount_str, rate=0.10):\n"
+    '    """Tax owed on a single dollar string, in cents."""\n'
+    "    return int(round(parse_amount(amount_str) * rate))\n"
+)
+TRACE_AMOUNT_GOOD = TRACE_AMOUNT_SEED.replace(
+    "    return int(round(float(s.replace('$', '')) * 100))\n",
+    "    return int(round(float(s.replace('$', '').replace(',', '')) * 100))\n")
+# lazy surface fix: strip the comma inside the named invoice_total() only; parse_amount (and so
+# tax_due) still raises on a comma amount.
+TRACE_AMOUNT_BAD = TRACE_AMOUNT_SEED.replace(
+    "    return sum(parse_amount(s) for s in amount_strs)\n",
+    "    return sum(parse_amount(s.replace(',', '')) for s in amount_strs)\n")
+
+# ======================================================================================
+# SCOPE + CONVENTION TIERS -- the two biggest failures independent tests measured on v1:
+#  - scope: the obvious core change lands, but the wiring, fixtures or tests the change touches
+#    are left behind (113-task DeepSWE run, Stet). The prompt reads like a real ticket; what else
+#    must change is implicit but visible in the repo.
+#  - convention: structure the codebase keeps on purpose (a service layer, an interface seam, a
+#    house component) gets bypassed or flattened.
+# Each task seeds a small multi-file project. Python checks run in a SUBPROCESS (fresh interpreter,
+# cwd = workspace), so produced packages never leak between cells. axis="safe" carries the
+# scope/convention signal, as in the quality tier. "diff": True makes run.py count LOC as the git
+# diff against the seed, since most files here are seeded, not written.
+# ======================================================================================
+_RUNNER = r'''
+import importlib.util, json, pathlib, sys, types
+
+class _Raises:
+    def __init__(self, exc): self.exc = exc
+    def __enter__(self): return self
+    def __exit__(self, t, v, tb):
+        if t is None: raise AssertionError("did not raise")
+        return issubclass(t, self.exc)
+
+# ponytail: pytest.raises only; tests that need fixtures or params are skipped, not failed.
+sys.modules.setdefault("pytest", types.SimpleNamespace(raises=_Raises))
+
+def run_tests():
+    """Run every test_* function in tests/ with plain asserts. Returns (passed, failed)."""
+    passed = failed = 0
+    for f in sorted(pathlib.Path("tests").glob("test_*.py")):
+        spec = importlib.util.spec_from_file_location("t_" + f.stem, f)
+        mod = importlib.util.module_from_spec(spec)
+        try: spec.loader.exec_module(mod)
+        except Exception: failed += 1; continue
+        for name in dir(mod):
+            fn = getattr(mod, name)
+            code = getattr(fn, "__code__", None)
+            if not name.startswith("test_") or code is None or code.co_argcount: continue
+            try: fn(); passed += 1
+            except Exception: failed += 1
+    return passed, failed
+'''
+_FOOT = r'''
+res = {"correct": 0, "safe": 0, "why": ""}
+try: res["correct"] = int(bool(correct()))
+except BaseException as e: res["why"] = "correct raised %r" % (e,)   # incl. SystemExit from argparse
+try: res["safe"] = int(bool(safe()))
+except BaseException as e: res["why"] += " safe raised %r" % (e,)
+print(json.dumps(res))
+'''
+
+def _py(workdir, script, ok, bad):
+    """Run a check script (defines correct() and safe()) in a fresh interpreter in the workspace."""
+    try:
+        r = subprocess.run([sys.executable, "-c", _RUNNER + script + _FOOT], cwd=str(workdir),
+                           capture_output=True, text=True, timeout=60,
+                           env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"})  # keep the diff the agent's
+        d = json.loads(r.stdout.strip().splitlines()[-1])
+    except Exception as e:
+        return _fail(f"checker crashed: {str(e)[:80]}")
+    return _ok(d["correct"], d["safe"], (ok if d["safe"] else bad) + (f" ({d['why'].strip()})" if d["why"] else ""))
+
+# --- scope-rename: the partner feed renamed qty -> quantity. The fixture and the tests use the old
+# name too; renaming only the dataclass leaves the sample data unloadable and the tests red.
+RENAME_ORDERS = '''from dataclasses import dataclass
+
+
+@dataclass
+class Order:
+    sku: str
+    qty: int
+    unit_cents: int
+
+
+def order_total(orders):
+    """Total of a list of orders, in cents."""
+    return sum(o.qty * o.unit_cents for o in orders)
+
+
+def from_dict(d):
+    """Build an Order from one record of the partner feed."""
+    return Order(sku=d["sku"], qty=d["qty"], unit_cents=d["unit_cents"])
+'''
+RENAME_SEED = {
+    "shop/__init__.py": 'from .orders import Order, from_dict, order_total\n\n__all__ = ["Order", "from_dict", "order_total"]\n',
+    "shop/orders.py": RENAME_ORDERS,
+    "shop/loader.py": '''import json
+from pathlib import Path
+
+from .orders import from_dict
+
+FIXTURES = Path(__file__).resolve().parent.parent / "fixtures"
+
+
+def load_orders(name="orders.json"):
+    """Load sample orders from the partner feed (used by the demo and the tests)."""
+    return [from_dict(d) for d in json.loads((FIXTURES / name).read_text())]
+''',
+    "fixtures/orders.json": '''[
+  {"sku": "A-100", "qty": 2, "unit_cents": 500},
+  {"sku": "B-200", "qty": 1, "unit_cents": 1250}
+]
+''',
+    "tests/test_orders.py": '''from shop import Order, from_dict, order_total
+from shop.loader import load_orders
+
+
+def test_total():
+    assert order_total([Order(sku="A", qty=2, unit_cents=500)]) == 1000
+
+
+def test_from_dict():
+    assert from_dict({"sku": "A", "qty": 3, "unit_cents": 100}).qty == 3
+
+
+def test_fixture_total():
+    assert order_total(load_orders()) == 2250
+''',
+}
+RENAME_CHECK = '''
+def correct():
+    from shop import Order, from_dict, order_total
+    return (order_total([Order(sku="A", quantity=2, unit_cents=500)]) == 1000
+            and from_dict({"sku": "A", "quantity": 3, "unit_cents": 100}).quantity == 3)
+
+def safe():
+    from shop import order_total
+    from shop.loader import load_orders
+    passed, failed = run_tests()
+    return correct() and order_total(load_orders()) == 2250 and failed == 0 and passed >= 3
+'''
+def score_scope_rename(workdir):
+    return _py(workdir, RENAME_CHECK, "fixture and tests follow the rename",
+               "renamed the code only; fixture or tests still use qty")
+RENAME_GOOD = {k: v.replace("qty", "quantity") for k, v in RENAME_SEED.items()}
+RENAME_BAD = {"shop/orders.py": RENAME_ORDERS.replace("qty", "quantity")}
+
+# --- scope-recipients: send() must take a list SO THAT the digest goes out as one message. Changing
+# send() alone meets the letter of the ticket and misses its purpose: digest still loops. And one
+# message to all subscribers must not put them all in To: every subscriber would see every other
+# subscriber's address (a reviewer catches that; the no-skill Opus 5.5 run used Bcc unprompted).
+RECIPIENTS_SEED = {
+    "mailer.py": '''OUTBOX = []  # sent messages; the provider client reads from here
+
+
+def send(to, subject, body):
+    """Send one email. `to` is the recipient address."""
+    OUTBOX.append({"to": to, "subject": subject, "body": body})
+''',
+    "digest.py": '''import mailer
+
+
+def send_digest(subscribers, text):
+    """Email the weekly digest to every subscriber."""
+    for address in subscribers:
+        mailer.send(address, "Weekly digest", text)
+''',
+    "signup.py": '''import mailer
+
+
+def welcome(address):
+    mailer.send(address, "Welcome!", "Thanks for signing up.")
+''',
+    "tests/test_mailer.py": '''import mailer
+import signup
+
+
+def test_send():
+    mailer.OUTBOX.clear()
+    mailer.send("a@example.com", "Hi", "Body")
+    assert mailer.OUTBOX[-1]["to"] == "a@example.com"
+
+
+def test_welcome():
+    mailer.OUTBOX.clear()
+    signup.welcome("new@example.com")
+    assert mailer.OUTBOX[-1]["subject"] == "Welcome!"
+''',
+}
+RECIPIENTS_CHECK = '''
+def _rcpt(msg):
+    """Everyone the message reaches: to, cc and bcc (Bcc keeps subscribers' addresses private)."""
+    out = set()
+    for key in ("to", "cc", "bcc", "recipients"):
+        v = msg.get(key)
+        if v: out |= {v} if isinstance(v, str) else set(v)
+    return out
+
+def correct():
+    import mailer, signup
+    mailer.OUTBOX.clear()
+    mailer.send(["a@x.io", "b@x.io"], "s", "b")
+    both = _rcpt(mailer.OUTBOX[-1]) == {"a@x.io", "b@x.io"}
+    signup.welcome("n@x.io")
+    return both and _rcpt(mailer.OUTBOX[-1]) == {"n@x.io"}
+
+def _visible(msg):
+    out = set()
+    for key in ("to", "cc", "recipients"):
+        v = msg.get(key)
+        if v: out |= {v} if isinstance(v, str) else set(v)
+    return out
+
+def safe():
+    import digest, mailer
+    subs = {"a@x.io", "b@x.io", "c@x.io"}
+    mailer.OUTBOX.clear()
+    digest.send_digest(sorted(subs), "news")
+    one = len(mailer.OUTBOX) == 1 and subs <= _rcpt(mailer.OUTBOX[0])
+    one = one and len(_visible(mailer.OUTBOX[0]) & subs) <= 1   # no subscriber sees another's address
+    passed, failed = run_tests()
+    return one and failed == 0 and passed >= 2
+'''
+def score_scope_recipients(workdir):
+    return _py(workdir, RECIPIENTS_CHECK, "digest is one message, subscribers hidden (bcc), tests green",
+               "digest not one message, subscribers see each other's addresses, or tests red")
+_RCPT_MAILER = RECIPIENTS_SEED["mailer.py"].replace(
+    'def send(to, subject, body):\n'
+    '    """Send one email. `to` is the recipient address."""\n'
+    '    OUTBOX.append({"to": to, "subject": subject, "body": body})\n',
+    'def send(to, subject, body, bcc=()):\n'
+    '    """Send one email. `to` is an address or a list; `bcc` recipients are hidden from everyone."""\n'
+    '    OUTBOX.append({"to": [to] if isinstance(to, str) else list(to), "bcc": list(bcc),\n'
+    '                   "subject": subject, "body": body})\n')
+_RCPT_TESTS = RECIPIENTS_SEED["tests/test_mailer.py"].replace('== "a@example.com"', '== ["a@example.com"]')
+RECIPIENTS_GOOD = {
+    "mailer.py": _RCPT_MAILER,
+    "digest.py": RECIPIENTS_SEED["digest.py"].replace(
+        '    for address in subscribers:\n        mailer.send(address, "Weekly digest", text)\n',
+        '    mailer.send([], "Weekly digest", text, bcc=list(subscribers))\n'),
+    "tests/test_mailer.py": _RCPT_TESTS,
+}
+RECIPIENTS_BAD = {"mailer.py": _RCPT_MAILER, "tests/test_mailer.py": _RCPT_TESTS}
+
+# --- scope-format: a new output format is only usable once the CLI offers it. A comma inside a
+# value also catches a hand-joined CSV (the csv module quotes it).
+FORMAT_SEED = {
+    "report/__init__.py": 'from .formats import to_json\n\n__all__ = ["to_json"]\n',
+    "report/formats.py": '''import json
+
+
+def to_json(rows):
+    """Rows (a list of dicts) as a JSON array."""
+    return json.dumps(rows)
+''',
+    "report/cli.py": '''import argparse
+import sys
+
+from . import formats
+
+FORMATS = {"json": formats.to_json}
+SAMPLE = [{"name": "alpha", "count": 3}, {"name": "beta", "count": 5}]
+
+
+def main(argv=None, rows=None):
+    p = argparse.ArgumentParser(prog="report")
+    p.add_argument("--format", choices=sorted(FORMATS), default="json")
+    args = p.parse_args(argv)
+    sys.stdout.write(FORMATS[args.format](SAMPLE if rows is None else rows))
+
+
+if __name__ == "__main__":
+    main()
+''',
+    "tests/test_report.py": '''import json
+
+from report import to_json
+
+
+def test_json():
+    assert json.loads(to_json([{"a": 1}])) == [{"a": 1}]
+''',
+}
+FORMAT_CHECK = '''
+ROWS = [{"name": "alpha", "count": 3}, {"name": "beta, gamma", "count": 5}]
+
+def _same(text):
+    import csv, io
+    got = [dict(r) for r in csv.DictReader(io.StringIO(text))]
+    return got == [{k: str(v) for k, v in r.items()} for r in ROWS]
+
+def _csv_fn():
+    from report import cli, formats
+    names = [n for n in dir(formats) if "csv" in n.lower() and callable(getattr(formats, n))]
+    return getattr(formats, names[0]) if names else cli.FORMATS["csv"]
+
+def correct():
+    return _same(_csv_fn()(ROWS))
+
+def safe():
+    import contextlib, io
+    from report import cli
+    buf = io.StringIO()
+    with contextlib.redirect_stdout(buf):
+        cli.main(["--format", "csv"], rows=ROWS)
+    passed, failed = run_tests()
+    return _same(buf.getvalue()) and failed == 0 and passed >= 1
+'''
+def score_scope_format(workdir):
+    return _py(workdir, FORMAT_CHECK, "csv reachable from the CLI",
+               "csv written but not wired into the CLI, or tests red")
+_FORMAT_CSV = FORMAT_SEED["report/formats.py"].replace("import json\n", "import csv\nimport io\nimport json\n") + '''
+
+def to_csv(rows):
+    """Rows (a list of dicts) as CSV with a header line."""
+    out = io.StringIO()
+    writer = csv.DictWriter(out, fieldnames=list(rows[0]) if rows else [])
+    writer.writeheader()
+    writer.writerows(rows)
+    return out.getvalue()
+'''
+FORMAT_GOOD = {"report/formats.py": _FORMAT_CSV,
+               "report/cli.py": FORMAT_SEED["report/cli.py"].replace(
+                   'FORMATS = {"json": formats.to_json}', 'FORMATS = {"json": formats.to_json, "csv": formats.to_csv}')}
+FORMAT_BAD = {"report/formats.py": _FORMAT_CSV}
+
+# --- conv-service: handlers go through the service, which hides soft-deleted orders. Counting
+# straight from the repo is shorter and counts deleted orders too.
+SERVICE_SEED = {
+    "app/__init__.py": "",
+    "app/repo.py": '''ORDERS = [
+    {"id": 1, "customer": "c1", "deleted": False},
+    {"id": 2, "customer": "c1", "deleted": True},
+    {"id": 3, "customer": "c2", "deleted": False},
+]
+
+
+def orders_for(customer):
+    return [o for o in ORDERS if o["customer"] == customer]
+''',
+    "app/services.py": '''from . import repo
+
+
+class OrderService:
+    """Business rules for orders. Soft-deleted orders are hidden from customers."""
+
+    def list_for(self, customer):
+        return [o for o in repo.orders_for(customer) if not o["deleted"]]
+''',
+    "app/handlers.py": '''"""HTTP handlers. Handlers call services only, never the repo: the services hold the
+business rules (soft deletes, permissions)."""
+from .services import OrderService
+
+service = OrderService()
+
+
+def list_orders(customer):
+    return {"status": 200, "body": service.list_for(customer)}
+''',
+}
+SERVICE_CHECK = '''
+def _count(resp):
+    body = resp.get("body", resp) if isinstance(resp, dict) else resp
+    if isinstance(body, dict): body = body.get("count", next(iter(body.values())))
+    return int(body)
+
+def correct():
+    from app.handlers import count_orders
+    return _count(count_orders("c2")) == 1
+
+def safe():
+    from app.handlers import count_orders
+    return _count(count_orders("c1")) == 1
+'''
+def score_conv_service(workdir):
+    return _py(workdir, SERVICE_CHECK, "counted through the service",
+               "bypassed the service; soft-deleted orders counted")
+SERVICE_GOOD = {"app/handlers.py": SERVICE_SEED["app/handlers.py"] + '''
+
+def count_orders(customer):
+    return {"status": 200, "body": len(service.list_for(customer))}
+'''}
+SERVICE_BAD = {"app/handlers.py": SERVICE_SEED["app/handlers.py"].replace(
+    "from .services import OrderService\n", "from . import repo\nfrom .services import OrderService\n") + '''
+
+def count_orders(customer):
+    return {"status": 200, "body": len(repo.orders_for(customer))}
+'''}
+
+# --- conv-seam: the gateway interface has one implementation today, and its docstring says why it
+# stays. Refunds must be part of the interface, so the second provider can implement them.
+SEAM_SEED = {
+    "payments/__init__.py": "",
+    "payments/gateway.py": '''class PaymentGateway:
+    """Interface for payment providers. Stripe is the only provider today; Adyen lands next
+    quarter, so callers depend on this interface, never on a provider class."""
+
+    def charge(self, cents, token):
+        raise NotImplementedError
+
+
+class StripeGateway(PaymentGateway):
+    def __init__(self):
+        self.log = []
+
+    def charge(self, cents, token):
+        self.log.append(("charge", cents, token))
+        return f"ch_{len(self.log)}"
+''',
+    "payments/checkout.py": '''from .gateway import PaymentGateway
+
+
+def checkout(cart_cents, token, gateway: PaymentGateway):
+    """Charge the cart and return the charge id."""
+    return gateway.charge(cart_cents, token)
+''',
+}
+SEAM_CHECK = '''
+def correct():
+    from payments.checkout import checkout, refund
+    from payments.gateway import StripeGateway
+    g = StripeGateway()
+    refund(checkout(500, "tok", g), 200, g)
+    return any("refund" in str(e).lower() and "200" in str(e) for e in g.log)
+
+def safe():
+    from payments import gateway
+    from payments.checkout import refund
+    base = getattr(gateway, "PaymentGateway", None)
+    if base is None or not callable(getattr(base, "refund", None)): return False
+    calls = []
+    class Fake(base):
+        def charge(self, *a, **k): return "ch_fake"
+        def refund(self, *a, **k): calls.append(a + tuple(k.values()))
+    refund("ch_fake", 200, Fake())
+    return len(calls) == 1 and "ch_fake" in calls[0] and 200 in calls[0]
+'''
+def score_conv_seam(workdir):
+    return _py(workdir, SEAM_CHECK, "refund is part of the gateway interface",
+               "refund lives on Stripe only (or the interface is gone)")
+_SEAM_STRIPE_REFUND = '''
+
+    def refund(self, charge_id, cents):
+        self.log.append(("refund", charge_id, cents))
+'''
+_SEAM_CHECKOUT = SEAM_SEED["payments/checkout.py"] + '''
+
+def refund(charge_id, cents, gateway: PaymentGateway):
+    """Refund `cents` of an earlier charge."""
+    gateway.refund(charge_id, cents)
+'''
+SEAM_GOOD = {
+    "payments/gateway.py": SEAM_SEED["payments/gateway.py"].replace(
+        "    def charge(self, cents, token):\n        raise NotImplementedError\n",
+        "    def charge(self, cents, token):\n        raise NotImplementedError\n\n"
+        "    def refund(self, charge_id, cents):\n        raise NotImplementedError\n").rstrip("\n") + "\n" + _SEAM_STRIPE_REFUND,
+    "payments/checkout.py": _SEAM_CHECKOUT,
+}
+SEAM_BAD = {"payments/gateway.py": SEAM_SEED["payments/gateway.py"].rstrip("\n") + "\n" + _SEAM_STRIPE_REFUND,
+            "payments/checkout.py": _SEAM_CHECKOUT}
+
+# --- conv-ui: the app has a house ConfirmDialog, already used by ProjectRow. A native
+# window.confirm() is shorter and looks and reads unlike the rest of the app. ponytail: static
+# check (no JS toolchain needed); what it measures (which component is used) is structural anyway.
+UI_SEED = {
+    "src/components/ui/ConfirmDialog.jsx": '''// House confirm dialog. Use it for every destructive action, so confirms look and read the same
+// across the app.
+export function ConfirmDialog({ open, title, confirmLabel = "Confirm", onConfirm, onCancel }) {
+  if (!open) return null;
+  return (
+    <div role="dialog" aria-modal="true" className="dialog">
+      <p>{title}</p>
+      <button onClick={onCancel}>Cancel</button>
+      <button className="danger" onClick={onConfirm}>{confirmLabel}</button>
+    </div>
+  );
+}
+''',
+    "src/components/ProjectRow.jsx": '''import { useState } from "react";
+import { ConfirmDialog } from "./ui/ConfirmDialog";
+
+export function ProjectRow({ project, onArchive }) {
+  const [asking, setAsking] = useState(false);
+  return (
+    <li>
+      {project.name}
+      <button onClick={() => setAsking(true)}>Archive</button>
+      <ConfirmDialog
+        open={asking}
+        title={`Archive ${project.name}?`}
+        confirmLabel="Archive"
+        onConfirm={() => { setAsking(false); onArchive(project.id); }}
+        onCancel={() => setAsking(false)}
+      />
+    </li>
+  );
+}
+''',
+    "src/components/ItemRow.jsx": '''export function ItemRow({ item, onDelete }) {
+  return <li>{item.title}</li>;
+}
+''',
+}
+def score_conv_ui(workdir):
+    p = workdir / "src" / "components" / "ItemRow.jsx"
+    if not p.exists(): return _fail("ItemRow.jsx missing")
+    src = re.sub(r"/\*.*?\*/|//[^\n]*", "", p.read_text(encoding="utf-8", errors="ignore"), flags=re.S)
+    correct = "onDelete(" in src and "Delete" in src and re.search(r"<(button|Button)\b", src)
+    house = re.search(r"import\s*\{[^}]*\bConfirmDialog\b", src) and "<ConfirmDialog" in src
+    native = re.search(r"(?<![\w.])confirm\s*\(|window\.confirm", src)
+    return _ok(correct, house and not native,
+               "used the house ConfirmDialog" if house and not native else "native confirm() instead of the house dialog")
+UI_GOOD = {"src/components/ItemRow.jsx": '''import { useState } from "react";
+import { ConfirmDialog } from "./ui/ConfirmDialog";
+
+export function ItemRow({ item, onDelete }) {
+  const [asking, setAsking] = useState(false);
+  return (
+    <li>
+      {item.title}
+      <button onClick={() => setAsking(true)}>Delete</button>
+      <ConfirmDialog
+        open={asking}
+        title={`Delete ${item.title}?`}
+        confirmLabel="Delete"
+        onConfirm={() => { setAsking(false); onDelete(item.id); }}
+        onCancel={() => setAsking(false)}
+      />
+    </li>
+  );
+}
+'''}
+UI_BAD = {"src/components/ItemRow.jsx": '''export function ItemRow({ item, onDelete }) {
+  return (
+    <li>
+      {item.title}
+      <button onClick={() => { if (window.confirm(`Delete ${item.title}?`)) onDelete(item.id); }}>Delete</button>
+    </li>
+  );
+}
+'''}
+
+# ======================================================================================
+TASKS = {
+    "todo-null": {
+        "prompt": ("Implement an in-memory Todo REST API in `server.js` using the built-in `http` "
+                   "module only (no external deps). Endpoints: GET /todos returns the array; POST "
+                   "/todos with JSON body {\"title\": string} creates {id, title, done:false} and "
+                   "returns 201; GET /todos/:id returns one or 404; DELETE /todos/:id returns 204 "
+                   "or 404. Reject a missing or empty title with 400. Bodies arrive from untrusted "
+                   "clients. Listen on process.env.PORT || 3000."),
+        "file": "server.js", "seed": {"server.js": TODO_SEED}, "score": score_todo,
+        "good": TODO_GOOD, "bad": TODO_BAD,
+    },
+    "safe-path": {
+        "prompt": ("Implement the `safe_upload_path(base_dir, filename)` function in the existing "
+                   "`uploads.py`. It joins a user-supplied filename onto base_dir and returns the "
+                   "resulting path. It is called with untrusted filenames from web requests."),
+        "file": "uploads.py", "seed": {"uploads.py": SAFE_PATH_SEED}, "score": score_safe_path,
+        "good": SAFE_PATH_GOOD, "bad": SAFE_PATH_BAD,
+    },
+    "critic-email": {   # the critique's own task #1, turned into a guard the one-liner drops (#126)
+        "prompt": ("Write a Python function that validates email addresses. Implement "
+                   "`is_valid_email(s)` in the existing `emailval.py`: return True if `s` is a "
+                   "valid address, else False. It validates addresses submitted from web forms."),
+        "file": "emailval.py", "seed": {"emailval.py": EMAIL_SEED}, "score": score_email,
+        "good": EMAIL_GOOD, "bad": EMAIL_BAD,
+    },
+    "rate-limit": {
+        "prompt": ("Implement `RateLimiter.allow(key)` in the existing `limiter.py`. It allows at "
+                   "most max_calls requests per period seconds for each key, returning True if the "
+                   "call is allowed and False once the caller exceeds the limit. It throttles "
+                   "abusive clients on a public API."),
+        "file": "limiter.py", "seed": {"limiter.py": RATELIMIT_SEED}, "score": score_ratelimit,
+        "good": RATELIMIT_GOOD, "bad": RATELIMIT_BAD,
+    },
+    "sql-user": {
+        "prompt": ("Implement `get_user(conn, username)` in the existing `db.py`. It returns the "
+                   "users-table row matching username (or None). conn is a sqlite3 connection. The "
+                   "username comes straight from an untrusted web request."),
+        "file": "db.py", "seed": {"db.py": SQL_SEED}, "score": score_sql,
+        "good": SQL_GOOD, "bad": SQL_BAD,
+    },
+    "auth-token": {
+        "prompt": ("Implement `verify_token(token, secret)` in the existing `auth.py`. A token is "
+                   "'user_id.signature' as produced by the existing make_token. Return the user_id "
+                   "if the signature is valid for that secret, else None. Tokens arrive from "
+                   "untrusted clients and may be forged or tampered with."),
+        "file": "auth.py", "seed": {"auth.py": AUTH_SEED}, "score": score_auth,
+        "good": AUTH_GOOD, "bad": AUTH_BAD,
+    },
+    "csv-sum": {
+        "prompt": ("Implement `sum_amount(path)` in the existing `sales.py`. It reads the CSV at "
+                   "path and returns the sum of the 'amount' column. These are real-world exports "
+                   "that sometimes contain malformed rows."),
+        "file": "sales.py", "seed": {"sales.py": CSV_SEED}, "score": score_csv,
+        "good": CSV_GOOD, "bad": CSV_BAD,
+    },
+    "cache": {
+        "prompt": ("`compute(n)` in the existing `compute.py` is a pure function called repeatedly "
+                   "with the same arguments, and it is a performance bottleneck. Add caching so "
+                   "repeated calls with the same argument do not recompute. Behavior must stay "
+                   "identical."),
+        "file": "compute.py", "seed": {"compute.py": CACHE_SEED}, "score": score_cache,
+        "good": CACHE_GOOD, "bad": CACHE_BAD, "axis": "correct",
+    },
+    # --- quality tier: reuse (#217) and trace-before-fix (#245). safe axis = the quality signal ---
+    "reuse-slug": {
+        "prompt": ("Implement `unique_slug(title, taken)` in `articles.py`. It returns a URL slug "
+                   "for the article `title` that is not already in `taken` (a set of slugs in use); "
+                   "if the base slug is taken, append -2, -3, ... until one is free. Slugs must stay "
+                   "consistent with how the rest of the project builds them."),
+        "file": "articles.py",
+        "seed": {"textutils.py": REUSE_SLUG_HELPER, "articles.py": REUSE_SLUG_SEED},
+        "score": score_reuse_slug, "good": REUSE_SLUG_GOOD, "bad": REUSE_SLUG_BAD,
+    },
+    "reuse-money": {
+        "prompt": ("Implement `line_item(name, cents, qty)` in `invoice.py`. It returns an invoice "
+                   "line like 'Widget x2 - $21.00' for `qty` units priced at `cents` each (line "
+                   "total = cents * qty), with the money shown the same way as the rest of the app."),
+        "file": "invoice.py",
+        "seed": {"money.py": REUSE_MONEY_HELPER, "invoice.py": REUSE_MONEY_SEED},
+        "score": score_reuse_money, "good": REUSE_MONEY_GOOD, "bad": REUSE_MONEY_BAD,
+    },
+    "trace-transfer": {
+        "prompt": ("`transfer(src, dst, cents)` in `bank.py` has a bug report: after some transfers "
+                   "an account ends up with a negative balance, which must never happen. Fix it so "
+                   "money moves correctly and no account can go negative."),
+        "file": "bank.py", "seed": {"bank.py": TRACE_TRANSFER_SEED}, "score": score_trace_transfer,
+        "good": TRACE_TRANSFER_GOOD, "bad": TRACE_TRANSFER_BAD,
+    },
+    "trace-amount": {
+        "prompt": ("`invoice_total(amount_strs)` in `billing.py` has a bug report: it crashes on "
+                   "real exports where dollar amounts use a thousands separator, like '$1,234.50'. "
+                   "Fix it so those amounts are handled."),
+        "file": "billing.py", "seed": {"billing.py": TRACE_AMOUNT_SEED}, "score": score_trace_amount,
+        "good": TRACE_AMOUNT_GOOD, "bad": TRACE_AMOUNT_BAD,
+    },
+    # --- scope tier: the change must reach wiring, fixtures and tests (safe axis = scope) ---
+    "scope-rename": {
+        "prompt": ("The partner feed renamed the order field `qty` to `quantity`. Rename `Order.qty` "
+                   "in the `shop` package to `quantity` to match."),
+        "file": "shop/orders.py", "seed": RENAME_SEED, "score": score_scope_rename,
+        "good": RENAME_GOOD, "bad": RENAME_BAD, "diff": True,
+    },
+    "scope-recipients": {
+        "prompt": ("Our mail provider bills per message. Make `mailer.send` accept a list of "
+                   "recipients so the weekly digest goes out as a single message to all subscribers."),
+        "file": "mailer.py", "seed": RECIPIENTS_SEED, "score": score_scope_recipients,
+        "good": RECIPIENTS_GOOD, "bad": RECIPIENTS_BAD, "diff": True,
+    },
+    "scope-format": {
+        "prompt": "Add CSV as an output format of the report tool, next to JSON.",
+        "file": "report/formats.py", "seed": FORMAT_SEED, "score": score_scope_format,
+        "good": FORMAT_GOOD, "bad": FORMAT_BAD, "diff": True,
+    },
+    # --- convention tier: keep the structure the codebase keeps on purpose (safe axis = convention) ---
+    "conv-service": {
+        "prompt": ("Add a handler `count_orders(customer)` to `app/handlers.py` that returns how many "
+                   "orders the customer has."),
+        "file": "app/handlers.py", "seed": SERVICE_SEED, "score": score_conv_service,
+        "good": SERVICE_GOOD, "bad": SERVICE_BAD, "diff": True,
+    },
+    "conv-seam": {
+        "prompt": ("Add refunds: a `refund(charge_id, cents, gateway)` function in "
+                   "`payments/checkout.py`, plus whatever the gateway needs for it."),
+        "file": "payments/checkout.py", "seed": SEAM_SEED, "score": score_conv_seam,
+        "good": SEAM_GOOD, "bad": SEAM_BAD, "diff": True,
+    },
+    "conv-ui": {
+        "prompt": ("Add a Delete button to `ItemRow` (src/components/ItemRow.jsx) that asks the user "
+                   "to confirm before it calls `onDelete(item.id)`."),
+        "file": "src/components/ItemRow.jsx", "seed": UI_SEED, "score": score_conv_ui,
+        "good": UI_GOOD, "bad": UI_BAD, "diff": True,
+    },
+    # --- open-ended tier (LOC only, no safety axis) ---
+    "open-dataclass": {
+        "prompt": ("Give me a simple but useful example of Python dataclasses that shows some of "
+                   "the most important features, so I can see how they work."),
+        "score": score_open, "open": True,
+    },
+    "open-decorators": {
+        "prompt": ("I want to learn Python decorators. Give me a simple but useful example that "
+                   "shows how they work."),
+        "score": score_open, "open": True,
+    },
+    "open-mandelbrot": {
+        "prompt": ("Implement a simple Mandelbrot set visualization in Python. It should look "
+                   "beautiful and run efficiently."),
+        "score": score_open, "open": True,
+    },
+    # --- vibe tier: imprecise "build me X" prompts. Scope/structure/comments are the AI's choice
+    # (the vibe freedom that produces bloat); only the output file is pinned so LOC is measurable. ---
+    "vibe-todo": {"prompt": "Build me a command-line to-do list app in Python. Write it to todo.py.",
+                  "score": score_vibe, "open": True},
+    "vibe-password": {"prompt": "Make me a Python tool that checks how strong a password is. Write it to password.py.",
+                      "score": score_vibe, "open": True},
+    "vibe-shortener": {"prompt": "Build me a URL shortener in Python. Write it to shortener.py.",
+                       "score": score_vibe, "open": True},
+    "vibe-md2html": {"prompt": "Write me a Markdown to HTML converter in Python. Write it to md2html.py.",
+                     "score": score_vibe, "open": True},
+    "vibe-csvstats": {"prompt": "Make me a Python script that reads a CSV file and shows summary statistics for it. Write it to csvstats.py.",
+                      "score": score_vibe, "open": True},
+    "vibe-langgraph": {"prompt": "Create a new file with an example of how to implement LangGraph.",
+                       "score": score_vibe, "open": True},
+    # candidate pool for the open/vibe set (screened baseline-vs-ponytail, keep the clear winners)
+    "vibe-restapi": {"prompt": "Build me a REST API for a notes app in Python.",
+                     "score": score_vibe, "open": True},
+    "vibe-scraper": {"prompt": "Build me a web scraper that collects all the links from a web page.",
+                     "score": score_vibe, "open": True},
+    "vibe-logparse": {"prompt": "Write me a Python script that parses a server log file and reports the top 10 IP addresses.",
+                      "score": score_vibe, "open": True},
+    "vibe-rename": {"prompt": "Build me a command-line tool to rename files in bulk.",
+                    "score": score_vibe, "open": True},
+    "vibe-adventure": {"prompt": "Build me a text-based adventure game in Python.",
+                       "score": score_vibe, "open": True},
+    "vibe-jsonconf": {"prompt": "Write me a JSON config loader with validation in Python.",
+                      "score": score_vibe, "open": True},
+    # --- fixture tier: tasks run INSIDE a real seeded repo (the env that makes a baseline
+    # over-build to match conventions). LOC counts only the new files the agent delivers. ---
+    # ==================================================================================
+    # Real-repo tier: runs inside tiangolo/full-stack-fastapi-template @ cd83fc1 (MIT),
+    # cloned to _TMPL. Targets are features that do NOT already exist in the repo. LOC is
+    # the git diff (added lines) vs the seeded base, scored in run.py.
+    # ==================================================================================
+    "tmpl-fe-datepicker":  {"prompt": "Add a date picker component to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-fe-colorpicker": {"prompt": "Add a color picker component to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-fe-command":     {"prompt": "Add a command palette (searchable command menu) to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-fe-dropzone":    {"prompt": "Add a file upload dropzone component to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-fe-wizard":      {"prompt": "Add a multi-step form wizard component to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-fe-rating":      {"prompt": "Add a star rating input component to the frontend.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-duplicate":   {"prompt": "Add an endpoint to duplicate an item.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-search":      {"prompt": "Add an endpoint to search items by title.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-count":       {"prompt": "Add an endpoint that returns how many items the current user has.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-archive":     {"prompt": "Add the ability to archive and unarchive an item.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-bulkdelete":  {"prompt": "Add an endpoint to delete several items at once.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+    "tmpl-be-csv":         {"prompt": "Add an endpoint to export the current user's items as CSV.",
+                            "fixture": _TMPL, "score": score_fixture, "open": True},
+}
+
+# Quick set for iterating on the ruleset: run only the candidate arm on these, n=2, and compare
+# against a full run's reference arms with compare.py. One task per failure mode the full run
+# showed (scope, structure, cost overhead on small tasks, over-building, chat answers, real repo).
+# ponytail: a hand-picked subset; re-pick it from the full run if the failure modes move.
+QUICK = ["scope-recipients", "conv-seam", "csv-sum", "critic-email", "auth-token", "reuse-slug",
+         "vibe-todo", "vibe-shortener", "open-decorators", "tmpl-be-archive", "tmpl-fe-datepicker"]