@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

package/suite/mine.py

@@ -1,154 +1,154 @@
-"""Turn judged bench-gateway calls into DRAFT claim-verification tasks.
-
-    python suite/mine.py                  # judged calls with claims not yet mined
-    python suite/mine.py p20261016-...    # draft tasks from one call
-
-THE SUITE GROWS FROM REAL WORK
-------------------------------
-Every bench-gateway call logs its prompt — including the files, exactly as the
-models saw them — and a verdict can record verified claims:
-
-    python metrics/offload.py verdict <call id> --useful \\
-        --real "<a finding it got right>" --false "<a claim it made that is untrue>"
-
-Each claim becomes one task: the files as fixtures, the claim as a neutral
-hypothesis, `VERDICT: TRUE` or `VERDICT: FALSE` on the first line. A --real
-claim is true, a --false one is not, so both kinds accumulate. The code is the
-version that was reviewed, frozen: a claim task tests verification, not
-whether today's code still has the bug.
-
-DRAFTS, NOT TASKS
------------------
-A mined task is written with "draft": true, and suite/run.py skips drafts
-unless asked for by --task. Read it before removing the flag: is the claim
-stated so the files alone decide it, with no model's name and no line number
-that gives the answer away? Is the fixture small enough for free-tier models?
-A claim the attached files cannot settle is a bad task, however true.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import pathlib
-import re
-import sys
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-sys.path.insert(0, str(ROOT / "metrics"))
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-TASKS = DATA / "suite" / "tasks"
-CALLS = DATA / "runs" / "gateway" / "calls.jsonl"
-sys.path.insert(0, str(ROOT / "metrics"))
-import offload  # noqa: E402
-
-PROMPT = ("Hypothesis about the attached code:\n\n{claim}\n\n"
-          "Decide whether the hypothesis is TRUE or FALSE for the code exactly as attached, using"
-          " standard Python and standard-library behaviour. Your first line must be exactly"
-          " `VERDICT: TRUE` or `VERDICT: FALSE`. Then quote the line or lines that decide it and"
-          " explain in at most five sentences. Plain ASCII.")
-HEADER = re.compile(r"^===== (?:FILE: )?(.+?)(?: =====)?$", re.M)
-BIG = 40_000      # characters of fixtures; past this, free models start failing
-
-
-def calls() -> dict[str, dict]:
-    out = {}
-    for line in CALLS.read_text(encoding="utf-8").splitlines() if CALLS.exists() else []:
-        try:
-            r = json.loads(line)
-        except ValueError:
-            continue
-        out[r["panel"]] = r
-    return out
-
-
-def files_in(prompt: str) -> dict[str, str]:
-    """`===== path` sections of a logged prompt -> {path: text}. A section
-    whose name is not a file (a git diff, say) is kept under a safe name."""
-    marks = list(HEADER.finditer(prompt))
-    out = {}
-    for i, m in enumerate(marks):
-        end = marks[i + 1].start() if i + 1 < len(marks) else len(prompt)
-        name = re.sub(r"[^\w.-]+", "-", m.group(1).strip()).strip("-") or f"part{i}"
-        out[name if "." in name else name + ".txt"] = prompt[m.end():end].strip("\n") + "\n"
-    return out
-
-
-def claims_for(panel: str) -> list[tuple[str, bool, str]]:
-    """(claim, is_true, seat call id) from every judged seat of the call."""
-    seen, out = set(), []
-    for e in offload.entries():
-        if not str(e.get("call", "")).startswith(panel + "."):
-            continue
-        for text, truth in [(c, True) for c in e.get("real", [])] + [(c, False) for c in e.get("false", [])]:
-            if text.lower() not in seen:
-                seen.add(text.lower())
-                out.append((text, truth, e["call"]))
-    return out
-
-
-def mined() -> set[str]:
-    return {json.loads(p.read_text(encoding="utf-8")).get("mined_from", "")
-            for p in TASKS.glob("*/task.json")}
-
-
-def draft(panel: str) -> list[str]:
-    record = calls().get(panel)
-    if record is None:
-        sys.exit(f"no gateway call {panel!r} in {CALLS.relative_to(ROOT)}")
-    files = files_in(record["prompt"])
-    if not files:
-        sys.exit(f"{panel}: its prompt attached no `===== file` sections, so nothing can be frozen")
-    found = claims_for(panel)
-    if not found:
-        sys.exit(f"{panel}: no verdict records --real or --false claims for it yet")
-    size = sum(len(t) for t in files.values())
-    written = []
-    for n, (claim, truth, call) in enumerate(found, 1):
-        task_id = f"mined-{panel.removeprefix('p')}-{n}"
-        d = TASKS / task_id
-        (d / "fixtures").mkdir(parents=True, exist_ok=True)
-        for name, text in files.items():
-            (d / "fixtures" / name).write_text(text, encoding="utf-8")
-        yes, no = ("verdict: true", "verdict: false") if truth else ("verdict: false", "verdict: true")
-        spec = {"id": task_id, "draft": True, "shape": "claim", "mined_from": call,
-                "title": f"Claim verification ({'true' if truth else 'false'} claim), mined",
-                "why": f"{'TRUE' if truth else 'FALSE'}, verified by the caller in the verdict on"
-                       f" {call} (model {record.get('model')}). Review before removing draft.",
-                # Its files are the snapshot copied above: a repo key no install
-                # maps, so build_prompt always reads the fixtures.
-                "prompt": PROMPT.format(claim=claim), "repo": "fixtures",
-                "attach": [f"snapshot/{name}" for name in files],
-                "must_find": [{"any_of": [yes]}], "must_not_claim": [no],
-                "score_in": "first_line", "min_answer_chars": 10}
-        (d / "task.json").write_text(json.dumps(spec, indent=2, ensure_ascii=False) + "\n",
-                                     encoding="utf-8")
-        written.append(task_id)
-    print(f"{len(written)} draft task(s) from {panel}: {', '.join(written)}")
-    if size > BIG:
-        print(f"  note: {size:,} characters of fixtures; trim them to what the claims need"
-              " before promoting, or free models will fail on size rather than judgement")
-    print("  review each task.json, then remove \"draft\": true to put it in the suite")
-    return written
-
-
-def main() -> None:
-    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("panel", nargs="?", help="a gateway call id, e.g. p20261016-165636-afd6")
-    args = ap.parse_args()
-    if args.panel:
-        draft(args.panel)
-        return
-    done = {c.rsplit(".", 1)[0] for c in mined()}
-    rows = [(p, len(claims_for(p))) for p in calls()]
-    rows = [(p, n) for p, n in rows if n and p not in done]
-    if not rows:
-        print("Nothing to mine: no judged gateway call carries --real/--false claims that are"
-              " not already tasks.")
-    for p, n in rows:
-        print(f"  {p}: {n} claim(s)   python suite/mine.py {p}")
-
-
-if __name__ == "__main__":
-    main()
+"""Turn judged bench-gateway calls into DRAFT claim-verification tasks.
+
+    python suite/mine.py                  # judged calls with claims not yet mined
+    python suite/mine.py p20261016-...    # draft tasks from one call
+
+THE SUITE GROWS FROM REAL WORK
+------------------------------
+Every bench-gateway call logs its prompt — including the files, exactly as the
+models saw them — and a verdict can record verified claims:
+
+    python metrics/offload.py verdict <call id> --useful \\
+        --real "<a finding it got right>" --false "<a claim it made that is untrue>"
+
+Each claim becomes one task: the files as fixtures, the claim as a neutral
+hypothesis, `VERDICT: TRUE` or `VERDICT: FALSE` on the first line. A --real
+claim is true, a --false one is not, so both kinds accumulate. The code is the
+version that was reviewed, frozen: a claim task tests verification, not
+whether today's code still has the bug.
+
+DRAFTS, NOT TASKS
+-----------------
+A mined task is written with "draft": true, and suite/run.py skips drafts
+unless asked for by --task. Read it before removing the flag: is the claim
+stated so the files alone decide it, with no model's name and no line number
+that gives the answer away? Is the fixture small enough for free-tier models?
+A claim the attached files cannot settle is a bad task, however true.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "metrics"))
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+TASKS = DATA / "suite" / "tasks"
+CALLS = DATA / "runs" / "gateway" / "calls.jsonl"
+sys.path.insert(0, str(ROOT / "metrics"))
+import offload  # noqa: E402
+
+PROMPT = ("Hypothesis about the attached code:\n\n{claim}\n\n"
+          "Decide whether the hypothesis is TRUE or FALSE for the code exactly as attached, using"
+          " standard Python and standard-library behaviour. Your first line must be exactly"
+          " `VERDICT: TRUE` or `VERDICT: FALSE`. Then quote the line or lines that decide it and"
+          " explain in at most five sentences. Plain ASCII.")
+HEADER = re.compile(r"^===== (?:FILE: )?(.+?)(?: =====)?$", re.M)
+BIG = 40_000      # characters of fixtures; past this, free models start failing
+
+
+def calls() -> dict[str, dict]:
+    out = {}
+    for line in CALLS.read_text(encoding="utf-8").splitlines() if CALLS.exists() else []:
+        try:
+            r = json.loads(line)
+        except ValueError:
+            continue
+        out[r["panel"]] = r
+    return out
+
+
+def files_in(prompt: str) -> dict[str, str]:
+    """`===== path` sections of a logged prompt -> {path: text}. A section
+    whose name is not a file (a git diff, say) is kept under a safe name."""
+    marks = list(HEADER.finditer(prompt))
+    out = {}
+    for i, m in enumerate(marks):
+        end = marks[i + 1].start() if i + 1 < len(marks) else len(prompt)
+        name = re.sub(r"[^\w.-]+", "-", m.group(1).strip()).strip("-") or f"part{i}"
+        out[name if "." in name else name + ".txt"] = prompt[m.end():end].strip("\n") + "\n"
+    return out
+
+
+def claims_for(panel: str) -> list[tuple[str, bool, str]]:
+    """(claim, is_true, seat call id) from every judged seat of the call."""
+    seen, out = set(), []
+    for e in offload.entries():
+        if not str(e.get("call", "")).startswith(panel + "."):
+            continue
+        for text, truth in [(c, True) for c in e.get("real", [])] + [(c, False) for c in e.get("false", [])]:
+            if text.lower() not in seen:
+                seen.add(text.lower())
+                out.append((text, truth, e["call"]))
+    return out
+
+
+def mined() -> set[str]:
+    return {json.loads(p.read_text(encoding="utf-8")).get("mined_from", "")
+            for p in TASKS.glob("*/task.json")}
+
+
+def draft(panel: str) -> list[str]:
+    record = calls().get(panel)
+    if record is None:
+        sys.exit(f"no gateway call {panel!r} in {CALLS.relative_to(ROOT)}")
+    files = files_in(record["prompt"])
+    if not files:
+        sys.exit(f"{panel}: its prompt attached no `===== file` sections, so nothing can be frozen")
+    found = claims_for(panel)
+    if not found:
+        sys.exit(f"{panel}: no verdict records --real or --false claims for it yet")
+    size = sum(len(t) for t in files.values())
+    written = []
+    for n, (claim, truth, call) in enumerate(found, 1):
+        task_id = f"mined-{panel.removeprefix('p')}-{n}"
+        d = TASKS / task_id
+        (d / "fixtures").mkdir(parents=True, exist_ok=True)
+        for name, text in files.items():
+            (d / "fixtures" / name).write_text(text, encoding="utf-8")
+        yes, no = ("verdict: true", "verdict: false") if truth else ("verdict: false", "verdict: true")
+        spec = {"id": task_id, "draft": True, "shape": "claim", "mined_from": call,
+                "title": f"Claim verification ({'true' if truth else 'false'} claim), mined",
+                "why": f"{'TRUE' if truth else 'FALSE'}, verified by the caller in the verdict on"
+                       f" {call} (model {record.get('model')}). Review before removing draft.",
+                # Its files are the snapshot copied above: a repo key no install
+                # maps, so build_prompt always reads the fixtures.
+                "prompt": PROMPT.format(claim=claim), "repo": "fixtures",
+                "attach": [f"snapshot/{name}" for name in files],
+                "must_find": [{"any_of": [yes]}], "must_not_claim": [no],
+                "score_in": "first_line", "min_answer_chars": 10}
+        (d / "task.json").write_text(json.dumps(spec, indent=2, ensure_ascii=False) + "\n",
+                                     encoding="utf-8")
+        written.append(task_id)
+    print(f"{len(written)} draft task(s) from {panel}: {', '.join(written)}")
+    if size > BIG:
+        print(f"  note: {size:,} characters of fixtures; trim them to what the claims need"
+              " before promoting, or free models will fail on size rather than judgement")
+    print("  review each task.json, then remove \"draft\": true to put it in the suite")
+    return written
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("panel", nargs="?", help="a gateway call id, e.g. p20261016-165636-afd6")
+    args = ap.parse_args()
+    if args.panel:
+        draft(args.panel)
+        return
+    done = {c.rsplit(".", 1)[0] for c in mined()}
+    rows = [(p, len(claims_for(p))) for p in calls()]
+    rows = [(p, n) for p, n in rows if n and p not in done]
+    if not rows:
+        print("Nothing to mine: no judged gateway call carries --real/--false claims that are"
+              " not already tasks.")
+    for p, n in rows:
+        print(f"  {p}: {n} claim(s)   python suite/mine.py {p}")
+
+
+if __name__ == "__main__":
+    main()

package/suite/pace.py

@@ -1,145 +1,145 @@
-"""Run suite tasks across the roster slowly, one call at a time, in the background.
-
-    python suite/pace.py                         # every seeded-hard task, every model, 120 s apart
-    python suite/pace.py --prefix claim- --gap 300
-    python suite/pace.py --dry-run               # list what would run, call nothing
-
-Started detached it works through the queue on its own and stops when done:
-
-    Start-Process .venv\\Scripts\\python.exe -ArgumentList suite\\pace.py -WindowStyle Hidden   # Windows
-    nohup .venv/bin/python suite/pace.py > /dev/null 2>&1 &                                    # macOS, Linux
-
-FREE TIERS LIMIT RATE, NOT TOTAL
---------------------------------
-A full roster pass of ten seeded pairs is ~300 calls of 8-14K characters. Sent
-back to back it trips every provider's per-minute limit and the run fills
-with "rate limit" rows that say nothing about the models. Sent one at a time,
-`--gap` seconds apart and ROUND-ROBIN — task by task across every model, so
-one model is asked once per lap rather than twenty times in a row — each model
-sees a call roughly every (models x gap) seconds, far under any limit.
-
-A call that comes back rate-limited is deleted (it is not an answer) and its
-model is rested for `--rest` seconds before being asked again later; the rest
-of the queue carries on. Pairs that already have a saved answer are skipped,
-so the run can be stopped and started again at any time and picks up where it
-was. It logs to runs/pace-<date>.log and calls nothing that suite/run.py
-would not.
-
-Models on a retiring alert, ad hoc models and decision models are left out:
-the first are failing their probes, the second cost money, the third cannot
-answer a prompt.
-"""
-
-from __future__ import annotations
-
-import argparse
-import datetime as dt
-import json
-import os
-import pathlib
-import subprocess
-import sys
-import time
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-sys.path.insert(0, str(ROOT / "metrics"))
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-TASKS = DATA / "suite" / "tasks"
-sys.path.insert(0, str(ROOT / "suite"))
-sys.path.insert(0, str(ROOT / "metrics"))
-import run  # noqa: E402
-
-PY = run.bench_python()
-
-
-def models() -> list[str]:
-    try:
-        import alerts
-        retiring = {a.get("alias") for a in alerts.active().values() if a.get("kind") == "retiring"}
-    except Exception:                                  # noqa: BLE001
-        retiring = set()
-    return [m for m in run.available_models() if m not in retiring and m not in run.ad_hoc_models()]
-
-
-def answered(task: str, model: str) -> bool:
-    """A saved file that is an answer. One that recorded a block (a rate
-    limit, an error, a timeout) is not, so the pair is asked again."""
-    out = TASKS / task / "output" / f"{model}.md"
-    if not out.exists():
-        return False
-    spec = json.loads((TASKS / task / "task.json").read_text(encoding="utf-8"))
-    return run.score(spec, out.read_text(encoding="utf-8", errors="replace"))["kind"] == "answered"
-
-
-def queue(prefix: str, roster: list[str]) -> list[tuple[str, str]]:
-    """(task, model) pairs without an answer yet, task by task across models."""
-    tasks = sorted(p.parent.name for p in TASKS.glob(f"{prefix}*/task.json")
-                   if not json.loads(p.read_text(encoding="utf-8")).get("draft"))
-    return [(t, m) for t in tasks for m in roster if not answered(t, m)]
-
-
-def rate_limited(task: str, model: str) -> bool:
-    out = TASKS / task / "output" / f"{model}.md"
-    if not out.exists():
-        return False
-    blocked = run.score(json.loads((TASKS / task / "task.json").read_text(encoding="utf-8")),
-                        out.read_text(encoding="utf-8", errors="replace")).get("blocked") or ""
-    return "rate limit" in blocked or "capacity" in blocked
-
-
-def main() -> None:
-    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    ap.add_argument("--prefix", default="seeded-hard-", help="task directories starting with this")
-    ap.add_argument("--gap", type=int, default=120, help="seconds between calls")
-    ap.add_argument("--rest", type=int, default=1800, help="seconds a rate-limited model is rested")
-    ap.add_argument("--timeout", type=int, default=420)
-    ap.add_argument("--dry-run", action="store_true")
-    args = ap.parse_args()
-
-    roster = models()
-    todo = queue(args.prefix, roster)
-    log_path = DATA / "runs" / f"pace-{dt.date.today().isoformat()}.log"
-
-    def log(msg: str) -> None:
-        line = f"{dt.datetime.now():%H:%M:%S} {msg}"
-        print(line, flush=True)
-        if not args.dry_run:
-            with log_path.open("a", encoding="utf-8") as fh:
-                fh.write(line + "\n")
-
-    eta = len(todo) * (args.gap + 60) / 3600
-    log(f"{len(todo)} calls queued across {len(roster)} models, {args.gap}s apart"
-        f" (about {eta:.1f} h): {', '.join(roster)}")
-    if args.dry_run:
-        for t, m in todo:
-            print(f"  {t}  {m}")
-        return
-    rested: dict[str, float] = {}
-    while todo:
-        now = time.monotonic()
-        pick = next(((t, m) for t, m in todo if rested.get(m, 0) <= now), None)
-        if pick is None:
-            # Only models with work left: a rest that already ended would make
-            # this negative, and time.sleep raises on a negative wait.
-            time.sleep(max(1, min(rested[m] for _, m in todo) - now + 1))
-            continue
-        todo.remove(pick)
-        task, model = pick
-        subprocess.run([str(PY), str(ROOT / "suite" / "run.py"), "--task", task, "--model", model,
-                        "--timeout", str(args.timeout)], cwd=str(ROOT), capture_output=True,
-                       env={**os.environ, "PYTHONIOENCODING": "utf-8"})
-        if rate_limited(task, model):
-            (TASKS / task / "output" / f"{model}.md").unlink()
-            rested[model] = time.monotonic() + args.rest
-            todo.append(pick)
-            log(f"rate-limited  {model} on {task}; resting it {args.rest}s, retried later")
-        else:
-            log(f"done          {model} on {task}  ({len(todo)} left)")
-        if todo:
-            time.sleep(args.gap)
-    log("queue empty; finished")
-
-
-if __name__ == "__main__":
-    main()
+"""Run suite tasks across the roster slowly, one call at a time, in the background.
+
+    python suite/pace.py                         # every seeded-hard task, every model, 120 s apart
+    python suite/pace.py --prefix claim- --gap 300
+    python suite/pace.py --dry-run               # list what would run, call nothing
+
+Started detached it works through the queue on its own and stops when done:
+
+    Start-Process .venv\\Scripts\\python.exe -ArgumentList suite\\pace.py -WindowStyle Hidden   # Windows
+    nohup .venv/bin/python suite/pace.py > /dev/null 2>&1 &                                    # macOS, Linux
+
+FREE TIERS LIMIT RATE, NOT TOTAL
+--------------------------------
+A full roster pass of ten seeded pairs is ~300 calls of 8-14K characters. Sent
+back to back it trips every provider's per-minute limit and the run fills
+with "rate limit" rows that say nothing about the models. Sent one at a time,
+`--gap` seconds apart and ROUND-ROBIN — task by task across every model, so
+one model is asked once per lap rather than twenty times in a row — each model
+sees a call roughly every (models x gap) seconds, far under any limit.
+
+A call that comes back rate-limited is deleted (it is not an answer) and its
+model is rested for `--rest` seconds before being asked again later; the rest
+of the queue carries on. Pairs that already have a saved answer are skipped,
+so the run can be stopped and started again at any time and picks up where it
+was. It logs to runs/pace-<date>.log and calls nothing that suite/run.py
+would not.
+
+Models on a retiring alert, ad hoc models and decision models are left out:
+the first are failing their probes, the second cost money, the third cannot
+answer a prompt.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import pathlib
+import subprocess
+import sys
+import time
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "metrics"))
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+TASKS = DATA / "suite" / "tasks"
+sys.path.insert(0, str(ROOT / "suite"))
+sys.path.insert(0, str(ROOT / "metrics"))
+import run  # noqa: E402
+
+PY = run.bench_python()
+
+
+def models() -> list[str]:
+    try:
+        import alerts
+        retiring = {a.get("alias") for a in alerts.active().values() if a.get("kind") == "retiring"}
+    except Exception:                                  # noqa: BLE001
+        retiring = set()
+    return [m for m in run.available_models() if m not in retiring and m not in run.ad_hoc_models()]
+
+
+def answered(task: str, model: str) -> bool:
+    """A saved file that is an answer. One that recorded a block (a rate
+    limit, an error, a timeout) is not, so the pair is asked again."""
+    out = TASKS / task / "output" / f"{model}.md"
+    if not out.exists():
+        return False
+    spec = json.loads((TASKS / task / "task.json").read_text(encoding="utf-8"))
+    return run.score(spec, out.read_text(encoding="utf-8", errors="replace"))["kind"] == "answered"
+
+
+def queue(prefix: str, roster: list[str]) -> list[tuple[str, str]]:
+    """(task, model) pairs without an answer yet, task by task across models."""
+    tasks = sorted(p.parent.name for p in TASKS.glob(f"{prefix}*/task.json")
+                   if not json.loads(p.read_text(encoding="utf-8")).get("draft"))
+    return [(t, m) for t in tasks for m in roster if not answered(t, m)]
+
+
+def rate_limited(task: str, model: str) -> bool:
+    out = TASKS / task / "output" / f"{model}.md"
+    if not out.exists():
+        return False
+    blocked = run.score(json.loads((TASKS / task / "task.json").read_text(encoding="utf-8")),
+                        out.read_text(encoding="utf-8", errors="replace")).get("blocked") or ""
+    return "rate limit" in blocked or "capacity" in blocked
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--prefix", default="seeded-hard-", help="task directories starting with this")
+    ap.add_argument("--gap", type=int, default=120, help="seconds between calls")
+    ap.add_argument("--rest", type=int, default=1800, help="seconds a rate-limited model is rested")
+    ap.add_argument("--timeout", type=int, default=420)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    roster = models()
+    todo = queue(args.prefix, roster)
+    log_path = DATA / "runs" / f"pace-{dt.date.today().isoformat()}.log"
+
+    def log(msg: str) -> None:
+        line = f"{dt.datetime.now():%H:%M:%S} {msg}"
+        print(line, flush=True)
+        if not args.dry_run:
+            with log_path.open("a", encoding="utf-8") as fh:
+                fh.write(line + "\n")
+
+    eta = len(todo) * (args.gap + 60) / 3600
+    log(f"{len(todo)} calls queued across {len(roster)} models, {args.gap}s apart"
+        f" (about {eta:.1f} h): {', '.join(roster)}")
+    if args.dry_run:
+        for t, m in todo:
+            print(f"  {t}  {m}")
+        return
+    rested: dict[str, float] = {}
+    while todo:
+        now = time.monotonic()
+        pick = next(((t, m) for t, m in todo if rested.get(m, 0) <= now), None)
+        if pick is None:
+            # Only models with work left: a rest that already ended would make
+            # this negative, and time.sleep raises on a negative wait.
+            time.sleep(max(1, min(rested[m] for _, m in todo) - now + 1))
+            continue
+        todo.remove(pick)
+        task, model = pick
+        subprocess.run([str(PY), str(ROOT / "suite" / "run.py"), "--task", task, "--model", model,
+                        "--timeout", str(args.timeout)], cwd=str(ROOT), capture_output=True,
+                       env={**os.environ, "PYTHONIOENCODING": "utf-8"})
+        if rate_limited(task, model):
+            (TASKS / task / "output" / f"{model}.md").unlink()
+            rested[model] = time.monotonic() + args.rest
+            todo.append(pick)
+            log(f"rate-limited  {model} on {task}; resting it {args.rest}s, retried later")
+        else:
+            log(f"done          {model} on {task}  ({len(todo)} left)")
+        if todo:
+            time.sleep(args.gap)
+    log("queue empty; finished")
+
+
+if __name__ == "__main__":
+    main()