@wendellresearch-ai/benchwarmers 0.3.2 → 0.4.0

package/serve.py

@@ -43,6 +43,7 @@ import re
 import subprocess
 import sys
 import threading
+import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 ROOT = pathlib.Path(__file__).resolve().parent
@@ -54,6 +55,7 @@ PORT = int(os.environ.get("BENCH_PORT") or 8122)
 # Once, at import: roster() used to insert two entries on every page render.
 for _d in ("metrics", "suite", "scripts"):
     sys.path.insert(0, str(ROOT / _d))
+import aliases  # noqa: E402
 from aliases import current as current_alias  # noqa: E402
 import connect  # noqa: E402
 import gateway  # noqa: E402  -- its routes are answered on this port too (one port)
@@ -164,15 +166,25 @@ def suite_scores() -> dict:
     The notes are the record, so they are also the source here rather than a
     parallel database that could disagree with them. Later files win, which is
     what makes a re-run show up.
+
+    Not a vision or image model's (suite/run.images_only): the suite is text
+    and code, which it is never seated for, and the answers it gave before it
+    had that role stay in the notes, uncounted. Real work, perf and reviews
+    still count it. Nor the vision tasks (vision-*): the Vision panel reads
+    those (vision_rows).
     """
+    from run import images_only
     row = re.compile(r"^\|\s*([\w-]+)\s*\|\s*`([^`]+)`\s*\|\s*([^|]+?)\s*\|"
                      r"\s*([^|]*?)\s*\|\s*([\d.]+)\s*\|")
     scores: dict[tuple[str, str], dict] = {}
+    skip = images_only()
     for path in sorted(DATA.glob("runs/suite-*.md")):
         for line in read(path).splitlines():
             if (m := row.match(line.strip())):
                 task, model, found, invented, seconds = m.groups()
                 model = current_alias(model)
+                if model in skip or task.startswith("vision-"):
+                    continue
                 scores[(task, model)] = {
                     "found": found, "invented": invented.strip(" —-"),
                     "seconds": float(seconds)}
@@ -444,6 +456,7 @@ def change_model(alias: str, new_alias: str | None, new_role: str | None,
         history = load_json(HISTORY_PATH)
         history[alias] = new_alias
         _write_atomic(HISTORY_PATH, _dump(history))
+        aliases.forget()                       # its one-second cache, so the next read has the rename
         changed.append("config/alias-history.json")
         # 3. configs keyed by alias
         roles = roles_config()
@@ -773,15 +786,28 @@ def restore_model(alias: str, who: str = "ui") -> dict:
 DISMISSED_PATH = DATA / "config" / "dismissed.json"
 
 
+_modalities_index: tuple = (None, {})
+
+
 def _modelsdev_modalities() -> dict[str, dict]:
     """models.dev's cached modalities by comparable model name
-    (sources.norm_model), read once per picker opening, not once per row."""
+    (sources.norm_model), not once per row, and built again only when the
+    daily fetch rewrites its copy (a second of every picker opening)."""
+    global _modalities_index
     import sources
+    try:
+        st = (sources.CACHE / "modelsdev.json").stat()
+        key = (st.st_mtime_ns, st.st_size)
+    except OSError:
+        key = None
+    if key and _modalities_index[0] == key:
+        return _modalities_index[1]
     out = {}
     for p in sources.cached("modelsdev").values():
         for k, m in ((p or {}).get("models") or {}).items():
             if (m.get("modalities") or {}).get("input"):
                 out.setdefault(sources.norm_model(k), m["modalities"])
+    _modalities_index = (key, out)
     return out
 
 
@@ -797,7 +823,14 @@ def _modalities(entry: dict, served: str, index: dict) -> dict:
     return {"inputs": m.get("input") or [], "outputs": m.get("output") or []}
 
 
-def onboard_candidates() -> dict:
+# What each endpoint answered, for LISTED_FOR seconds: name -> (when, its
+# config, live, conns). Only the asking is kept; what is on the bench, retired
+# or dismissed is taken off fresh on every opening.
+LISTED_FOR = 300
+_listed: dict[str, tuple] = {}
+
+
+def onboard_candidates(fresh: bool = False) -> dict:
     """What every endpoint serves that the bench could onboard, for the picker.
 
     Served, minus registered, minus retired, minus dismissed (those come back
@@ -811,8 +844,13 @@ def onboard_candidates() -> dict:
     keys): provider-tags.json is written by that script on its schedule, so a
     model added to Open WebUI since was "unknown connection" until it ran.
     The file is the fallback when the hub's connections cannot be read.
+
+    An endpoint's answer is reused for LISTED_FOR seconds, unless `fresh` (the
+    picker's Refresh) or its entry in endpoints.json changed since; a failure
+    is never kept. Open WebUI alone can take ten seconds to list.
     """
-    names = list(endpoints.load())
+    config = endpoints.load()
+    names = list(config)
     if not names:
         return {"ok": False, "errors": ["no endpoint yet: add one first (Endpoints, then Manage, then"
                                         " Add an endpoint), and what it serves is listed here"]}
@@ -827,10 +865,13 @@ def onboard_candidates() -> dict:
     from concurrent.futures import ThreadPoolExecutor
 
     def ask(name: str):
+        kept = _listed.get(name)
+        if not fresh and kept and kept[1] == config[name] and time.time() - kept[0] < LISTED_FOR:
+            return kept[2], kept[3], kept[0]
         try:
             live = endpoints.models(name)
         except Exception as exc:                       # noqa: BLE001
-            return exc, []
+            return exc, [], 0
         conns = []
         if endpoints.hub(name) == "openwebui":
             import provider_tags
@@ -838,11 +879,14 @@ def onboard_candidates() -> dict:
                 conns = provider_tags.connection_names(name)
             except Exception:                          # noqa: BLE001 -- the tags file answers instead
                 pass
-        return live, conns
+        _listed[name] = (time.time(), config[name], live, conns)
+        return live, conns, _listed[name][0]
     with ThreadPoolExecutor(max_workers=len(names)) as pool:   # all at once, not in turn
         answers = dict(zip(names, pool.map(ask, names)))
+    # The oldest answer shown, so the picker can say how old its list is.
+    listed = min((at for _, _, at in answers.values() if at), default=time.time())
     mods = _modelsdev_modalities()
-    for name, (live, conns) in answers.items():
+    for name, (live, conns, _) in answers.items():
         if isinstance(live, Exception):
             errors.append(f"cannot reach {name}: {live}")
             continue
@@ -869,7 +913,8 @@ def onboard_candidates() -> dict:
             (gone if key in dismissed else open_).append(row)
     if not open_ and not gone and errors:
         return {"ok": False, "errors": errors}
-    return {"ok": True, "errors": errors, "candidates": open_, "dismissed": gone}
+    return {"ok": True, "errors": errors, "candidates": open_, "dismissed": gone,
+            "age": round(time.time() - listed)}
 
 
 def dismiss(served, undo: bool = False, who: str = "ui") -> dict:
@@ -1524,11 +1569,15 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
 # Quick: the probe only; the scheduled `suite --next` passes fill the suite in,
 # six tasks a run, so the same calls are spread out, not saved. Quick + ad
 # hoc: the probe only, and the bench never calls it on its own: the only one
-# that saves calls, for a metered, capped or trial key.
+# that saves calls, for a metered, capped or trial key. Apart from the mode,
+# `vision` (the picker's Vision model box) gives it role vision as it is
+# registered: a seat only for requests that carry images, and its suite is the
+# vision tasks alone (suite/run.applies), now or through the scheduled passes.
 ONBOARD_MODES = {"full": [], "quick": ["--skip-suite"], "quick-adhoc": ["--skip-suite", "--ad-hoc"]}
 
 
-def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full") -> dict:
+def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str = "full",
+                  vision: bool = False) -> dict:
     """Kick off `scripts/onboard.py` for a served model, detached, logging to runs/.
 
     The script does its own checks in the right order (is it served, derive
@@ -1538,7 +1587,7 @@ def start_onboard(served: str, endpoint: str = "", who: str = "ui", mode: str =
     hands the rest to the script. It runs for minutes and spends calls, so it
     runs in the background and the page tails its log.
     """
-    return _spawn_onboard(served, endpoint, who, mode)[0]
+    return _spawn_onboard(served, endpoint, who, mode, vision)[0]
 
 
 def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str]:
@@ -1567,7 +1616,7 @@ def _onboard_refusal(served: str, endpoint: str, mode: str = "full") -> list[str
 
 
 def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
-                   mode: str = "full") -> tuple[dict, subprocess.Popen | None]:
+                   mode: str = "full", vision: bool = False) -> tuple[dict, subprocess.Popen | None]:
     """start_onboard's result, and the process (None when refused) for a caller that waits on it."""
     served = (served or "").strip()
     endpoint = (endpoint or "").strip() or endpoints.default()
@@ -1579,7 +1628,7 @@ def _spawn_onboard(served: str, endpoint: str = "", who: str = "ui",
     log = DATA / "runs" / f"onboard-{stamp}-{safe}.log"
     from run import bench_python
     cmd = [bench_python(), str(ROOT / "scripts" / "onboard.py"), served, "--endpoint", endpoint,
-           *ONBOARD_MODES[mode]]
+           *ONBOARD_MODES[mode], *(["--vision"] if vision else [])]
     # The log handle goes straight to the child, unbuffered, so the page sees
     # each step as it prints. The old `cmd /c ... > log & echo` wrapper with
     # DETACHED_PROCESS sent the output to a console window instead, so every
@@ -1648,19 +1697,19 @@ def queue_onboard(items, who: str = "ui") -> dict:
         endpoint = str(it.get("endpoint") or "").strip() or endpoints.default()
         mode = str(it.get("mode") or "full")
         errors += _onboard_refusal(served, endpoint, mode)
-        picked.append((served, endpoint, mode))
+        picked.append((served, endpoint, mode, it.get("vision") is True))
     with _OQ_LOCK:
         waiting = {(i["served"], i["endpoint"]) for i in _OQ["items"] if i["state"] in ("queued", "running")}
-        errors += [f"{s} from {e} is already in the queue" for s, e, _ in picked if (s, e) in waiting]
-        if len({(s, e) for s, e, _ in picked}) < len(picked):
+        errors += [f"{s} from {e} is already in the queue" for s, e, *_ in picked if (s, e) in waiting]
+        if len({(s, e) for s, e, *_ in picked}) < len(picked):
             errors.append("a model is picked twice")
         if errors:
             return {"ok": False, "errors": errors}
         if _OQ["worker"] is None:
             _OQ["items"] = []
         _OQ["stop"] = False                              # added after a Stop: these run
-        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "state": "queued", "alias": None, "log": None,
-                          "exit": None, "note": ""} for s, e, m in picked]
+        _OQ["items"] += [{"served": s, "endpoint": e, "mode": m, "vision": v, "state": "queued", "alias": None,
+                          "log": None, "exit": None, "note": ""} for s, e, m, v in picked]
         if _OQ["worker"] is None:
             _OQ["worker"] = threading.Thread(target=_onboard_worker, args=(who,), daemon=True)
             _OQ["worker"].start()
@@ -1712,7 +1761,8 @@ def _onboard_worker(who: str) -> None:
                 return
             nxt["state"] = "running"
         try:
-            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"))
+            res, proc = _spawn_onboard(nxt["served"], nxt["endpoint"], who, nxt.get("mode", "full"),
+                                       nxt.get("vision", False))
             if proc is None:                             # refused now: the bench changed since it was queued
                 state, note, code = "failed", "; ".join(res["errors"]), None
             else:
@@ -2174,13 +2224,34 @@ def _family(task: str) -> str:
     return "hand-built"
 
 
+_wrong: tuple = (0.0, {})
+
+
+def wrong_table() -> dict:
+    """wrong.table(), shared by the three cards of one render: it scores every
+    suite answer, and asking three times was a second of each reload."""
+    global _wrong
+    import wrong
+    from run import images_only
+    seen = _wrong
+    if time.monotonic() - seen[0] > 5:
+        table = wrong.table()
+        # A vision or image model keeps its reviews (real work, judged) but not
+        # its code-task answers: the suite is not what it is for (suite_scores).
+        for alias in images_only() & set(table):
+            r = table[alias]
+            r.update(answers=0, wrong=0, by={k: [0, 0] for k in r["by"]}, hunt=dict.fromkeys(r["hunt"], 0))
+        seen = _wrong = (time.monotonic(), table)    # stamped once scored, not before
+    return seen[1]
+
+
 def charts_data() -> dict:
     """Everything the Benchmarks charts draw, for bench.js. Registered models
     only: the tables keep retired ones behind "Show retired", a chart has no toggle."""
     import route
     import wrong
     live = on_bench()
-    recall, wr = route.suite_recall(), wrong.table()
+    recall, wr = route.suite_recall(), wrong_table()
     overview = [{"model": m, "recall": recall[m][0] / recall[m][1], "wrong": wr[m]["wrong"] / wr[m]["answers"],
                  "found": f"{recall[m][0]}/{recall[m][1]}", "wrongN": f"{wr[m]['wrong']}/{wr[m]['answers']}"}
                 for m in sorted(live) if recall.get(m) and recall[m][1] and wr.get(m, {}).get("answers")]
@@ -2198,7 +2269,7 @@ def charts_data() -> dict:
     return {"overview": overview, "heat": heat, "families": ["hand-built", "claims", "seeded, function",
                                                              "seeded, whole file", "real bugs"],
             "wrong": bars, "wrongTotals": totals, "kinds": list(wrong.KINDS), "hunt": hunt,
-            "work": work_rows(), **perf_charts()}
+            "work": work_rows(), "vision": vision_rows(), **perf_charts()}
 
 
 def work_rows() -> list[dict]:
@@ -2215,6 +2286,184 @@ def work_rows() -> list[dict]:
             for m, s in sorted(summary().items()) if m in live and s["judged"]]
 
 
+VISION_STATES = ("right", "misread", "left out", "trap taken", "no answer")
+VISION_SEATS = ("checked, helped", "checked, did not help", "useful, no claims", "no use, no claims",
+                "not judged", "no answer")
+_vision_work: tuple = (None, {})
+
+
+def vision_rows() -> dict:
+    """The Vision panel's evidence, in two kinds kept apart.
+
+    TASKS: the vision tasks' saved answers (suite/tasks/vision-*), scored by
+    the suite's own scorer and then field by field: right, misread (the line
+    is there with another value), left out, trap taken (a value for something
+    the picture does not show, or a switch that is off called on), no answer.
+    The keys exist by construction (suite/vision_tasks.py), so this is the
+    controlled comparison.
+
+    REAL WORK: every seat of the gateway's vision panels (runs/gateway/
+    calls.jsonl, shape vision) and how it was judged (metrics/offloads.jsonl).
+    A verdict that lists claims was checked claim by claim against the
+    picture; one without claims says only useful or not, which for thicken's
+    face corners means it agreed with another seat. Agreement is not a check
+    (the bench's own rule: seats fail together), so the two are never added.
+    """
+    import run
+    live = on_bench()
+    heat, tasks = [], {}
+    for path in sorted((DATA / "suite" / "tasks").glob("vision-*/task.json")):
+        spec = json.loads(read(path))
+        if spec.get("draft"):
+            continue
+        names = list(dict.fromkeys(c["label"].split(":")[0] for c in spec["must_find"]))
+        traps = {c["label"].split(":")[0] for c in spec.get("must_not_claim", [])}
+        tasks[spec["id"]] = {"title": spec["title"], "fields": names, "family": _vision_family(spec)}
+        for answer in sorted((path.parent / "output").glob("*.md")):
+            alias = current_alias(answer.stem)
+            if alias not in live:
+                continue
+            text = read(answer)
+            s = run.score(spec, text)
+            missed = {lbl.split(":")[0] for lbl in s["missed"]}
+            invented = {lbl.split(":")[0] for lbl in s["invented"]}
+            for name in names:
+                there = re.search(rf"(?m)^[^\w\n]*{re.escape(name).replace('_', '[ _]')}[^\w\n]*:", text.lower())
+                state = ("no answer" if s["kind"] != "answered" else "trap taken" if name in invented
+                         else "right" if name not in missed else "misread" if there else "left out")
+                heat.append({"model": alias, "task": spec["id"], "field": name, "key": f"{spec['id']}·{name}",
+                             "state": state, "trap": name in traps})
+    models: dict[str, dict] = {}
+    for h in heat:
+        m = models.setdefault(h["model"], {"asked": set(), "answered": set(), "fields": 0, "right": 0,
+                                           "traps": 0, "taken": 0})
+        m["asked"].add(h["task"])
+        if h["state"] != "no answer":
+            m["answered"].add(h["task"])
+            m["fields"] += 1
+            m["right"] += h["state"] == "right"
+            m["traps"] += h["trap"]
+            m["taken"] += h["state"] == "trap taken"
+    summary = {a: {"asked": len(m["asked"]), "answered": len(m["answered"]), "fields": m["fields"],
+                   "right": m["right"], "traps": m["traps"], "taken": m["taken"]} for a, m in models.items()}
+    return {"tasks": tasks, "heat": heat, "summary": summary, "states": list(VISION_STATES),
+            "seatKinds": list(VISION_SEATS), "work": _vision_work_rows(live), "held": run.vision_held()}
+
+
+VISION_FAMILIES = (("vision-cord-", "receipts"), ("vision-screenqa-", "screens"), ("vision-tablevqa-", "tables"),
+                   ("vision-pope-", "objects"))
+
+
+def _vision_family(spec: dict) -> str:
+    """The grid's group: a public source (suite/vision_sample.py), the made
+    pictures (suite/vision_tasks.py), or the install's own (written by hand)."""
+    if spec.get("made_by", "").startswith("suite/vision_tasks.py"):
+        return "made"
+    return next((name for prefix, name in VISION_FAMILIES if spec["id"].startswith(prefix)), "own")
+
+
+def _vision_work_rows(live: set[str]) -> dict:
+    """alias -> {seat kind: count, claims right, claims wrong, seconds}; read
+    again only when the gateway's log or the verdicts change (5 MB a render)."""
+    global _vision_work
+    calls, verdicts = DATA / "runs" / "gateway" / "calls.jsonl", DATA / "metrics" / "offloads.jsonl"
+    try:
+        key = tuple((p.stat().st_mtime_ns, p.stat().st_size) for p in (calls, verdicts)) + (frozenset(live),)
+    except OSError:
+        return {}
+    if _vision_work[0] == key:
+        return _vision_work[1]
+    from offload import entries
+    judged = {e["call"]: e for e in entries() if e.get("call") and e.get("useful") is not None}
+    out: dict[str, dict] = {}
+    for raw in read(calls).splitlines():
+        try:
+            panel = json.loads(raw)
+        except ValueError:
+            continue
+        if panel.get("shape") != "vision":
+            continue
+        for s in panel.get("seats", []):
+            alias = current_alias(s.get("alias", ""))
+            if alias not in live:
+                continue
+            r = out.setdefault(alias, {**dict.fromkeys(VISION_SEATS, 0), "real": 0, "false": 0, "seconds": []})
+            v = judged.get(s.get("call"))
+            if s.get("status") != "ok" or not (s.get("answer") or "").strip():
+                kind = "no answer"
+            elif v is None:
+                kind = "not judged"
+            elif v.get("real") or v.get("false"):
+                kind = "checked, helped" if v["useful"] else "checked, did not help"
+                r["real"] += len(v.get("real") or [])
+                r["false"] += len(v.get("false") or [])
+            else:
+                kind = "useful, no claims" if v["useful"] else "no use, no claims"
+            r[kind] += 1
+            if kind != "no answer" and s.get("seconds"):
+                r["seconds"].append(float(s["seconds"]))
+    for r in out.values():
+        secs = sorted(r.pop("seconds"))
+        r["seconds"] = secs[len(secs) // 2] if secs else None
+    _vision_work = (key, out)
+    return out
+
+
+def vision_card() -> str:
+    """Reading images: the vision tasks, then real work on images."""
+    v = vision_rows()
+    s, w = v["summary"], v["work"]
+    body = ["<table><thead><tr>", th("Model"), th("Tasks answered", True), th("Fields read right", True),
+            th("Traps taken", True), th("Image seats", True), th("Checked claims right", True),
+            th("Checked claims wrong", True), th("Useful, no claims", True), th("Not judged", True),
+            th("Median seconds", True), "</tr></thead><tbody>"]
+
+    def cell(n, d):
+        return (f"<td class='num' data-sort='{n / d:.3f}'>{n}/{d}</td>" if d
+                else "<td class='num muted' data-sort='-1'>—</td>")
+    for m in sorted(set(s) | set(w)):
+        t, r = s.get(m, {}), w.get(m, {})
+        seats = sum(r.get(k, 0) for k in VISION_SEATS)
+        claims = r.get("real", 0) + r.get("false", 0)
+        body.append(f"{tr(m)}<td>{named(m)}</td>" + cell(t.get("answered", 0), t.get("asked", 0))
+                    + cell(t.get("right", 0), t.get("fields", 0)) + cell(t.get("taken", 0), t.get("traps", 0))
+                    + f"<td class='num'>{seats or '—'}</td>" + cell(r.get("real", 0), claims)
+                    + cell(r.get("false", 0), claims) + f"<td class='num'>{r.get('useful, no claims', 0) or '—'}</td>"
+                    + f"<td class='num'>{r.get('not judged', 0) or '—'}</td>"
+                    + (f"<td class='num'>{r['seconds']:.1f}</td>" if r.get("seconds") else "<td class='num muted'>—</td>")
+                    + "</tr>")
+    body.append("</tbody></table>")
+    key = """<dl>
+      <dt>The vision tasks</dt><dd>Grouped by where the pictures come from: <b>made</b> with their answer
+        key (<code>suite/vision_tasks.py</code>), your <b>own</b> photos keyed by hand, and items sampled
+        from public datasets with people's answers (<code>suite/vision_sample.py</code>): photographed
+        <b>receipts</b>, app <b>screens</b>, <b>tables</b>, and <b>objects</b> in photos. Each field is
+        <b>right</b>, <b>misread</b> (another value), <b>left out</b>, or a <b>trap taken</b>: a value for
+        something the picture does not show, where the right answer is <code>not shown</code> (a dot marks
+        those fields). They run for every model that takes images; a vision model gets only these, never
+        the code tasks.</dd>
+      <dt>Real work on images</dt><dd>Every seat of a vision panel through the gateway. <b>Checked</b> means
+        its verdict lists claims, each compared with the picture by hand: the claims right and wrong
+        come from these alone. <b>Useful, no claims</b> is a verdict without them, which on thicken's
+        photos means the seat agreed with another seat. Agreement is not a check, since seats fail
+        together, so it is never counted as right.</dd>
+      <dt>The number beside a name</dt><dd>How much there is: fields asked on the tasks, seats on real work.</dd>
+    </dl>"""
+    held = (f"<p class='hint'>The vision tasks wait until {esc(v['held'])}: no run picks them before then"
+            " (<code>vision_from</code> in <code>config/suite.json</code>; remove it to start them now).</p>"
+            if v["held"] else "")
+    return ("<div class='panel'><h2>Vision</h2><p class='sub'>Reading images: fields read right on pictures with"
+            " a known answer, then real work on images and how much of it was actually checked.</p>" + held +
+            "<h3 class='chart-head'>On the vision tasks, field by field</h3><div class='chart' id='chart-vision-fields'></div>"
+            "<h3 class='chart-head'>On real work with images, by how each seat was judged</h3>"
+            "<div class='chart' id='chart-vision-work'></div>"
+            "<details class='key'><summary>Table</summary><div class='toolbar'><input type='search'"
+            " data-filter placeholder='Filter by model or connection' aria-label='Filter rows'>"
+            "<label><input type='checkbox' data-retired> Show retired</label></div>"
+            f"<div class='scroll'>{''.join(body)}</div></details>"
+            f"<details class='key'><summary>How to read this</summary>{key}</details></div>")
+
+
 def perf_charts() -> dict:
     """Reliability over the last 14 days of probes, and speed on real tasks
     against how much a model finds, for the Performance panel of Benchmarks."""
@@ -2239,8 +2488,7 @@ def perf_charts() -> dict:
 
 def bughunt_card() -> str:
     """Seeded and real-bug tasks against their answer keys."""
-    import wrong
-    rows = {m: r["hunt"] for m, r in wrong.table().items() if m in current() and any(r["hunt"].values())}
+    rows = {m: r["hunt"] for m, r in wrong_table().items() if m in current() and any(r["hunt"].values())}
     body = ["<table><thead><tr>", th("Model"), th("Found", True), th("Wrong line", True), th("Missed", True),
             th("Clean said clean", True), th("False alarm", True), "</tr></thead><tbody>"]
     for m in sorted(rows, key=lambda m: -rows[m]["found"]):
@@ -2360,8 +2608,9 @@ def external_card() -> str:
 def wrong_card() -> str:
     """metrics/wrong.py: how often each model asserts something false."""
     import wrong
-    rows = wrong.table()
-    live = current()
+    from run import images_only
+    rows = wrong_table()
+    live, images = current(), images_only()
     body = ["<table><thead><tr>", th("Model"), th("Confidently wrong", True),
             *[th(k.capitalize(), True) for k in wrong.KINDS], th("Reviews", True), "</tr></thead><tbody>"]
 
@@ -2378,7 +2627,8 @@ def wrong_card() -> str:
                      f" style='width:{max(4, round(rate * 100))}%'></span></span>"
                      f"<span class='mono'>{r['wrong']}/{r['answers']}</span></span>")
         else:
-            rate, meter = -1, "<span class='muted'>no tasks yet</span>"
+            rate, meter = -1, ("<span class='muted'>reads images: no code tasks</span>" if alias in images
+                               else "<span class='muted'>no tasks yet</span>")
         body.append(f"<tr><td>{named(alias)}</td><td data-sort='{rate:.3f}'>{meter}</td>"
                     + "".join(cell(*r["by"][k]) for k in wrong.KINDS)
                     + (f"<td class='num' data-sort='{r['review_false']}'>{r['review_false']} in {r['reviews']}</td>"
@@ -2404,7 +2654,8 @@ def wrong_card() -> str:
 
 def benchmarks_tab() -> str:
     data = json.dumps(charts_data(), ensure_ascii=False).replace("</", "<\\/")
-    return (overview_card() + bughunt_card() + work_card() + wrong_card() + suite_card() + perf_card() + external_card()
+    return (overview_card() + bughunt_card() + work_card() + wrong_card() + suite_card() + vision_card()
+            + perf_card() + external_card()
             + f"<script type='application/json' id='charts-data'>{data}</script>")
 
 
@@ -3620,6 +3871,9 @@ def editor() -> str:
         " with progress in the header. A model from somewhere else? Add its endpoint first.</p>"
         "<input type='search' id='o-find' placeholder='Search models or connections'"
         " aria-label='Search models'>"
+        # How old the list is (serve.LISTED_FOR), and asking again now.
+        "<p class='muted' id='o-age' hidden><span></span>"
+        " <button class='linkish' type='button' id='o-refresh'>Refresh</button></p>"
         "<div class='picker' id='o-picker' role='group' aria-label='Served models'></div>"
         # serve.ONBOARD_MODES: how much each onboarding spends.
         "<label class='check'>Onboard as <select id='o-mode'>"
@@ -3627,6 +3881,8 @@ def editor() -> str:
         "<option value='quick'>Quick: probe now, the suite through the scheduled passes</option>"
         "<option value='quick-adhoc'>Quick + ad hoc: probe only; the bench never calls it on its own"
         " (metered, capped or trial keys)</option></select></label>"
+        "<label class='check'><input type='checkbox' id='o-vision'> Vision model: a seat only for questions"
+        " about images, and its suite is the vision tasks</label>"
         "</div>"
         "<div id='o-run' hidden><p class='served' id='o-result'></p><ol class='steps' id='o-steps'></ol>"
         "<details><summary class='muted'>Full log</summary><pre class='log' id='o-log'></pre></details></div>"
@@ -3761,8 +4017,8 @@ class Handler(gateway.Handler):
             log = parse_qs(urlparse(self.path).query).get("log", [""])[0]
             self._json(200, onboard_status(log))
             return
-        if self.path == "/api/candidates":
-            self._json(200, onboard_candidates())
+        if self.path in ("/api/candidates", "/api/candidates?fresh=1"):
+            self._json(200, onboard_candidates(fresh=self.path.endswith("fresh=1")))
             return
         if self.path == "/api/onboard-queue":
             self._json(200, onboard_queue())
@@ -3881,7 +4137,8 @@ class Handler(gateway.Handler):
                                                            endpoint=str(p.get("endpoint") or "")),
                "/api/onboard": lambda p: start_onboard(str(p.get("served", "")),
                                                        str(p.get("endpoint", "")),
-                                                       mode=str(p.get("mode") or "full")),
+                                                       mode=str(p.get("mode") or "full"),
+                                                       vision=p.get("vision") is True),
                "/api/onboard-queue": lambda p: queue_onboard(p.get("items")),
                "/api/onboard-queue/stop": lambda p: stop_onboard_queue(),
                "/api/onboard-queue/clear": lambda p: clear_onboard_queue(),

package/suite/README.md

@@ -107,6 +107,39 @@ The scorer ignores a `must_not_claim` phrase where it is negated in its own
 sentence ("there are no INSERT statements"), so ruling something out is never
 counted as claiming it.
 
+## Vision tasks
+
+A fifth shape, added 2026-10-27: **reading images**. A task with `"shape":
+"vision"` and `"images"` (files in its `fixtures/`) is sent with its pictures
+as image parts, through the built-in client, to every model the catalog says
+takes images; a vision model (role `vision`) gets only these, and a model that
+cannot see never gets one (`run.applies`). The answer is `key: value` lines,
+`not shown` for what the picture does not show, scored by
+`suite/vision_keys.py`: a value for a field the picture cannot answer is an
+invention, like a trap. They stay out of the routing recall and the code
+reports; the Benchmarks page's Vision panel shows them field by field.
+
+Three sources:
+
+- **Made pictures** (`suite/vision_tasks.py`, needs Pillow to regenerate): a
+  receipt, a grading label, a bar chart, a table, a settings screen, shapes
+  and an invoice turned sideways, each key built from the data that drew it.
+  Shipped in `suite/tasks/vision-*` and copied into the data folder on the
+  next run (`run.seed_shipped`). Never seen in training; clean renders.
+- **Public datasets** (`python suite/vision_sample.py --count 5`): photographed
+  receipts (CORD v2), app screens with an unanswerable question (RICO-ScreenQA),
+  tables (TableVQA-Bench) and absent-object questions (POPE), with the
+  datasets' human answers. Written into the data folder only, so each install
+  samples its own and nothing is redistributed.
+- **Your own**: any task folder you write into the data folder's
+  `suite/tasks/vision-*` in the same form, from photos whose answers you have
+  checked. Real pictures of the real job, never in anyone's training data:
+  the strongest of the three.
+
+To hold them back for a while (a hub being changed, say), put the day they may
+start in the data folder's `config/suite.json`: `{"vision_from": "2026-10-30"}`.
+Until then no run picks a vision task; `--task <id>` still runs one.
+
 ## Adding a task
 
 A task is one directory under `suite/tasks/` containing:

package/suite/pace.py

@@ -72,10 +72,14 @@ def answered(task: str, model: str) -> bool:
 
 
 def queue(prefix: str, roster: list[str]) -> list[tuple[str, str]]:
-    """(task, model) pairs without an answer yet, task by task across models."""
-    tasks = sorted(p.parent.name for p in TASKS.glob(f"{prefix}*/task.json")
-                   if not json.loads(p.read_text(encoding="utf-8")).get("draft"))
-    return [(t, m) for t in tasks for m in roster if not answered(t, m)]
+    """(task, model) pairs without an answer yet, task by task across models,
+    each only where the task is the model's (run.applies)."""
+    specs = [s for s in run.load_tasks(None) if s["id"].startswith(prefix)]
+    skip, seeing = run.images_only(), {}
+    if any(s.get("images") for s in specs):
+        seeing = {m: run.sees_images(m) for m in roster}
+    return [(s["id"], m) for s in specs for m in roster
+            if run.applies(s, m, skip, seeing) and not answered(s["id"], m)]
 
 
 def rate_limited(task: str, model: str) -> bool: