@wendellresearch-ai/benchwarmers 0.3.3 → 0.4.0

package/suite/test_vision_tasks.py

@@ -0,0 +1,201 @@
+"""The vision tasks: each answer key takes a right answer whole and catches a
+guess on its trap; a vision task goes only to a model that takes images, a
+text task only to one that is not images only; the shipped tasks reach the
+data folder without touching its saved answers.
+
+    python suite/test_vision_tasks.py
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import sys
+import tempfile
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import run  # noqa: E402
+
+RIGHT = {
+    "vision-receipt": "store: Harbor Lane Grocery\ndate: 2026-03-14\nitems: 4\npriciest: Cheddar 200g\n"
+                      "total: $22.17\ntip: not shown",
+    "vision-label": "**grader:** Northline Grading\ngrade: MINT 9\nyear: 2019\nnumber: #148\nplayer: Maren Okafor\n"
+                    "cert: 40731958\nsurface: 8.5\nautograph: no",
+    "vision-bars": "tallest: Billing\nshortest: Data\nweb: 35\nover_30: 2\ntarget: not shown",
+    "vision-table": "south_q3: 1,120\nwest_q1: 870\ncentral_q3: 1098\nq4_highest: East\nnortheast_q2: not shown",
+    "vision-screen": "toggles_on: Back up photos, Notify on conflicts\nconflict: Keep both\ndisabled: Sync now\n"
+                     "error_code: E-213\naccount: not shown",
+    "vision-shapes": "- blue_circles: 6\n- red_squares: 3\n- green_triangles: 2\n- top_right: yellow star\n"
+                     "- purple_shapes: 0",
+    "vision-invoice": "invoice: INV-2026-0417\ndue: 2026-05-02\namount_due: EUR 1,284.50\nbill_to: Kestrel Labs BV\n"
+                      "po_number: not shown",
+}
+# Each one guesses the trap, and gets one field wrong besides.
+GUESS = {
+    "vision-receipt": RIGHT["vision-receipt"].replace("tip: not shown", "tip: $3.00").replace("items: 4", "items: 5"),
+    "vision-label": RIGHT["vision-label"].replace("autograph: no", "autograph: yes").replace("40731958", "40781958"),
+    "vision-bars": RIGHT["vision-bars"].replace("target: not shown", "target: 40").replace("over_30: 2", "over_30: 3"),
+    "vision-table": RIGHT["vision-table"].replace("northeast_q2: not shown", "northeast_q2: 1,210")
+                                         .replace("central_q3: 1098", "central_q3: 1160"),
+    "vision-screen": RIGHT["vision-screen"].replace("account: not shown", "account: sam@example.com")
+                                           .replace("Notify on conflicts", "Notify on conflicts, Sync on mobile data"),
+    "vision-shapes": RIGHT["vision-shapes"].replace("purple_shapes: 0", "purple_shapes: 1")
+                                           .replace("blue_circles: 6", "blue_circles: 5"),
+    "vision-invoice": RIGHT["vision-invoice"].replace("po_number: not shown", "po_number: 7781")
+                                             .replace("due: 2026-05-02", "due: 2026-04-02"),
+}
+
+
+def main() -> None:
+    specs = {}
+    for path in sorted((HERE / "tasks").glob("vision-*/task.json")):
+        spec = json.loads(path.read_text(encoding="utf-8"))
+        spec["dir"] = path.parent
+        specs[spec["id"]] = spec
+    assert set(specs) == set(RIGHT), sorted(specs)
+    for tid, spec in specs.items():
+        assert spec["shape"] == "vision" and spec["images"], tid
+        assert all((spec["dir"] / "fixtures" / n).exists() for n in spec["images"]), tid
+        good = run.score(spec, RIGHT[tid])
+        assert good["kind"] == "answered" and good["found"] == good["of"] and not good["invented"], (tid, good)
+        bad = run.score(spec, GUESS[tid])
+        assert bad["found"] < bad["of"] and bad["invented"], (tid, bad)
+        assert run.image_urls(spec)[0].startswith("data:image/png;base64,"), tid
+
+    # Who each task is for: pictures to a model that takes them, text to one that is not images only.
+    text, pic = {"id": "t"}, {"id": "v", "images": ["x.png"]}
+    skip, seeing = {"eyes-only"}, {"eyes-only": True, "general-sees": True, "general-blind": False}
+    assert run.applies(text, "general-sees", skip, seeing) and run.applies(pic, "general-sees", skip, seeing)
+    assert run.applies(text, "general-blind", skip, seeing) and not run.applies(pic, "general-blind", skip, seeing)
+    assert not run.applies(text, "eyes-only", skip, seeing) and run.applies(pic, "eyes-only", skip, seeing)
+
+    # The shipped tasks reach the data folder; its saved answers stay.
+    kept = run.TASKS, run.SETTINGS
+    with tempfile.TemporaryDirectory() as tmp:
+        run.TASKS = pathlib.Path(tmp)
+        run.SETTINGS = pathlib.Path(tmp) / "suite.json"
+        try:
+            # A hold keeps them out of every run until its day; --task still runs one.
+            run.SETTINGS.write_text(json.dumps({"vision_from": "2999-01-01"}), encoding="utf-8")
+            assert run.vision_held() == "2999-01-01"
+            assert not [t for t in run.load_tasks(None) if t.get("images")]
+            assert run.load_tasks("vision-bars")[0]["images"]
+            run.SETTINGS.write_text(json.dumps({"vision_from": "2000-01-01"}), encoding="utf-8")
+            assert run.vision_held() == "" and len([t for t in run.load_tasks(None) if t.get("images")]) == len(RIGHT)
+            saved = run.TASKS / "vision-bars" / "output" / "m.md"
+            saved.parent.mkdir(parents=True)
+            saved.write_text("an answer", encoding="utf-8")
+            (run.TASKS / "vision-bars" / "task.json").write_text("{}", encoding="utf-8")   # an old key
+            run.seed_shipped()
+            for tid in RIGHT:
+                assert (run.TASKS / tid / "task.json").read_bytes() == (HERE / "tasks" / tid / "task.json").read_bytes()
+            assert saved.read_text(encoding="utf-8") == "an answer"
+        finally:
+            run.TASKS, run.SETTINGS = kept
+    panel()
+    sample()
+    print("ok")
+
+
+def sample() -> None:
+    """vision_sample.py builds each source's key from rows shaped as datasets-server
+    returns them (no network: rows and pictures stand in)."""
+    import vision_sample as vs
+    kept = vs.rows, vs.image
+    receipt = {"menu": [{"nm": "EGG TART", "cnt": "1", "price": "13,000"}, {"nm": "TEA", "cnt": "2", "price": "8,000"}],
+               "total": {"total_price": "21.000", "cashprice": "50.000", "changeprice": "29.000"}}
+    data = {
+        "naver-clova-ix/cord-v2": [{"ground_truth": json.dumps({"gt_parse": receipt})},
+                                   {"ground_truth": json.dumps({"gt_parse": {**receipt, "total": {
+                                       **receipt["total"], "creditcardprice": "21.000"}}})}],   # paid by card: skipped
+        "bevaya/RICO-ScreenQA": [
+            {"screen_id": "7", "question": "What is the version?",
+             "ground_truth": [{"full_answer": "It is 1.69", "ui_elements": [{"text": "1.69"}]}]},
+            {"screen_id": "7", "question": "Is GPS on?", "ground_truth": [{"full_answer": "on", "ui_elements": [{"text": "ON"}]}]},
+            {"screen_id": "7", "question": "What is the lawn type?", "ground_truth": [{"full_answer": "<no answer>", "ui_elements": []}]}],
+        "lmms-lab-encoder/POPE": [{"image_source": "COCO_val2014_000000000042", "question": f"Is there a {t} in the image?",
+                                   "answer": a} for t, a in (("dog", "yes"), ("cat", "no"), ("ball", "yes"), ("car", "no"))],
+    }
+    vs.rows = lambda dataset, config, split, offset, length: data.get(dataset, [])
+    vs.image = lambda row, field="image": (b"\xff\xd8", ".jpg")
+    try:
+        (c, _, _), = list(vs.cord(0, 10))
+        (s, _, _), = list(vs.screenqa(0, 10))
+        (p, _, _), = list(vs.pope(0, 10))
+    finally:
+        vs.rows, vs.image = kept
+    for spec, right, wrong in (
+            (c, "total: Rp 21.000\ncash: 50,000\nchange: 29000\nitems: 2\ncard: not shown", "card: 4417"),
+            (s, "\n".join(f"q{n}: {a}" for n, a in enumerate(
+                ["not shown" if "lawn" in ln else "1.69" if "version" in ln else "on"
+                 for ln in s["prompt"].splitlines() if ln.startswith("q")], 1)), None),
+            (p, "\n".join(f"q{n}: {'yes' if ('dog' in ln or 'ball' in ln) else 'no'}"
+                          for n, ln in enumerate((ln for ln in p["prompt"].splitlines() if ln.startswith("q")), 1)), None)):
+        got = run.score(spec, right)
+        assert got["found"] == got["of"] and not got["invented"], (spec["id"], got, right)
+    assert run.score(c, "total: 21.000\ncard: 4417")["invented"], c
+    yes = "\n".join(f"q{n}: yes" for n in range(1, 5))
+    assert len(run.score(p, yes)["invented"]) == 2                     # a yes to each absent thing
+
+
+def panel() -> None:
+    """serve.vision_rows: the tasks field by field, and real work by how it was judged."""
+    sys.path.insert(0, str(HERE.parent))
+    import offload
+    import serve
+    kept = serve.DATA, serve.on_bench, offload.LOG, serve._vision_work
+    with tempfile.TemporaryDirectory() as tmp:
+        data = pathlib.Path(tmp)
+        for tid in ("vision-bars", "vision-shapes"):
+            out = data / "suite" / "tasks" / tid / "output"
+            out.mkdir(parents=True)
+            (out.parent / "task.json").write_bytes((HERE / "tasks" / tid / "task.json").read_bytes())
+            (out / "sharp.md").write_text(RIGHT[tid], encoding="utf-8")
+            (out / "guesser.md").write_text(GUESS[tid], encoding="utf-8")
+            (out / "gone.md").write_text(RIGHT[tid], encoding="utf-8")            # not on the bench now
+        (data / "suite" / "tasks" / "vision-bars" / "output" / "down.md").write_text(
+            "Error: Error code: 503 - service unavailable", encoding="utf-8")
+        seats = [{"call": "p1.m1", "alias": "sharp", "status": "ok", "answer": "x", "seconds": 4},
+                 {"call": "p1.m2", "alias": "guesser", "status": "ok", "answer": "x", "seconds": 9},
+                 {"call": "p2.m1", "alias": "sharp", "status": "ok", "answer": "x", "seconds": 6},
+                 {"call": "p2.m2", "alias": "guesser", "status": "error", "answer": "", "seconds": 1},
+                 {"call": "p3.m1", "alias": "sharp", "status": "ok", "answer": "x", "seconds": 5}]
+        (data / "runs" / "gateway").mkdir(parents=True)
+        (data / "runs" / "gateway" / "calls.jsonl").write_text(
+            json.dumps({"shape": "vision", "seats": seats[:2]}) + "\n" + json.dumps({"shape": "vision", "seats": seats[2:4]})
+            + "\n" + json.dumps({"shape": "general", "seats": [dict(seats[0], call="p9.m1")]})
+            + "\n" + json.dumps({"shape": "vision", "seats": seats[4:]}) + "\n", encoding="utf-8")
+        (data / "metrics").mkdir()
+        log = data / "metrics" / "offloads.jsonl"
+        log.write_text("\n".join(json.dumps(e) for e in (
+            {"date": "2026-10-27", "model": "sharp", "task": "t", "call": "p1.m1", "useful": True,
+             "real": ["grade=9", "cert=1"], "false": []},
+            {"date": "2026-10-27", "model": "guesser", "task": "t", "call": "p1.m2", "useful": False,
+             "real": ["grade=9"], "false": ["cert=2"]},
+            {"date": "2026-10-27", "model": "sharp", "task": "t", "call": "p2.m1", "useful": True,
+             "note": "agreed with another seat"})) + "\n", encoding="utf-8")
+        serve.DATA, offload.LOG, serve._vision_work = data, log, (None, {})
+        serve.on_bench = lambda: {"sharp", "guesser", "down"}
+        try:
+            v = serve.vision_rows()
+        finally:
+            serve.DATA, serve.on_bench, offload.LOG, serve._vision_work = kept
+    s = v["summary"]
+    assert set(s) == {"sharp", "guesser", "down"}, s                                  # "gone" is not on the bench
+    assert s["sharp"] == {"asked": 2, "answered": 2, "fields": 10, "right": 10, "traps": 2, "taken": 0}, s["sharp"]
+    assert s["guesser"]["right"] == 6 and s["guesser"]["taken"] == 2, s["guesser"]
+    assert s["down"] == {"asked": 1, "answered": 0, "fields": 0, "right": 0, "traps": 0, "taken": 0}, s["down"]
+    states = {(h["model"], h["task"], h["field"]): h["state"] for h in v["heat"]}
+    assert states[("guesser", "vision-bars", "over_30")] == "misread"
+    assert states[("guesser", "vision-bars", "target")] == "trap taken"
+    assert states[("down", "vision-bars", "web")] == "no answer"
+    w = v["work"]
+    assert w["sharp"]["checked, helped"] == 1 and w["sharp"]["useful, no claims"] == 1 and w["sharp"]["not judged"] == 1, w
+    assert (w["sharp"]["real"], w["sharp"]["false"], w["sharp"]["seconds"]) == (2, 0, 5.0), w["sharp"]
+    assert w["guesser"]["checked, did not help"] == 1 and w["guesser"]["no answer"] == 1, w["guesser"]
+    assert (w["guesser"]["real"], w["guesser"]["false"]) == (1, 1), w["guesser"]
+
+
+if __name__ == "__main__":
+    main()

package/suite/vision_keys.py

@@ -0,0 +1,74 @@
+"""How every vision task asks and is scored: answers as `key: value` lines,
+a key per field, `not shown` for what the picture does not show.
+
+Shared by suite/vision_tasks.py (the made pictures), suite/vision_sample.py
+(items from public datasets) and any task written by hand, so one answer
+style is scored one way. Standard library only.
+"""
+
+from __future__ import annotations
+
+import re
+
+NOT_SHOWN = (r"(not shown|not visible|not present|not listed|not given|not stated|not on the|not in the"
+             r"|not provided|not included|none shown|n/?a\b|none\b|no such|does not show|doesn't show"
+             r"|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)")
+PROMPT = ("Read the attached image and answer in exactly these lines, one per line, nothing else:\n\n{lines}\n\n"
+          "If the image does not show something, write `not shown` for it. Do not guess.")
+
+
+def key(name: str, value: str) -> str:
+    """A regex for the line `name: value`, markdown around the key allowed."""
+    k = re.escape(name).replace("_", "[ _]")
+    return rf"(?m)^[^\w\n]*{k}[^\w\n]*:[ \t*_`\"']*{value}"
+
+
+def clause(name: str, value: str, label: str) -> dict:
+    return {"regex": key(name, value), "label": label}
+
+
+def not_shown(name: str) -> tuple[dict, dict]:
+    """(must_find, must_not_claim) for a field the picture does not answer:
+    `not shown` is right, any other value an invention."""
+    return (clause(name, NOT_SHOWN, f"{name}: not shown"),
+            clause(name, rf"(?!{NOT_SHOWN}|unknown|unclear)\w", f"{name}: a value (the picture does not answer it)"))
+
+
+def number(text: str) -> str:
+    """A regex for a printed amount however its thousands are separated:
+    "45,500" also matches 45.500, 45 500 and 45500; a currency word or sign
+    before it is allowed."""
+    digits = re.sub(r"[^\d.,]", "", text)
+    m = re.fullmatch(r"(.*?)[.,](\d{1,2})", digits)              # one or two digits after the last mark: cents
+    whole, cents = (m.group(1), m.group(2)) if m else (digits, None)
+    groups = [g for g in re.split(r"[.,]", whole) if g]
+    body = r"[.,\s]?".join(groups)
+    if cents:
+        body += rf"[.,]{cents}"
+    return rf"(?:[a-z]{{1,3}}\.?\s*|[$€£¥]\s*)?{body}(?!\d)"
+
+
+def phrase(text: str) -> str:
+    """A regex for a short answer anywhere on its line, spacing and case loose."""
+    words = [re.escape(w) for w in text.lower().split()]
+    return r".*" + r"\s+".join(words)
+
+
+def _selftest() -> None:
+    n = number("45,500")
+    for ok in ("45,500", "45.500", "45 500", "45500", "rp 45.500", "Rp. 45,500"):
+        assert re.fullmatch(n, ok.lower()), (ok, n)
+    for bad in ("45,5000", "455,00"):
+        assert not re.fullmatch(n, bad), (bad, n)
+    assert re.fullmatch(number("3.14%").removesuffix("(?!\\d)"), "3.14"), number("3.14%")
+    assert re.fullmatch(number("$1,284.50"), "eur 1,284.50") and re.fullmatch(number("23.000"), "23000")
+    find, claim = not_shown("q3")
+    # Answers arrive lowercased (run.score).
+    assert re.search(find["regex"], "q3: not shown") and not re.search(claim["regex"], "q3: not shown")
+    assert re.search(claim["regex"], "q3: 12 questions") and re.search(find["regex"], "**q3**: cannot be determined")
+    assert re.search(key("q1", phrase("Unauthorized transactions")), "q1: from unauthorized  transactions.")
+
+
+if __name__ == "__main__":
+    _selftest()
+    print("ok")

package/suite/vision_sample.py

@@ -0,0 +1,232 @@
+"""Vision tasks sampled from public datasets, each item's answer verified by people.
+
+    python suite/vision_sample.py --count 5                  # five of each source
+    python suite/vision_sample.py --source cord --count 3    # one source
+    python suite/vision_sample.py --dry-run                  # what would be written
+
+The made pictures (suite/vision_tasks.py) are clean renders: exact keys, never
+seen in training, but no blur, glare or crumpled paper. These are the real
+thing, read through Hugging Face's datasets-server like suite/rebench.py, no
+`datasets` package:
+
+  cord      naver-clova-ix/cord-v2 (CC BY 4.0): photographed receipts with
+            their totals, cash and change as annotated. Cash receipts only, so
+            a card number is a field the picture does not show.
+  screenqa  bevaya/RICO-ScreenQA (CC BY 4.0): app screenshots, two questions
+            people answered from the screen and one they marked `<no answer>`.
+  tablevqa  terryoo/TableVQA-Bench (CC BY 4.0): tables from financial reports
+            and Wikipedia as images, a short answer each.
+  pope      lmms-lab-encoder/POPE (no licence on its card; COCO photos): "is
+            there a <thing> in the image?", two present and two absent, the
+            absent ones chosen to be likely (its adversarial split). A yes to
+            an absent thing is the invention.
+
+Written to the data folder, never the repository: each install samples its
+own, so nothing of theirs is redistributed. Written ready to run, not as
+drafts: the answers are the datasets' own, checked by people. Look at a few
+anyway (`--dry-run` prints each key); a source's quirks show in its first items.
+
+POPE's photos are often in training sets (COCO is everywhere); the receipts,
+screens and tables less so, and the made pictures and your own photos not
+at all. The Vision panel shows each task, so a source that is too easy shows.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import pathlib
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from rebench import api  # noqa: E402  -- datasets-server JSON, retrying its busy 500s
+from vision_keys import NOT_SHOWN, PROMPT, clause, key, not_shown, number, phrase  # noqa: E402
+
+sys.path.insert(0, str(HERE.parent / "metrics"))
+from paths import DATA  # noqa: E402
+TASKS = DATA / "suite" / "tasks"
+PAGE = 100                                           # datasets-server's largest page
+
+
+def rows(dataset: str, config: str, split: str, offset: int, length: int) -> list[dict]:
+    out = []
+    while len(out) < length:
+        page = api("rows", dataset=dataset, config=config, split=split, offset=offset + len(out),
+                   length=min(PAGE, length - len(out)))
+        got = [r["row"] for r in page.get("rows", [])]
+        out += got
+        if len(got) < PAGE:
+            break
+    return out
+
+
+def image(row: dict, field: str = "image") -> tuple[bytes, str]:
+    """The row's picture, fetched now: datasets-server's links expire."""
+    src = row[field]["src"]
+    req = urllib.request.Request(src, headers={"User-Agent": "benchwarmers"})
+    with urllib.request.urlopen(req, timeout=90) as r:
+        data = r.read()
+    ext = pathlib.PurePosixPath(urllib.parse.urlparse(src).path).suffix.lower() or ".jpg"
+    return data, ext
+
+
+def alternatives(name: str, answers: list[str], label: str) -> dict:
+    """Any of several accepted answers, anywhere on the field's line."""
+    alts = "|".join(phrase(a)[2:] for a in answers)
+    return clause(name, rf".*(?:{alts})", label)
+
+
+def spec(task_id: str, title: str, source: str, dataset: str, row_ref: str, licence: str,
+         lines: list[str], find: list[dict], not_claim: list[dict], why: str) -> dict:
+    return {"id": task_id, "title": title, "shape": "vision", "prompt": PROMPT.format(lines="\n".join(lines)),
+            "min_answer_chars": 8, "must_find": find, "must_not_claim": not_claim, "why": why,
+            "source": f"{dataset} {row_ref}", "source_url": f"https://huggingface.co/datasets/{dataset}",
+            "license": licence, "made_by": f"suite/vision_sample.py --source {source}"}
+
+
+# ---- the sources: each yields (spec without images, picture bytes, extension) ----
+
+def cord(offset: int, scan: int):
+    """Cash receipts: total, cash given, change, how many menu lines."""
+    ds = "naver-clova-ix/cord-v2"
+    for i, r in enumerate(rows(ds, "default", "test", offset, scan)):
+        gt = json.loads(r["ground_truth"]).get("gt_parse", {})
+        menu, total = gt.get("menu"), gt.get("total") or {}
+        menu = [menu] if isinstance(menu, dict) else menu or []
+        paid = {k: total.get(k) for k in ("total_price", "cashprice", "changeprice")}
+        if (not all(isinstance(v, str) and re.search(r"\d", v) for v in paid.values())
+                or any(k in total for k in ("creditcardprice", "emoneyprice"))
+                or not 1 <= len(menu) <= 8 or any("sub" in m for m in menu)):
+            continue
+        find = [clause("total", number(paid["total_price"]), f"total: {paid['total_price']}"),
+                clause("cash", number(paid["cashprice"]), f"cash: {paid['cashprice']}"),
+                clause("change", number(paid["changeprice"]), f"change: {paid['changeprice']}"),
+                clause("items", rf"{len(menu)}\b", f"items: {len(menu)}")]
+        trap_find, trap_claim = not_shown("card")
+        yield (spec(f"vision-cord-{offset + i}", "Read a photographed receipt", "cord", ds, f"test row {offset + i}",
+                    "CC BY 4.0",
+                    ["total: <the total to pay>", "cash: <cash given>", "change: <change given back>",
+                     "items: <how many different items are listed>", "card: <the last four digits of the card used>"],
+                    find + [trap_find], [trap_claim],
+                    "A receipt as photographed, from CORD v2 (Park et al., 2019). Paid in cash, so no card is"
+                    " shown. Items: " + "; ".join(f"{m.get('nm', '?')} x{m.get('cnt', '?')}" for m in menu)),
+               *image(r))
+
+
+def screenqa(offset: int, scan: int):
+    """App screens: two questions answered from the screen, one it cannot answer."""
+    ds = "bevaya/RICO-ScreenQA"
+    screens: dict[str, list[dict]] = {}
+    for r in rows(ds, "default", "test", offset, scan):
+        screens.setdefault(r["screen_id"], []).append(r)
+    for sid, qs in screens.items():
+        def short(q):
+            return sorted({g["ui_elements"][0]["text"].strip() for g in q["ground_truth"] if g["ui_elements"]})
+        answered = [q for q in qs if 1 <= len(short(q)) <= 2 and all(0 < len(a) <= 40 for a in short(q))]
+        blank = [q for q in qs if all(g["full_answer"] == "<no answer>" for g in q["ground_truth"])]
+        if len(answered) < 2 or not blank:
+            continue
+        picked = answered[:2]
+        at = int(hashlib.sha1(sid.encode()).hexdigest(), 16) % 3      # where the unanswerable one sits
+        picked.insert(at, blank[0])
+        lines, find, claim = [], [], []
+        for n, q in enumerate(picked, 1):
+            name = f"q{n}"
+            lines.append(f"{name}: <answer to: {q['question']}>")
+            if q is blank[0]:
+                f, c = not_shown(name)
+                find.append(f)
+                claim.append(c)
+            else:
+                find.append(alternatives(name, short(q), f"{name}: {' or '.join(short(q))}"))
+        yield (spec(f"vision-screenqa-{sid}", "Answer questions about an app screen", "screenqa", ds,
+                    f"screen {sid}", "CC BY 4.0", lines, find, claim,
+                    "A RICO app screenshot with questions from ScreenQA (Hsiao et al., 2022); people marked one"
+                    " as having no answer on this screen."),
+               *image(picked[0]))
+
+
+def tablevqa(offset: int, scan: int):
+    """Tables as images: one question, one short answer."""
+    ds = "terryoo/TableVQA-Bench"
+    splits = ("fintabnetqa", "vwtq")
+    pages = [list(enumerate(rows(ds, "default", s, offset, scan // 2))) for s in splits]
+    # The two kinds in turn: a report's table, then a Wikipedia one.
+    for (i, r), split in ((item, s) for pair in zip(*pages) for item, s in zip(pair, splits)):
+        gt = (r.get("gt") or "").strip().rstrip(".")
+        if not gt or len(gt) > 24 or "," in gt and not re.fullmatch(r"[\d,.$%()-]+", gt):
+            continue
+        value = number(gt) if re.fullmatch(r"[\d,.$%()\s-]+", gt) and re.search(r"\d", gt) else phrase(gt)[2:]
+        yield (spec(f"vision-tablevqa-{split}-{offset + i}", "Read a table", "tablevqa", ds,
+                    f"{split} row {offset + i}", "CC BY 4.0", [f"answer: <answer to: {r['question']}>"],
+                    # Anywhere on the line, but whole: 3.14 is not the end of 13.14.
+                    [clause("answer", rf".*(?<![\w.,]){value}", f"answer: {gt}")], [],
+                    f"A table from {'a financial report (FinTabNet)' if split == 'fintabnetqa' else 'Wikipedia (WTQ)'},"
+                    " via TableVQA-Bench (Kim et al., 2024)."),
+               *image(r))
+
+
+def pope(offset: int, scan: int):
+    """Photos: is there a <thing>? Two present, two absent but likely."""
+    ds = "lmms-lab-encoder/POPE"
+    photos: dict[str, list[dict]] = {}
+    for r in rows(ds, "Full", "adversarial", offset, scan):
+        photos.setdefault(r["image_source"], []).append(r)
+    for src, qs in photos.items():
+        yes = [q for q in qs if q["answer"].strip().lower() == "yes"][:2]
+        no = [q for q in qs if q["answer"].strip().lower() == "no"][:2]
+        if len(yes) < 2 or len(no) < 2:
+            continue
+        order = [yes[0], no[0], no[1], yes[1]] if int(hashlib.sha1(src.encode()).hexdigest(), 16) % 2 else \
+            [no[0], yes[0], yes[1], no[1]]
+        lines, find, claim = [], [], []
+        for n, q in enumerate(order, 1):
+            name, thing = f"q{n}", re.sub(r"^is there an? |\s*in the image\?$", "", q["question"].lower())
+            lines.append(f"{name}: <yes or no: {q['question']}>")
+            find.append(clause(name, rf"{q['answer'].lower()}\b", f"{name}: {q['answer'].lower()} ({thing})"))
+            if q["answer"].lower() == "no":
+                claim.append(clause(name, r"yes\b", f"{name}: yes, a {thing} (there is none)"))
+        yield (spec(f"vision-pope-{src.rsplit('_', 1)[-1].lstrip('0')}", "Say which things a photo shows", "pope", ds,
+                    src, "no licence on the dataset card; COCO photos", lines, find, claim,
+                    f"COCO photo {src}, questions from POPE's adversarial split (Li et al., 2023): the absent"
+                    " things are ones that often appear with what is there."),
+               *image(qs[0]))
+
+
+SOURCES = {"cord": cord, "screenqa": screenqa, "tablevqa": tablevqa, "pope": pope}
+
+
+def main() -> None:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--source", choices=[*SOURCES, "all"], default="all")
+    ap.add_argument("--count", type=int, default=5, help="tasks to write per source")
+    ap.add_argument("--offset", type=int, default=0, help="start this far into each source's rows")
+    ap.add_argument("--scan", type=int, default=300, help="rows to read per source, at most")
+    ap.add_argument("--dry-run", action="store_true", help="fetch and print each key, write nothing")
+    args = ap.parse_args()
+    for name in SOURCES if args.source == "all" else [args.source]:
+        made = 0
+        for s, picture, ext in SOURCES[name](args.offset, args.scan):
+            folder = TASKS / s["id"]
+            if folder.exists():
+                continue                                  # sampled before: kept, answers and all
+            made += 1
+            print(f"{s['id']}: " + "; ".join(c["label"] for c in s["must_find"]))
+            if not args.dry_run:
+                (folder / "fixtures").mkdir(parents=True)
+                s["images"] = [f"{s['id'].removeprefix('vision-')}{ext}"]
+                (folder / "fixtures" / s["images"][0]).write_bytes(picture)
+                (folder / "task.json").write_text(json.dumps(s, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+            if made >= args.count:
+                break
+        print(f"{name}: {made} {'would be ' if args.dry_run else ''}written")
+
+
+if __name__ == "__main__":
+    main()