@wendellresearch-ai/benchwarmers 0.3.3 → 0.5.0

package/suite/README.md

@@ -107,6 +107,39 @@ The scorer ignores a `must_not_claim` phrase where it is negated in its own
 sentence ("there are no INSERT statements"), so ruling something out is never
 counted as claiming it.
 
+## Vision tasks
+
+A fifth shape, added 2026-10-27: **reading images**. A task with `"shape":
+"vision"` and `"images"` (files in its `fixtures/`) is sent with its pictures
+as image parts, through the built-in client, to every model the catalog says
+takes images; a vision model (role `vision`) gets only these, and a model that
+cannot see never gets one (`run.applies`). The answer is `key: value` lines,
+`not shown` for what the picture does not show, scored by
+`suite/vision_keys.py`: a value for a field the picture cannot answer is an
+invention, like a trap. They stay out of the routing recall and the code
+reports; the Benchmarks page's Vision panel shows them field by field.
+
+Three sources:
+
+- **Made pictures** (`suite/vision_tasks.py`, needs Pillow to regenerate): a
+  receipt, a grading label, a bar chart, a table, a settings screen, shapes
+  and an invoice turned sideways, each key built from the data that drew it.
+  Shipped in `suite/tasks/vision-*` and copied into the data folder on the
+  next run (`run.seed_shipped`). Never seen in training; clean renders.
+- **Public datasets** (`python suite/vision_sample.py --count 5`): photographed
+  receipts (CORD v2), app screens with an unanswerable question (RICO-ScreenQA),
+  tables (TableVQA-Bench) and absent-object questions (POPE), with the
+  datasets' human answers. Written into the data folder only, so each install
+  samples its own and nothing is redistributed.
+- **Your own**: any task folder you write into the data folder's
+  `suite/tasks/vision-*` in the same form, from photos whose answers you have
+  checked. Real pictures of the real job, never in anyone's training data:
+  the strongest of the three.
+
+To hold them back for a while (a hub being changed, say), put the day they may
+start in the data folder's `config/suite.json`: `{"vision_from": "2026-10-30"}`.
+Until then no run picks a vision task; `--task <id>` still runs one.
+
 ## Adding a task
 
 A task is one directory under `suite/tasks/` containing:

package/suite/pace.py

@@ -72,10 +72,14 @@ def answered(task: str, model: str) -> bool:
 
 
 def queue(prefix: str, roster: list[str]) -> list[tuple[str, str]]:
-    """(task, model) pairs without an answer yet, task by task across models."""
-    tasks = sorted(p.parent.name for p in TASKS.glob(f"{prefix}*/task.json")
-                   if not json.loads(p.read_text(encoding="utf-8")).get("draft"))
-    return [(t, m) for t in tasks for m in roster if not answered(t, m)]
+    """(task, model) pairs without an answer yet, task by task across models,
+    each only where the task is the model's (run.applies)."""
+    specs = [s for s in run.load_tasks(None) if s["id"].startswith(prefix)]
+    skip, seeing = run.images_only(), {}
+    if any(s.get("images") for s in specs):
+        seeing = {m: run.sees_images(m) for m in roster}
+    return [(s["id"], m) for s in specs for m in roster
+            if run.applies(s, m, skip, seeing) and not answered(s["id"], m)]
 
 
 def rate_limited(task: str, model: str) -> bool:

package/suite/run.py

@@ -146,6 +146,50 @@ def available_models(cache: dict | None = None,
     return models
 
 
+def images_only() -> set[str]:
+    """Aliases whose role is vision or image (model_roles). No text shape
+    would ever seat them, so the text and code tasks are not theirs: their
+    answers only spent calls and put a reader of images on the code charts.
+    Routing still sees them (available_models), where a vision model sits
+    for images."""
+    import model_roles
+    return {a for a, r in model_roles.load()["models"].items() if r in ("vision", "image")}
+
+
+def sees_images(alias: str) -> bool:
+    """Whether the catalog says `alias` takes images, by the gateway's rule
+    (gateway.cannot_see): its input types, the Vision / OCR override in its
+    model popup, or the capability seen working. One the catalog knows
+    nothing about is not asked: a model that cannot see answers anyway,
+    describing a picture it never got, and that would score as reading badly."""
+    import catalog
+    import endpoints
+    if endpoints.entry_of(alias).get("api") == "responses":    # the client sends it no images yet
+        return False
+    return bool(catalog.info(alias).get("vision"))
+
+
+def applies(spec: dict, model: str, skip: set[str] | None = None, seeing: dict | None = None) -> bool:
+    """Whether `spec` is a task for `model`: a vision task (one with images)
+    for a model that takes them, any other for one that is not vision or
+    image only. So a vision model's suite is the vision tasks, a general one
+    that takes images gets both, and a text-only one never sees a picture.
+    `skip` (images_only()) and `seeing` (alias -> sees_images) save asking
+    again in a loop."""
+    if spec.get("images"):
+        if seeing is not None and model in seeing:
+            return seeing[model]
+        return sees_images(model)
+    return model not in (images_only() if skip is None else skip)
+
+
+def tasks_for(model: str, tasks: list[dict]) -> list[dict]:
+    skip, seeing = images_only(), {}
+    if any(t.get("images") for t in tasks):
+        seeing[model] = sees_images(model)
+    return [t for t in tasks if applies(t, model, skip, seeing)]
+
+
 def _registered_names() -> dict[str, str]:
     """alias -> the model name the endpoint knows it by; {} with no yaml yet."""
     import endpoints
@@ -227,9 +271,9 @@ def next_free(tasks_dir: pathlib.Path) -> str | None:
     is picked up on the next run rather than after a full cycle.
     """
     best, best_age = None, None
-    held = ad_hoc_models()
+    held, specs = ad_hoc_models(), load_tasks(None)
     for model in free_models():
-        if model in held:
+        if model in held or not tasks_for(model, specs):
             continue
         outputs = list(tasks_dir.glob(f"*/output/{model}.md"))
         if not outputs:
@@ -267,17 +311,23 @@ def next_model(tasks_dir: pathlib.Path) -> str | None:
          A gap is a worse blind spot than a stale number: an old score is
          still evidence, a missing one is none.
       3. oldest, so the set refreshes on its own
+
+    Complete means every task that is the model's (tasks_for): a vision
+    model is complete on the vision tasks alone.
     """
-    total = len([p for p in tasks_dir.iterdir() if p.is_dir()])
+    specs = load_tasks(None)
     free = set(free_models())
     best, best_key = None, None
     for model in available_models():
         if model in free:
             continue
-        outputs = list(tasks_dir.glob(f"*/output/{model}.md"))
+        mine = tasks_for(model, specs)
+        if not mine:
+            continue
+        outputs = [p for p in (t["dir"] / "output" / f"{model}.md" for t in mine) if p.exists()]
         if not outputs:
             return model                      # never scored: go first
-        key = (len(outputs) >= total,         # False (incomplete) sorts first
+        key = (len(outputs) >= len(mine),     # False (incomplete) sorts first
                max(o.stat().st_mtime for o in outputs))
         if best_key is None or key < best_key:
             best, best_key = model, key
@@ -314,7 +364,44 @@ def pick_tasks(tasks: list[dict], model: str, n: int = PER_RUN) -> list[dict]:
     return sorted(tasks, key=lambda s: answer(s).stat().st_mtime)[:n]
 
 
+SHIPPED = HERE / "tasks"
+SHIPPED_PREFIX = "vision-"
+
+
+def seed_shipped() -> None:
+    """Copy the vision tasks this checkout ships (suite/tasks/vision-*: made
+    images with their answer keys, suite/vision_tasks.py) into the data
+    folder, and keep their prompt, key and images current there. The saved
+    answers (output/) are the install's and are never touched. Other tasks
+    are the install's own and are not shipped this way."""
+    if SHIPPED.resolve() == TASKS.resolve():
+        return
+    for task in SHIPPED.glob(f"{SHIPPED_PREFIX}*/task.json"):
+        for src in [task, *sorted((task.parent / "fixtures").glob("*"))]:
+            dst = TASKS / src.relative_to(SHIPPED)
+            if not dst.exists() or dst.read_bytes() != src.read_bytes():
+                dst.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(src, dst)
+
+
+SETTINGS = DATA / "config" / "suite.json"
+
+
+def vision_held() -> str:
+    """The day the vision tasks start, while it is still ahead: `vision_from`
+    (YYYY-MM-DD) in config/suite.json, else "". Until then no run picks them
+    (scheduled, onboarding, pace); `--task` still runs one. A hold, set at a
+    release so the first vision runs wait for a quiet hub."""
+    try:
+        day = str(json.loads(SETTINGS.read_text(encoding="utf-8")).get("vision_from") or "")
+    except (OSError, ValueError, AttributeError):
+        return ""
+    return day if day > dt.date.today().isoformat() else ""
+
+
 def load_tasks(only: str | None) -> list[dict]:
+    seed_shipped()
+    held = "" if only else vision_held()
     tasks = []
     for path in sorted(TASKS.glob("*/task.json")):
         spec = json.loads(path.read_text(encoding="utf-8"))
@@ -325,6 +412,8 @@ def load_tasks(only: str | None) -> list[dict]:
         # A mined task is a draft until someone has read it (suite/mine.py).
         if spec.get("draft") and not only:
             continue
+        if held and spec.get("images"):
+            continue
         tasks.append(spec)
     if only and not tasks:
         sys.exit(f"no task called {only!r}")
@@ -348,6 +437,23 @@ def build_prompt(spec: dict) -> str:
     return "\n".join(parts)
 
 
+MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp"}
+
+
+def image_urls(spec: dict) -> list[str]:
+    """A vision task's pictures (`images`, files in its fixtures/) as data:
+    URLs, the way the gateway passes a request's images on."""
+    import base64
+    urls = []
+    for name in spec.get("images", []):
+        path = spec["dir"] / "fixtures" / name
+        if not path.exists():
+            raise MissingAttachment(f"{spec['id']}: image not found: fixtures/{name}")
+        mime = MIME.get(path.suffix.lower(), "image/png")
+        urls.append(f"data:{mime};base64,{base64.b64encode(path.read_bytes()).decode('ascii')}")
+    return urls
+
+
 # A model that was never allowed to answer has not answered badly, and a
 # service that fell over has not answered at all. Reporting either as 0/N is a
 # lie about the model — and reporting them as the SAME thing is a second lie,
@@ -810,12 +916,20 @@ def _child_env() -> dict:
 
 
 def ask_once(binary: str, model: str, prompt: str,
-             timeout: int) -> tuple[str, float]:
+             timeout: int, images: list[str] | None = None) -> tuple[str, float]:
     # BENCH_CLIENT=builtin: the same call without the `llm` subprocess, giving
     # the same text for the same failure (metrics/client.py).
     if str(ROOT / "metrics") not in sys.path:
         sys.path.insert(0, str(ROOT / "metrics"))
     import client
+    if images:
+        # Pictures go as the API's own image parts, through the built-in
+        # client whichever is chosen: `llm -m` reads a prompt, not parts.
+        import formats
+        content = [formats.text_part(prompt), *(formats.image(u) for u in images)]
+        res = client.chat(model, "", timeout=timeout, messages=[{"role": "user", "content": content}],
+                          purpose="vision task")
+        return client.output(res, timeout), res["seconds"]
     if client.backend(model) == "builtin":
         res = client.chat(model, prompt, timeout=timeout)
         return client.output(res, timeout), res["seconds"]
@@ -842,7 +956,7 @@ def ask_once(binary: str, model: str, prompt: str,
 
 
 def ask(binary: str, model: str, prompt: str, timeout: int,
-        attempts: int = 3) -> tuple[str, float, int]:
+        attempts: int = 3, images: list[str] | None = None) -> tuple[str, float, int]:
     """Ask, retrying UPSTREAM FAILURES only.
 
     A rate limit or an overloaded service is a fact about the afternoon, not
@@ -853,7 +967,7 @@ def ask(binary: str, model: str, prompt: str, timeout: int,
     """
     total = 0.0
     for attempt in range(1, attempts + 1):
-        body, seconds = ask_once(binary, model, prompt, timeout)
+        body, seconds = ask_once(binary, model, prompt, timeout, images)
         total += seconds
         kind = (outcome(body) or ("answered", ""))[0]
         if kind != "error" or attempt == attempts:
@@ -895,7 +1009,7 @@ def main() -> int:
         if chosen is None:
             sys.exit("no :free model registered")
         args.models = [chosen]
-        tasks = pick_tasks(tasks, chosen)
+        tasks = pick_tasks(tasks_for(chosen, tasks), chosen)
         print(f"--next-free chose {chosen}: {len(tasks)} tasks "
               f"(one model per run; the free tier is 50 calls a day, shared)")
 
@@ -904,10 +1018,10 @@ def main() -> int:
         if chosen is None:
             sys.exit("no paid model registered")
         args.models = [chosen]
-        total = len([p for p in TASKS.iterdir() if p.is_dir()])
-        have = len(list(TASKS.glob(f"*/output/{chosen}.md")))
-        tasks = pick_tasks(tasks, chosen)
-        print(f"--next chose {chosen} ({have}/{total} tasks scored; "
+        mine = tasks_for(chosen, tasks)
+        have = sum((t["dir"] / "output" / f"{chosen}.md").exists() for t in mine)
+        tasks = pick_tasks(mine, chosen)
+        print(f"--next chose {chosen} ({have}/{len(mine)} tasks scored; "
               f"{len(tasks)} this run)")
 
     models = args.models or [m for m in available_models()
@@ -919,10 +1033,20 @@ def main() -> int:
     ready = []
     for spec in tasks:
         try:
-            ready.append((spec, build_prompt(spec)))
+            ready.append((spec, build_prompt(spec), image_urls(spec)))
         except MissingAttachment as exc:
             print(f"skipped {exc}")
-    tasks, prompts = [s for s, _ in ready], {s["id"]: p for s, p in ready}
+    tasks, prompts = [s for s, _, _ in ready], {s["id"]: p for s, p, _ in ready}
+    pictures = {s["id"]: u for s, _, u in ready}
+    # Each pair only where the task is the model's (applies): pictures to a
+    # model that takes them, text and code to one that is not images only.
+    skip = images_only()
+    seeing = {m: sees_images(m) for m in models} if any(pictures.values()) else {}
+    pairs = {(s["id"], m) for s in tasks for m in models if applies(s, m, skip, seeing)}
+    left = len(tasks) * len(models) - len(pairs)
+    if left:
+        print(f"{left} task/model pair(s) left out: vision tasks go to models that take images,"
+              " text and code tasks to models that are not vision or image only")
 
     if args.dry_run:
         for spec in tasks:
@@ -939,16 +1063,20 @@ def main() -> int:
 
     for spec in tasks:
         prompt = prompts[spec["id"]]
+        if not any((spec["id"], m) in pairs for m in models):
+            continue
         print(f"\n## {spec['id']} — {spec['title']}")
         print(f"   {len(prompt):,} chars, {len(spec.get('must_find', []))} findings\n")
         for model in models:
+            if (spec["id"], model) not in pairs:
+                continue
             # N attempts at the SAME question. The spread is the point: a
             # single sample is a measurement of one afternoon, and this suite
             # has been reporting those as though they were properties.
             takes = []
             for _ in range(max(1, args.repeat)):
                 body, seconds, attempts = ask(binary, model, prompt,
-                                              args.timeout)
+                                              args.timeout, images=pictures[spec["id"]])
                 takes.append((score(spec, body), seconds, attempts, body))
                 if args.repeat > 1:
                     time.sleep(2)

package/suite/tasks/vision-bars/task.json

@@ -0,0 +1,40 @@
+{
+  "id": "vision-bars",
+  "title": "Read a bar chart",
+  "shape": "vision",
+  "images": [
+    "bars.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\ntallest: <team>\nshortest: <team>\nweb: <Web's value>\nover_30: <how many teams are above 30, not counting 30 itself>\ntarget: <the target line's value>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*tallest[^\\w\\n]*:[ \\t*_`\\\"']*billing",
+      "label": "tallest: Billing"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*shortest[^\\w\\n]*:[ \\t*_`\\\"']*data",
+      "label": "shortest: Data"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*web[^\\w\\n]*:[ \\t*_`\\\"']*35\\b",
+      "label": "web: 35"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*over[ _]30[^\\w\\n]*:[ \\t*_`\\\"']*2\\b",
+      "label": "over_30: 2"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*target[^\\w\\n]*:[ \\t*_`\\\"']*(not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "target: not shown"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*target[^\\w\\n]*:[ \\t*_`\\\"']*\\d",
+      "label": "target: a value (the chart has no target line)"
+    }
+  ],
+  "why": "Values read against gridlines every 5, labelled every 10; one count asks for care at the boundary.",
+  "made_by": "suite/vision_tasks.py"
+}

package/suite/tasks/vision-invoice/task.json

@@ -0,0 +1,40 @@
+{
+  "id": "vision-invoice",
+  "title": "Read an invoice turned on its side",
+  "shape": "vision",
+  "images": [
+    "invoice.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\ninvoice: <invoice number>\ndue: <YYYY-MM-DD>\namount_due: <amount>\nbill_to: <company>\npo_number: <purchase order number>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*invoice[^\\w\\n]*:[ \\t*_`\\\"']*inv-2026-0417\\b",
+      "label": "invoice: INV-2026-0417"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*due[^\\w\\n]*:[ \\t*_`\\\"']*2026-05-02\\b",
+      "label": "due: 2026-05-02"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*amount[ _]due[^\\w\\n]*:[ \\t*_`\\\"']*(eur|€)?\\s*1,?284\\.50\\b",
+      "label": "amount_due: EUR 1,284.50"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*bill[ _]to[^\\w\\n]*:[ \\t*_`\\\"']*kestrel labs",
+      "label": "bill_to: Kestrel Labs BV"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*po[ _]number[^\\w\\n]*:[ \\t*_`\\\"']*(not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "po_number: not shown"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*po[ _]number[^\\w\\n]*:[ \\t*_`\\\"']*\\d",
+      "label": "po_number: a value (the invoice has none)"
+    }
+  ],
+  "why": "A page a scanner or a phone delivered sideways; two dates close together, only one asked for.",
+  "made_by": "suite/vision_tasks.py"
+}

package/suite/tasks/vision-label/task.json

@@ -0,0 +1,52 @@
+{
+  "id": "vision-label",
+  "title": "Read a graded card's label",
+  "shape": "vision",
+  "images": [
+    "label.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\ngrader: <company>\ngrade: <the number>\nyear: <year>\nnumber: <card number>\nplayer: <name>\ncert: <certification number>\nsurface: <surface subgrade>\nautograph: <is the card signed: yes or no>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*grader[^\\w\\n]*:[ \\t*_`\\\"']*northline",
+      "label": "grader: Northline"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*grade[^\\w\\n]*:[ \\t*_`\\\"']*(mint\\s*)?9\\b(?!\\.)",
+      "label": "grade: 9"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*year[^\\w\\n]*:[ \\t*_`\\\"']*2019\\b",
+      "label": "year: 2019"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*number[^\\w\\n]*:[ \\t*_`\\\"']*#?\\s*148\\b",
+      "label": "number: 148"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*player[^\\w\\n]*:[ \\t*_`\\\"']*maren okafor",
+      "label": "player: Maren Okafor"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*cert[^\\w\\n]*:[ \\t*_`\\\"']*#?\\s*40731958\\b",
+      "label": "cert: 40731958"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*surface[^\\w\\n]*:[ \\t*_`\\\"']*8\\.5\\b",
+      "label": "surface: 8.5"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*autograph[^\\w\\n]*:[ \\t*_`\\\"']*(no\\b|not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "autograph: no, or not shown"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*autograph[^\\w\\n]*:[ \\t*_`\\\"']*(yes|signed|auto)",
+      "label": "autograph: yes (the label says nothing of one)"
+    }
+  ],
+  "why": "A label read field by field; the cert number must be exact to the digit.",
+  "made_by": "suite/vision_tasks.py"
+}

package/suite/tasks/vision-receipt/task.json

@@ -0,0 +1,44 @@
+{
+  "id": "vision-receipt",
+  "title": "Read a receipt",
+  "shape": "vision",
+  "images": [
+    "receipt.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\nstore: <name>\ndate: <YYYY-MM-DD>\nitems: <how many line items were bought>\npriciest: <the most expensive item>\ntotal: <amount paid>\ntip: <tip amount>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*store[^\\w\\n]*:[ \\t*_`\\\"']*harbor lane",
+      "label": "store: Harbor Lane Grocery"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*date[^\\w\\n]*:[ \\t*_`\\\"']*2026-03-14\\b",
+      "label": "date: 2026-03-14"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*items[^\\w\\n]*:[ \\t*_`\\\"']*4\\b",
+      "label": "items: 4"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*priciest[^\\w\\n]*:[ \\t*_`\\\"']*cheddar",
+      "label": "priciest: Cheddar 200g"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*total[^\\w\\n]*:[ \\t*_`\\\"']*\\$?\\s*22\\.17\\b",
+      "label": "total: 22.17"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*tip[^\\w\\n]*:[ \\t*_`\\\"']*(not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "tip: not shown"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*tip[^\\w\\n]*:[ \\t*_`\\\"']*\\$?\\s*\\d",
+      "label": "tip: a value (the receipt shows none)"
+    }
+  ],
+  "why": "Printed text, a date written differently from how it is asked for, a count and a maximum.",
+  "made_by": "suite/vision_tasks.py"
+}

package/suite/tasks/vision-screen/task.json

@@ -0,0 +1,48 @@
+{
+  "id": "vision-screen",
+  "title": "Read a settings screen",
+  "shape": "vision",
+  "images": [
+    "screen.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\ntoggles_on: <every switch that is on>\nconflict: <the option chosen>\ndisabled: <the button that cannot be pressed>\nerror_code: <code>\naccount: <the signed-in email>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*toggles[ _]on[^\\w\\n]*:[ \\t*_`\\\"']*.*back up photos",
+      "label": "toggles_on: Back up photos"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*toggles[ _]on[^\\w\\n]*:[ \\t*_`\\\"']*.*notify on conflicts",
+      "label": "toggles_on: Notify on conflicts"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*conflict[^\\w\\n]*:[ \\t*_`\\\"']*keep both",
+      "label": "conflict: Keep both"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*disabled[^\\w\\n]*:[ \\t*_`\\\"']*sync now",
+      "label": "disabled: Sync now"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*error[ _]code[^\\w\\n]*:[ \\t*_`\\\"']*e-?213\\b",
+      "label": "error_code: E-213"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*account[^\\w\\n]*:[ \\t*_`\\\"']*(not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "account: not shown"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*toggles[ _]on[^\\w\\n]*:[ \\t*_`\\\"']*.*(mobile data|battery)",
+      "label": "toggles_on: a switch that is off"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*account[^\\w\\n]*:[ \\t*_`\\\"']*\\S+@\\S+",
+      "label": "account: an email (the screen shows none)"
+    }
+  ],
+  "why": "A screen's state: which switches are on, which option is picked, which button is greyed out.",
+  "made_by": "suite/vision_tasks.py"
+}

package/suite/tasks/vision-shapes/task.json

@@ -0,0 +1,40 @@
+{
+  "id": "vision-shapes",
+  "title": "Count shapes and say where one is",
+  "shape": "vision",
+  "images": [
+    "shapes.png"
+  ],
+  "prompt": "Read the attached image and answer in exactly these lines, one per line, nothing else:\n\nblue_circles: <how many>\nred_squares: <how many>\ngreen_triangles: <how many>\ntop_right: <the shape in the top-right corner, with its colour>\npurple_shapes: <how many>\n\nIf the image does not show something, write `not shown` for it. Do not guess.",
+  "min_answer_chars": 20,
+  "must_find": [
+    {
+      "regex": "(?m)^[^\\w\\n]*blue[ _]circles[^\\w\\n]*:[ \\t*_`\\\"']*6\\b",
+      "label": "blue_circles: 6"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*red[ _]squares[^\\w\\n]*:[ \\t*_`\\\"']*3\\b",
+      "label": "red_squares: 3"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*green[ _]triangles[^\\w\\n]*:[ \\t*_`\\\"']*2\\b",
+      "label": "green_triangles: 2"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*top[ _]right[^\\w\\n]*:[ \\t*_`\\\"']*.*star",
+      "label": "top_right: a yellow star"
+    },
+    {
+      "regex": "(?m)^[^\\w\\n]*purple[ _]shapes[^\\w\\n]*:[ \\t*_`\\\"']*(0\\b|none\\b|zero\\b|no\\b|not shown|not visible|not present|not listed|not given|not stated|not on the|not in the|not provided|not included|none shown|n/?a\\b|none\\b|no such|does not show|doesn't show|cannot be (?:determined|answered|seen)|can't be (?:determined|answered|seen)|unanswerable)",
+      "label": "purple_shapes: 0"
+    }
+  ],
+  "must_not_claim": [
+    {
+      "regex": "(?m)^[^\\w\\n]*purple[ _]shapes[^\\w\\n]*:[ \\t*_`\\\"']*[1-9]",
+      "label": "purple_shapes: a count above 0 (there are none)"
+    }
+  ],
+  "why": "Counting to six without losing one, and a corner told apart from its neighbours.",
+  "made_by": "suite/vision_tasks.py"
+}