@wendellresearch-ai/benchwarmers 0.3.2 → 0.4.0

package/README.md

@@ -6,18 +6,24 @@
 </h1>
 
 <p align="center">
-  <a href="https://github.com/WendellResearch-AI/benchwarmers/releases"><img src="https://img.shields.io/github/v/release/WendellResearch-AI/benchwarmers?color=990000" alt="Latest release"></a>
+  <a href="https://github.com/WendellResearch-AI/benchwarmers/releases/latest"><img src="https://img.shields.io/github/v/release/WendellResearch-AI/benchwarmers?logo=github&color=181717" alt="Latest release"></a>
   <a href="https://www.npmjs.com/package/@wendellresearch-ai/benchwarmers"><img src="https://img.shields.io/npm/v/@wendellresearch-ai/benchwarmers?color=cb3837&logo=npm" alt="npm"></a>
   <a href="https://pypi.org/project/benchwarmers/"><img src="https://img.shields.io/pypi/v/benchwarmers?color=3775a9&logo=pypi&logoColor=white" alt="PyPI"></a>
-  <a href="https://github.com/WendellResearch-AI/benchwarmers/actions/workflows/check.yml"><img src="https://github.com/WendellResearch-AI/benchwarmers/actions/workflows/check.yml/badge.svg" alt="Checks"></a>
+  <a href="https://github.com/WendellResearch-AI/benchwarmers/actions/workflows/check.yml"><img src="https://img.shields.io/github/actions/workflow/status/WendellResearch-AI/benchwarmers/check.yml?branch=main&logo=github&label=CI%20tests" alt="CI tests"></a>
+  <a href="https://registry.modelcontextprotocol.io/?q=benchwarmers"><img src="https://img.shields.io/badge/MCP_Registry-111111?logo=modelcontextprotocol&logoColor=white" alt="In the official MCP Registry"></a>
+  <a href="https://smithery.ai/skills/wendellresearch-ai/benchwarmers"><img src="https://img.shields.io/badge/Smithery-FF5601?logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxODggMjExIiBmaWxsPSIjZmZmIj48cGF0aCBkPSJNNDEuNzAxMSA3Ny40OEgwVjEwMC4yNjRDMCAxMTguMTcxIDE0LjUyNzUgMTMyLjY5OCAzMi40MzQyIDEzMi42OThINTUuMjE4M1Y5MC45OTcxQzU1LjIxODMgODMuNTQxOCA0OS4xNTY1IDc3LjQ4IDQxLjcwMTEgNzcuNDhaIi8%2BPHBhdGggZD0iTTY2LjA4NzkgOTAuOTk3MVYxMzIuNjk4SDEwNy43ODlDMTE1LjI0NCAxMzIuNjk4IDEyMS4zMDYgMTI2LjYzNiAxMjEuMzA2IDExOS4xODFWNzcuNDhINzkuNjA1MUM3Mi4xNDk3IDc3LjQ4IDY2LjA4NzkgODMuNTQxOCA2Ni4wODc5IDkwLjk5NzFaIi8%2BPHBhdGggZD0iTTE1NC45OTUgNzcuNDhIMTMyLjIxVjExOS4xODFDMTMyLjIxIDEyNi42MzYgMTM4LjI3MiAxMzIuNjk4IDE0NS43MjggMTMyLjY5OEgxODcuNDI5VjEwNi4xNTJDMTg3LjQyOSA4OC4yNDQ5IDE3Mi45MDEgNzcuNTE0OCAxNTQuOTk1IDc3LjUxNDhWNzcuNDhaIi8%2BPHBhdGggZD0iTTAgNDUuOTUxNFY2Ni42MTAzSDQxLjcwMTFDNDkuMTU2NSA2Ni42MTAzIDU1LjIxODMgNjAuNTQ4NSA1NS4yMTgzIDUzLjA5MzJWMEMyNS44NDk4IDAuMjA5MDI4IDAuMDM0ODM4IDE0LjY2NjggMCA0NS45NTE0WiIvPjxwYXRoIGQ9Ik02Ni4wODc5IDUzLjA5M0M2Ni4wODc5IDYwLjU0ODMgNzIuMTQ5NyA2Ni42MTAxIDc5LjYwNTEgNjYuNjEwMUgxMjEuMzA2VjExLjQ2MTVDMTAwLjkyNiA4LjM2MDkyIDkwLjMwMDMgMS43NDE2OSA2Ni4wODc5IDAuMjQzNjUyVjUzLjA1ODFWNTMuMDkzWiIvPjxwYXRoIGQ9Ik0xNDQuMDkgMTMuMDYzOUMxMzkuNzcgMTMuMDYzOSAxMzUuODM0IDEyLjkyNDYgMTMyLjIxIDEyLjY4MDdWNjYuNjFIMTU0Ljk5NUMxNzIuOTAxIDY2LjYxIDE4Ny40MjkgNTIuMDgyNSAxODcuNDI5IDM0LjE3NThWMC4xNzM4MjhDMTc3LjI5MSA4LjExNjkgMTYyLjkwMyAxMy4wNjM5IDE0NC4wOSAxMy4wNjM5WiIvPjxwYXRoIGQ9Ik0wIDE3Ni4wMDJWMjEwLjAwNEMxMC4xMzc5IDIwMi4wNjEgMjQuNTI2IDE5Ny4xMTQgNDMuMzM4NSAxOTcuMTE0QzQ3LjY1ODQgMTk3LjExNCA1MS41OTUxIDE5Ny4yNTMgNTUuMjE4MyAxOTcuNTMyVjE0My42MDNIMzIuNDM0MkMxNC41Mjc1IDE0My42MDMgMCAxNTguMTMgMCAxNzYuMDM3TDAgMTc2LjAwMloiLz48cGF0aCBkPSJNMTA3Ljc4OSAxNDMuNjAzSDY2LjA4NzlWMTk4LjcxNkM4Ni40NjgxIDIwMS44MTcgOTcuMDkzOCAyMDguNDM2IDEyMS4zMDYgMjA5LjkzNFYxNTcuMTJDMTIxLjMwNiAxNDkuNjY0IDExNS4yNDQgMTQzLjYwMyAxMDcuNzg5IDE0My42MDNaIi8%2BPHBhdGggZD0iTTEzMi4yMSAxNTcuMDg1VjIxMC4xNzhDMTYxLjU0NCAyMDkuOTY5IDE4Ny4zMjQgMTk1LjUxMSAxODcuNDI5IDE2NC4yOTZWMTQzLjU2OEgxNDUuNzI4QzEzOC4yNzIgMTQzLjU2OCAxMzIuMjEgMTQ5LjYzIDEzMi4yMSAxNTcuMDg1WiIvPjwvc3ZnPg%3D%3D" alt="The benchwarmers skill on Smithery"></a>
+  <a href="https://skills.sh/wendellresearch-ai/benchwarmers/benchwarmers"><img src="https://img.shields.io/badge/skills.sh-111111" alt="On skills.sh"></a>
+  <a href="https://glama.ai/mcp/servers/WendellResearch-AI/benchwarmers"><img src="https://glama.ai/mcp/servers/WendellResearch-AI/benchwarmers/badges/score.svg" alt="Benchwarmers MCP server, quality and maintenance score on Glama"></a>
+  <a href="https://cursor.directory/plugins/benchwarmers"><img src="https://img.shields.io/badge/cursor.directory-111111?logo=cursor&logoColor=white" alt="On cursor.directory"></a>
+  <a href="#from-claude-code-and-other-agents"><img src="https://img.shields.io/badge/Antigravity-plugin-3186FF?labelColor=111111&logo=data%3Aimage%2Fsvg%2Bxml%3Bbase64%2CPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjEyIDE3IDg2IDgwIj48cGF0aCBkPSJNODkuNyA5My43YzQuNjcgMy41IDExLjY3IDEuMTcgNS4yNS01LjI1Qzc1LjcgNjkuNzggNzkuNzggMTguNDUgNTUuODcgMTguNDVTMzYuMDMgNjkuNzggMTYuNzggODguNDVjLTcgNyAuNTggOC43NSA1LjI1IDUuMjVDNDAuMTIgODEuNDUgMzguOTUgNTkuODYgNTUuODcgNTkuODZTNzEuNjIgODEuNDUgODkuNyA5My43WiIgZmlsbD0iIzMxODZGRiIvPjxtYXNrIGlkPSJtIiBtYXNrVW5pdHM9InVzZXJTcGFjZU9uVXNlIiB4PSIxMyIgeT0iMTgiIHdpZHRoPSI4NSIgaGVpZ2h0PSI3OCI%2BPHBhdGggZD0iTTg5LjcgOTMuN2M0LjY3IDMuNSAxMS42NyAxLjE3IDUuMjUtNS4yNUM3NS43IDY5Ljc4IDc5Ljc4IDE4LjQ1IDU1Ljg3IDE4LjQ1UzM2LjAzIDY5Ljc4IDE2Ljc4IDg4LjQ1Yy03IDcgLjU4IDguNzUgNS4yNSA1LjI1QzQwLjEyIDgxLjQ1IDM4Ljk1IDU5Ljg2IDU1Ljg3IDU5Ljg2UzcxLjYyIDgxLjQ1IDg5LjcgOTMuN1oiIGZpbGw9IiNmZmYiLz48L21hc2s%2BPGcgbWFzaz0idXJsKCNtKSI%2BPGcgZmlsdGVyPSJ1cmwoI2EpIj48ZWxsaXBzZSBjeD0iOTYuNDkiIGN5PSIzNS4xMiIgcng9IjI5LjUiIHJ5PSIzMC4xNSIgdHJhbnNmb3JtPSJyb3RhdGUoNzYuOTIgOTYuNDkgMzUuMTIpIiBmaWxsPSIjRkM0MTNEIi8%2BPC9nPjxnIGZpbHRlcj0idXJsKCNiKSI%2BPGVsbGlwc2UgY3g9IjkuMDMiIGN5PSI0MS42NiIgcng9IjMwLjgzIiByeT0iMzkuOTQiIHRyYW5zZm9ybT0icm90YXRlKDc0LjEzIDkuMDMgNDEuNjYpIiBmaWxsPSIjMDBCOTVDIi8%2BPC9nPjxnIGZpbHRlcj0idXJsKCNjKSI%2BPGVsbGlwc2UgY3g9Ijc1Ljc1IiBjeT0iMTA0LjgyIiByeD0iMjkuMDIiIHJ5PSIyNy45NCIgdHJhbnNmb3JtPSJyb3RhdGUoNzYuOTIgNzUuNzUgMTA0LjgyKSIgZmlsbD0iIzMxODZGRiIvPjwvZz48ZyBmaWx0ZXI9InVybCgjZCkiPjxlbGxpcHNlIGN4PSIzMy41NyIgY3k9IjM1LjQiIHJ4PSIzMy41NyIgcnk9IjM1LjQiIHRyYW5zZm9ybT0ibWF0cml4KC0uNDEgLjkxMiAtLjkxMiAtLjQxIDEwMS4yNSAtMTUuMTcpIiBmaWxsPSIjRkJCQzA0Ii8%2BPC9nPjwvZz48ZGVmcz48ZmlsdGVyIGlkPSJhIiB4PSIyOCIgeT0iLTMzIiB3aWR0aD0iMTM3IiBoZWlnaHQ9IjEzNiIgZmlsdGVyVW5pdHM9InVzZXJTcGFjZU9uVXNlIj48ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIxOC44Ii8%2BPC9maWx0ZXI%2BPGZpbHRlciBpZD0iYiIgeD0iLTYzIiB5PSItMjMiIHdpZHRoPSIxNDQiIGhlaWdodD0iMTI5IiBmaWx0ZXJVbml0cz0idXNlclNwYWNlT25Vc2UiPjxmZUdhdXNzaWFuQmx1ciBzdGREZXZpYXRpb249IjE2Ii8%2BPC9maWx0ZXI%2BPGZpbHRlciBpZD0iYyIgeD0iMTYiIHk9IjQ0IiB3aWR0aD0iMTE5IiBoZWlnaHQ9IjEyMSIgZmlsdGVyVW5pdHM9InVzZXJTcGFjZU9uVXNlIj48ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIxNS4yIi8%2BPC9maWx0ZXI%2BPGZpbHRlciBpZD0iZCIgeD0iLTgiIHk9Ii02MSIgd2lkdGg9IjEyNyIgaGVpZ2h0PSIxMjQiIGZpbHRlclVuaXRzPSJ1c2VyU3BhY2VPblVzZSI%2BPGZlR2F1c3NpYW5CbHVyIHN0ZERldmlhdGlvbj0iMTMuOCIvPjwvZmlsdGVyPjwvZGVmcz48L3N2Zz4%3D" alt="Antigravity plugin"></a>
   <a href="#docker-and-hosting"><img src="https://img.shields.io/badge/Docker-compose-2496ED?logo=docker&logoColor=white" alt="Runs in Docker"></a>
+  <a href="https://render.com/deploy?repo=https://github.com/WendellResearch-AI/benchwarmers"><img src="https://img.shields.io/badge/Render-deploy-8A05FF?logo=render&logoColor=white" alt="Deploy to Render in one click"></a>
   <br>
   <a href="GATEWAY.md#other-api-formats"><img src="https://img.shields.io/badge/API-OpenAI_%7C_Responses-111111" alt="OpenAI chat completions and Responses API"></a>
   <a href="GATEWAY.md#other-api-formats"><img src="https://img.shields.io/badge/API-Anthropic_Messages-111111" alt="Anthropic Messages API"></a>
   <a href="GATEWAY.md#other-api-formats"><img src="https://img.shields.io/badge/API-Gemini-111111" alt="Gemini API"></a>
   <a href="GATEWAY.md#other-api-formats"><img src="https://img.shields.io/badge/API-Ollama-111111" alt="Ollama API"></a>
   <a href="GATEWAY.md#route-mode-agent-clients"><img src="https://img.shields.io/badge/agents-tool_calling-111111" alt="Agent clients with tool calling, through route mode"></a>
-  <a href="https://registry.modelcontextprotocol.io/v0/servers?search=io.github.WendellResearch-AI/benchwarmers"><img src="https://img.shields.io/badge/MCP-Registry-111111" alt="MCP server in the official MCP Registry"></a>
   <br>
   <a href="#try-it"><img src="https://img.shields.io/badge/Python-3.11_%7C_3.12_%7C_3.13-3776AB?logo=python&logoColor=white" alt="Python 3.11 to 3.13"></a>
   <a href="#try-it"><img src="https://img.shields.io/badge/core-stdlib_only-2ea44f" alt="The core needs only the Python standard library"></a>
@@ -155,7 +161,9 @@ dependencies either way: the app is the Python standard library.
    each is registered, probed and run through the suite, one at a time, with
    the progress beside the button. **Quick** probes now and leaves the suite
    to the scheduled passes; **Quick + ad hoc** also keeps the bench from ever
-   calling the model on its own, for a metered or trial key.
+   calling the model on its own, for a metered or trial key. Tick **Vision
+   model** for one that reads images and little else: it sits only for
+   questions about images, and is tested on the vision tasks, not the code ones.
 
 5. **Make a collection** on the Collections page (the models a panel always
    includes, and what it is for). The dashboard answers the gateway's routes on
@@ -309,6 +317,13 @@ Benchwarmers running, the hook shows a hook-error notice at the end of each
 turn, as `/mcp` shows the server as failed; start the server, or disable the
 plugin for that session.
 
+**Cursor and Antigravity plugins.** The same skill, with the stdio server
+below (so they need Node and Python 3.11 or later): Cursor's is
+[on cursor.directory](https://cursor.directory/plugins/benchwarmers), and
+Antigravity's installs from the repository:
+
+    agy plugin install https://github.com/WendellResearch-AI/benchwarmers
+
 **Any other MCP client** that speaks HTTP:
 
 ```json

package/USAGE.md

@@ -57,7 +57,14 @@ spends:
   its own (no explorer seat, no scheduled runs). The one that saves calls, for
   a metered, capped or trial key. A collection that lists it still calls it.
 
-`bench onboard <served> --quick [--ad-hoc]` does the same from a terminal.
+**Vision model**, the box beside it, gives the model role **vision** as it is
+registered: it sits only for requests that carry images, and its suite is the
+vision tasks (photos of documents, charts, labels and screens with an answer
+key) rather than the text and code ones. Full runs them now, Quick through the
+scheduled passes. For a model that reads images and little else; a model that
+does everything and also takes images stays general and gets both.
+
+`bench onboard <served> [--quick [--ad-hoc]] [--vision]` does the same from a terminal.
 Onboarding takes minutes and spends calls, so models run **one at a time**,
 in a queue the dashboard keeps:
 

package/VERSION

@@ -1 +1 @@
-0.3.2
+0.4.0

package/cli.py

@@ -261,7 +261,8 @@ def cmd_onboard(args) -> None:
         if args.ad_hoc and not args.quick:
             sys.exit("--ad-hoc goes with --quick: an ad hoc model gets no suite run of its own")
         mode = "quick-adhoc" if args.ad_hoc else "quick" if args.quick else "full"
-        items = [{"served": s, "endpoint": args.endpoint or "", "mode": mode} for s in args.served]
+        items = [{"served": s, "endpoint": args.endpoint or "", "mode": mode, "vision": args.vision}
+                 for s in args.served]
         reply = call("POST", "/api/onboard-queue", {"items": items})
         if args.wait:
             while reply.get("running"):
@@ -598,6 +599,8 @@ def parser() -> argparse.ArgumentParser:
                    help="probe only; the scheduled suite passes fill the suite in later")
     s.add_argument("--ad-hoc", action="store_true",
                    help="with --quick: the bench never calls it on its own (metered, capped or trial keys)")
+    s.add_argument("--vision", action="store_true",
+                   help="role vision: a seat only for questions about images, and its suite is the vision tasks")
 
     s = cmd("dismiss", cmd_dismiss, "stop offering served models as candidates")
     s.add_argument("served", nargs="+"), s.add_argument("--endpoint"), s.add_argument("--undo", action="store_true")

package/metrics/aliases.py

@@ -16,17 +16,38 @@ from __future__ import annotations
 
 import json
 import pathlib
+import time
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
 HISTORY = DATA / "config" / "alias-history.json"
 
 
+_cache: tuple = (None, {}, 0.0)
+
+
 def history() -> dict[str, str]:
+    """The file, read again only when it changes, and looked at no more than
+    once a second: one dashboard render asks tens of thousands of times, and
+    even a stat each time was seconds on Windows. A rename written here
+    (serve.rename) calls forget(), so it shows at once."""
+    global _cache
+    seen, now = _cache, time.monotonic()          # one read: forget() may swap it on another thread
+    if seen[0] and seen[0][0] == str(HISTORY) and now - seen[2] < 1:
+        return seen[1]
     try:
-        return json.loads(HISTORY.read_text(encoding="utf-8"))
-    except (OSError, ValueError):
-        return {}
+        st = HISTORY.stat()
+        key = (str(HISTORY), st.st_mtime_ns, st.st_size)
+        table = seen[1] if seen[0] == key else json.loads(HISTORY.read_text(encoding="utf-8"))
+    except (OSError, ValueError):                 # none yet, or half-written: no renames, for a second
+        key, table = (str(HISTORY), None, None), {}
+    _cache = (key, table, now)
+    return table
+
+
+def forget() -> None:
+    global _cache
+    _cache = (None, {}, 0.0)
 
 
 def current(alias: str, table: dict[str, str] | None = None) -> str:

package/metrics/endpoints.py

@@ -154,13 +154,29 @@ def connections() -> dict:
     return data if isinstance(data, dict) else {}
 
 
+_default: tuple = (None, "")
+
+
 def default() -> str:
+    """The default endpoint's name, read again only when endpoints.json
+    changes: snapshot_key asks once per model, hundreds of times a page."""
+    global _default
+    try:
+        st = FILE.stat()
+        key = (str(FILE), st.st_mtime_ns, st.st_size)
+    except OSError:
+        key = None
+    seen = _default
+    if key and seen[0] == key:
+        return seen[1]
     data = load()
     if not data:
         raise RuntimeError(f"no endpoints configured: add one from the dashboard (python serve.py), run"
                            f" scripts/init.py --endpoint NAME BASE_URL, or create {FILE}"
                            ' like {"name": {"base": "https://.../v1", "key": "name", "default": true}}')
-    return next((n for n, e in data.items() if e.get("default")), next(iter(data)))
+    name = next((n for n, e in data.items() if e.get("default")), next(iter(data)))
+    _default = (key, name)
+    return name
 
 
 def kind(name: str, data: dict | None = None) -> str:
@@ -322,13 +338,29 @@ def endpoint_name(reg: dict, data: dict | None = None) -> str | None:
     return next((n for n in same if data[n].get("key") == reg.get("api_key_name")), same[0] if same else None)
 
 
+_parsed: tuple = (None, {})
+
+
 def registrations() -> dict[str, dict]:
     """alias -> its yaml entry as written by yaml_block: model_name, api_base,
-    api_key_name and a `headers` dict. Not a YAML parser; the file is ours."""
+    api_key_name and a `headers` dict. Not a YAML parser; the file is ours.
+
+    Parsed again only when the file changes (a dashboard render asks hundreds
+    of times); every caller gets its own copies to change."""
+    global _parsed
+    try:
+        st = YAML.stat()
+    except OSError:                                # a fresh install has registered nothing yet
+        return {}
+    key = (str(YAML), st.st_mtime_ns, st.st_size)
+    if _parsed[0] != key:
+        _parsed = (key, _parse_registrations())
+    return {a: dict(r, headers=dict(r["headers"])) for a, r in _parsed[1].items()}
+
+
+def _parse_registrations() -> dict[str, dict]:
     out: dict[str, dict] = {}
     current = None
-    if not YAML.exists():                          # a fresh install has registered nothing yet
-        return out
     for line in YAML.read_text(encoding="utf-8").splitlines():
         if (m := re.match(r"\s*-\s*model_id:\s*(.+)", line)):
             current = out.setdefault(m.group(1).strip().strip("\"'"), {"headers": {}})

package/metrics/model_roles.py

@@ -20,7 +20,9 @@ roles:   general  reasoning / do-it-all: eligible for every shape
                   charts), weak at text-only work: admitted by the vision shape
                   alone, and a collection member with this role sits only when
                   the request carries images (gateway.py). A model that reads
-                  images AND does everything else stays general.
+                  images AND does everything else stays general. `bench onboard
+                  --vision` (the picker's Vision model box) onboards one this
+                  way; its suite is the vision tasks alone (suite/run.applies).
          decision typed answers with probabilities through a Decisions API, not
                   chat; no shape admits them (config/decision-models.json)
 

package/metrics/route.py

@@ -250,8 +250,9 @@ def suite_recall() -> dict[str, tuple[int, int, int]]:
         # Claim-verification tasks are a true/false verdict, where a guess
         # scores half: folding them into finding-recall would blur the one
         # number that picks the primary. They are reported on their own, as
-        # are seeded and real-bug (rebench) FAULT/NO FAULT tasks.
-        if spec.get("shape") in ("claim", "seeded", "rebench") or spec.get("draft"):
+        # are seeded and real-bug (rebench) FAULT/NO FAULT tasks, and reading
+        # images (vision), which is not what a text panel is picked for.
+        if spec.get("shape") in ("claim", "seeded", "rebench", "vision") or spec.get("draft"):
             continue
         for answer in sorted((task_dir.parent / "output").glob("*.md")):
             alias = current_alias(answer.stem)      # a renamed model keeps its record

package/metrics/wrong.py

@@ -84,7 +84,9 @@ def table() -> dict[str, dict]:
                  "hunt": dict.fromkeys(HUNT, 0)})
     for path in sorted(TASKS.glob("*/task.json")):
         spec = json.loads(path.read_text(encoding="utf-8"))
-        if spec.get("draft"):
+        # Reading images has its own record (serve.vision_rows): a trap taken
+        # there is a picture misread, not code.
+        if spec.get("draft") or spec.get("shape") == "vision":
             continue
         for answer in sorted((path.parent / "output").glob("*.md")):
             text = answer.read_text(encoding="utf-8", errors="replace")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/check.py

@@ -73,6 +73,8 @@ CHECKS = [
     ("suite/test_seed.py", [], False),
     ("suite/test_rebench.py", [], False),
     ("suite/test_pick.py", [], False),
+    ("suite/test_vision_tasks.py", [], False),
+    ("suite/vision_keys.py", [], False),
     ("scripts/test_client.py", [], True),          # runs the llm CLI against a fake server
 ]
 

package/scripts/onboard.py

@@ -235,6 +235,9 @@ def main() -> None:
     ap.add_argument("--ad-hoc", action="store_true",
                     help="mark it ad hoc as it is registered: the bench never calls it on its own"
                          " (for a metered, capped or trial key)")
+    ap.add_argument("--vision", action="store_true",
+                    help="role vision as it is registered: it sits only for requests that carry"
+                         " images, and its suite is the vision tasks (suite/run.applies)")
     args = ap.parse_args()
 
     name = args.served_name
@@ -296,6 +299,12 @@ def main() -> None:
                       lambda roles: roles.__setitem__("ad_hoc", sorted({*roles.get("ad_hoc", []), alias})),
                       ensure_ascii=False)
         print("        ad hoc: the bench never calls it on its own")
+    if args.vision:
+        # Before the probe too: a text panel or a suite pass in between would seat it.
+        locked.update(DATA / "config" / "model-roles.json",
+                      lambda roles: roles.setdefault("models", {}).__setitem__(alias, "vision"),
+                      ensure_ascii=False)
+        print("        role vision: a seat only for requests that carry images")
     run("scripts/sync_config.py", timeout=300)
     # Open WebUI's tags name the connection behind the hub; without a hub the
     # endpoint is the connection, and route.connection_of needs no tags.
@@ -359,10 +368,12 @@ def main() -> None:
     record_issue(alias, None)
 
     # 5. the tier that decides routing
+    # A vision model's suite is the vision tasks alone (suite/run.applies picks
+    # them by its role); a general model that takes images gets both.
     if args.skip_suite:
         print("  [5/5] suite skipped by request")
     else:
-        print("  [5/5] running the internal suite...")
+        print("  [5/5] running the internal suite" + (" (its vision tasks)..." if args.vision else "..."))
         try:
             suite = run("suite/run.py", "--model", alias, "--timeout", "300",
                         timeout=3600)

package/scripts/test_no_owui.py

@@ -241,12 +241,15 @@ def owner(fake: str) -> None:
     OWUI_MODELS.append({"id": "just-added", "urlIdx": 1})
     real_names = provider_tags.connection_names
     try:
-        live = {x["served"]: x["group"] for x in serve.onboard_candidates()["candidates"]}
+        # The picker's list is kept a few minutes (serve.LISTED_FOR); Refresh asks again.
+        kept = serve.onboard_candidates()
+        assert "just-added" not in {x["served"] for x in kept["candidates"]}, kept
+        live = {x["served"]: x["group"] for x in serve.onboard_candidates(fresh=True)["candidates"]}
 
         def refused(hub):
             raise OSError("401")
         provider_tags.connection_names = refused
-        c = serve.onboard_candidates()
+        c = serve.onboard_candidates(fresh=True)
     finally:
         provider_tags.connection_names = real_names
         OWUI_MODELS.pop()

package/scripts/test_serve_edit.py

@@ -360,9 +360,9 @@ def main() -> int:
 
     modes = {}
 
-    def fake_spawn(served, endpoint="", who="ui", mode="full"):
+    def fake_spawn(served, endpoint="", who="ui", mode="full", vision=False):
         order.append(served)
-        modes[served] = mode
+        modes[served] = mode + (" vision" if vision else "")
         (runs / f"onboard-{served}.log").write_text(f"onboarding {served} from e  ->  alias {served}-x\n"
                                                     + outcome[served][1], encoding="utf-8")
         return {"ok": True, "errors": [], "log": f"onboard-{served}.log"}, Proc(served)
@@ -397,7 +397,7 @@ def main() -> int:
         settle()
         stopped = {i["served"]: i["state"] for i in serve.onboard_queue()["items"]}
         order.remove("quiet")
-        serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m} for s, m in
+        serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m, "vision": s == "quiet"} for s, m in
                              (("ok", "full"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
         settle()
         batch = {i["served"]: (i["state"], i["alias"], bool(i["note"])) for i in serve.onboard_queue()["items"]}
@@ -412,7 +412,7 @@ def main() -> int:
           and batch == {"ok": ("done", "ok-x", False), "quiet": ("no answer", "quiet-x", True),
                         "bad": ("failed", "bad-x", True)}
           and cleared == [] and bad_input == (False, False, False)
-          and modes == {"a": "full", "ok": "full", "quiet": "quick", "bad": "quick-adhoc"})
+          and modes == {"a": "full", "ok": "full", "quiet": "quick vision", "bad": "quick-adhoc"})
     print(("ok   " if ok else "FAIL ") + "onboarding queue" + ("" if ok else f" -> {stopped} {order} {batch} {modes}"))
     failures += 0 if ok else 1
 

package/scripts/test_server_json.py

@@ -11,7 +11,10 @@ repository URL spells it, since the match is case-sensitive); every version is
 VERSION; package.json carries the name as mcpName (npm's proof) and the README
 PyPI shows carries `mcp-name: <name>` (PyPI's); the identifiers are the real
 package names; the arguments start the MCP server through launcher.py; the
-environment variables are the ones mcp_server.py reads. Standard library
+environment variables are the ones mcp_server.py reads. The root's mcp.json,
+mcp_config.json (Antigravity's name for it), plugin.json, .cursor-plugin and
+skills/ (cursor.directory, Glama and Antigravity read the repo root, not
+plugin/) start the same server and carry the plugin's skill. Standard library
 only: the full schema is validated by mcp-publisher in the release itself.
 """
 
@@ -71,7 +74,24 @@ def main() -> int:
         assert set(names) == read and len(names) == len(read), f"{kind}: lists {names}, mcp_server.py reads {read}"
         for e in p["environmentVariables"]:
             assert e.get("isSecret", False) == (e["name"] == "BENCHWARMERS_KEY"), e
-    print(f"server.json ok: {name} {version}, npm and pypi")
+
+    # The repo root as cursor.directory and Glama read it: mcp.json starts the npm
+    # package's server, and skills/ is the Claude Code plugin's skill, word for word.
+    root_mcp = json.loads((ROOT / "mcp.json").read_text(encoding="utf-8"))["mcpServers"]["benchwarmers"]
+    assert root_mcp["command"] == "npx" and root_mcp["args"][:2] == ["-y", npm["name"]], root_mcp
+    assert launcher.plan(root_mcp["args"][2:]).get("script") == "mcp_server.py", root_mcp["args"]
+    agy = json.loads((ROOT / "mcp_config.json").read_text(encoding="utf-8"))["mcpServers"]["benchwarmers"]
+    assert agy == root_mcp, f"mcp_config.json (Antigravity's) {agy} differs from mcp.json {root_mcp}"
+    assert json.loads((ROOT / "plugin.json").read_text(encoding="utf-8"))["name"] == "benchwarmers"
+    cursor = json.loads((ROOT / ".cursor-plugin" / "plugin.json").read_text(encoding="utf-8"))
+    assert (cursor["skills"], cursor["mcpServers"]) == ("./skills/", "./mcp.json"), cursor
+    lf = lambda p: p.read_text(encoding="utf-8").replace("\r\n", "\n")  # noqa: E731
+    copies = sorted(p.name for p in (ROOT / "skills").iterdir())
+    assert copies == sorted(p.name for p in (ROOT / "plugin" / "skills").iterdir()), copies
+    for skill in copies:
+        assert lf(ROOT / "skills" / skill / "SKILL.md") == lf(ROOT / "plugin" / "skills" / skill / "SKILL.md"), \
+            f"skills/{skill}/SKILL.md differs from plugin/skills/{skill}/SKILL.md: copy one over the other"
+    print(f"server.json ok: {name} {version}, npm and pypi; mcp.json, .cursor-plugin and skills/ match")
     return 0