@wendellresearch-ai/benchwarmers 0.3.3 → 0.5.0

package/scripts/test_hubs.py

@@ -278,16 +278,21 @@ def discover_add() -> None:
         os.environ["BENCHWARMERS_KEY_CPA_ADMIN"] = "mgmt-test"
 
     onboarded = []
-    serve.queue_onboard = lambda items, who="ui": onboarded.append((items[0]["served"], items[0]["endpoint"])) or {
+    serve.queue_onboard = lambda items, who="ui": onboarded.append(
+        (items[0]["served"], items[0]["endpoint"], items[0]["mode"], items[0]["vision"])) or {
         "ok": True, "errors": [], "items": []}
     owui.add = lambda c, m: (m, [f"owui added {m} on {c}"])
     r = serve.discover_add("groq", "vendor/new", endpoint="lite")
-    assert r["ok"] and "/lite/model/new" in writes() and onboarded[-1] == ("vendor/new", "lite"), r
-    r = serve.discover_add("openrouter", "vendor/y:free", endpoint="cpa")
-    assert r["ok"] and onboarded[-1] == ("vendor/y:free", "cpa"), r
+    assert r["ok"] and "/lite/model/new" in writes() and onboarded[-1] == ("vendor/new", "lite", "full", False), r
+    # The confirm's Onboard as and Vision box reach the queue; a bad mode is refused before the hub has it.
+    r = serve.discover_add("openrouter", "vendor/y:free", endpoint="cpa", mode="quick-adhoc", vision=True)
+    assert r["ok"] and onboarded[-1] == ("vendor/y:free", "cpa", "quick-adhoc", True), r
     assert [m for m, p, _b in SEEN if "management" in p and m == "PATCH"] == ["PATCH"]
+    before = len(SEEN)
+    assert "mode must be" in serve.discover_add("openrouter", "vendor/q", endpoint="cpa", mode="slow")["errors"][0]
+    assert len(SEEN) == before, SEEN[before:]
     r = serve.discover_add("groq-api", "vendor/z")                                 # no endpoint: Open WebUI, as before
-    assert r["steps"] == ["owui added vendor/z on groq-api"] and onboarded[-1] == ("vendor/z", "owui"), r
+    assert r["steps"] == ["owui added vendor/z on groq-api"] and onboarded[-1] == ("vendor/z", "owui", "full", False), r
     assert serve.discover_add("groq-api", "vendor/z", endpoint="owui")["ok"]
     assert "not a hub" in serve.discover_add("groq", "v/z", endpoint="plain")["errors"][0]
     changes = [json.loads(line)["field"] for line in serve.CHANGES_PATH.read_text(encoding="utf-8").splitlines()]

package/scripts/test_no_owui.py

@@ -132,6 +132,15 @@ def stranger(fake: str) -> None:
     assert c["ok"] and not c["errors"], c
     assert sorted(x["served"] for x in c["candidates"]) == ["vendor/c", "vendor/d", "vendor/e", "vendor/f"], c
     assert {x["group"] for x in c["candidates"]} == {"openrouter (direct)"} and not any(x["new"] for x in c["candidates"])
+    # Queued or onboarding now: left out until the queue is done with it.
+    kept_items = serve._OQ["items"]
+    serve._OQ["items"] = [{"served": "vendor/c", "endpoint": "openrouter", "state": "running"},
+                          {"served": "vendor/d", "endpoint": "openrouter", "state": "done"}]
+    try:
+        busy = sorted(x["served"] for x in serve.onboard_candidates()["candidates"])
+    finally:
+        serve._OQ["items"] = kept_items
+    assert busy == ["vendor/d", "vendor/e", "vendor/f"], busy
 
     # The served check: the registered models only, not the aggregator's catalogue.
     assert check_deprecations.served() == {"vendor/a:free", "vendor/b:free"}
@@ -241,12 +250,15 @@ def owner(fake: str) -> None:
     OWUI_MODELS.append({"id": "just-added", "urlIdx": 1})
     real_names = provider_tags.connection_names
     try:
-        live = {x["served"]: x["group"] for x in serve.onboard_candidates()["candidates"]}
+        # The picker's list is kept a few minutes (serve.LISTED_FOR); Refresh asks again.
+        kept = serve.onboard_candidates()
+        assert "just-added" not in {x["served"] for x in kept["candidates"]}, kept
+        live = {x["served"]: x["group"] for x in serve.onboard_candidates(fresh=True)["candidates"]}
 
         def refused(hub):
             raise OSError("401")
         provider_tags.connection_names = refused
-        c = serve.onboard_candidates()
+        c = serve.onboard_candidates(fresh=True)
     finally:
         provider_tags.connection_names = real_names
         OWUI_MODELS.pop()

package/scripts/test_quota.py

@@ -290,6 +290,16 @@ def main() -> int:
     check("rate limit (not sent" in r["error"] and "a calendar month" in r["error"] and "this month" in r["error"],
           "rpmo 3: the fourth call this month is refused", r["error"])
     check(client.chat("t-ok", "hi")["text"] != "", "another connection is not held by that cap")
+    # Seating asks the same question first (gateway.refused_now): held past any
+    # seat's wait, so not seated; a model with room is.
+    sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
+    import gateway
+    import pacing
+    h = pacing.held("t-month", "monthprov")
+    check(h is not None and h[0] > gateway.SEAT_TIMEOUT and "this month" in h[1], "held() sees the monthly cap", h)
+    check(pacing.held("t-ok", gateway.route.connection_of("t-ok")) is None, "held() is None with room")
+    out = gateway.refused_now(["t-month", "t-ok"], 10, gateway.SEAT_TIMEOUT)
+    check(list(out) == ["t-month"] and "rate limit" in out["t-month"], "refused_now names only the held model", out)
 
     # The free-tier guard: a model that went paid is held until allowed.
     before = len(seen.get("paid", []))
@@ -322,8 +332,8 @@ def main() -> int:
           and "rpmo" not in ((serve.catalog.overrides().get("t-ok") or {}).get("limits") or {}),
           "a bad rpmo is refused; a model's own limits have none", bad)
     page = serve.quotas_page()
-    check("What providers said is left" in page and "fakeprov" in page and "What the bench sent" in page,
-          "the Quotas page shows the records and the ledger's counts")
+    check("id='quota-data'" in page and "fakeprov" in page and "What the bench sent" in page,
+          "the Quotas page carries its tiles and the ledger's counts")
 
     srv.shutdown()
     print("\nthe ledger, quotas and pacing hold" if not failures else f"\n{failures} FAILED")

package/scripts/test_serve_edit.py

@@ -360,9 +360,9 @@ def main() -> int:
 
     modes = {}
 
-    def fake_spawn(served, endpoint="", who="ui", mode="full"):
+    def fake_spawn(served, endpoint="", who="ui", mode="full", vision=False):
         order.append(served)
-        modes[served] = mode
+        modes[served] = mode + (" vision" if vision else "")
         (runs / f"onboard-{served}.log").write_text(f"onboarding {served} from e  ->  alias {served}-x\n"
                                                     + outcome[served][1], encoding="utf-8")
         return {"ok": True, "errors": [], "log": f"onboard-{served}.log"}, Proc(served)
@@ -397,8 +397,8 @@ def main() -> int:
         settle()
         stopped = {i["served"]: i["state"] for i in serve.onboard_queue()["items"]}
         order.remove("quiet")
-        serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m} for s, m in
-                             (("ok", "full"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
+        serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m, "vision": s == "quiet"} for s, m in
+                             (("ok", "full-adhoc"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
         settle()
         batch = {i["served"]: (i["state"], i["alias"], bool(i["note"])) for i in serve.onboard_queue()["items"]}
         cleared = serve.clear_onboard_queue()["items"]
@@ -412,10 +412,55 @@ def main() -> int:
           and batch == {"ok": ("done", "ok-x", False), "quiet": ("no answer", "quiet-x", True),
                         "bad": ("failed", "bad-x", True)}
           and cleared == [] and bad_input == (False, False, False)
-          and modes == {"a": "full", "ok": "full", "quiet": "quick", "bad": "quick-adhoc"})
+          and modes == {"a": "full", "ok": "full-adhoc", "quiet": "quick vision", "bad": "quick-adhoc"}
+          and serve.ONBOARD_MODES["full-adhoc"] == ["--ad-hoc"])            # the suite now, then ad hoc
     print(("ok   " if ok else "FAIL ") + "onboarding queue" + ("" if ok else f" -> {stopped} {order} {batch} {modes}"))
     failures += 0 if ok else 1
 
+    # Restart: refused unless serve.py's parent started it (nothing would start it
+    # again); refused while a job runs, unless forced; then the server shuts down.
+    import scheduler
+    kept = os.environ.get("BENCH_SERVED"), serve.SERVER
+    shut = []
+    try:
+        os.environ.pop("BENCH_SERVED", None)
+        alone = serve.restart()
+        os.environ["BENCH_SERVED"] = "1"
+        serve.SERVER = type("Server", (), {"shutdown": lambda self: shut.append(1)})()
+        scheduler.RUNNING["suite"] = True
+        busy, forced = serve.restart(), serve.restart(force=True)
+        time.sleep(0.8)                                  # the shutdown waits half a second
+    finally:
+        scheduler.RUNNING.pop("suite", None)
+        serve.SERVER = kept[1]
+        os.environ.pop("BENCH_SERVED", None) if kept[0] is None else os.environ.update(BENCH_SERVED=kept[0])
+    ok = (not alone["ok"] and busy.get("busy") and not busy["ok"] and "suite" in busy["errors"][0]
+          and forced["ok"] and forced["cut"] and shut == [1])
+    print(("ok   " if ok else "FAIL ") + "restart" + ("" if ok else f" -> {alone} {busy} {forced} {shut}"))
+    failures += 0 if ok else 1
+
+    # Action needed: an alert whose models have all left the bench is not shown,
+    # before the daily check rewrites the file (2026-10-28: a kilo connection
+    # alert stayed after Retire all 4).
+    kept_data, kept_reg = serve.DATA, serve.registered
+    with tempfile.TemporaryDirectory() as tmp:
+        (pathlib.Path(tmp) / "metrics").mkdir()
+        (pathlib.Path(tmp) / "metrics" / "alerts.json").write_text(json.dumps({
+            "connection:held:gone.example": {"kind": "connection", "message": "every model on gone.example is failing",
+                                             "aliases": ["g1", "g2"], "first_seen": "2026-10-28"},
+            "connection:held:here.example": {"kind": "connection", "message": "every model on here.example is failing",
+                                             "aliases": ["h1", "g1"], "first_seen": "2026-10-28"},
+            "retiring:x:old": {"kind": "retiring", "alias": "old", "message": "old retires", "first_seen": "2026-10-28"}}),
+            encoding="utf-8")
+        serve.DATA, serve.registered = pathlib.Path(tmp), lambda: {"h1-served": "h1"}
+        try:
+            shown = serve.problems()
+        finally:
+            serve.DATA, serve.registered = kept_data, kept_reg
+    ok = "here.example" in shown and "gone.example" not in shown and "old retires" not in shown
+    print(("ok   " if ok else "FAIL ") + "action needed drops alerts on retired models")
+    failures += 0 if ok else 1
+
     # Dismiss: one picker key or many in one write, a change-log line each;
     # undo brings back only the ones named; nothing named is refused.
     serve.DISMISSED_PATH = cfg / "dismissed.json"
@@ -433,6 +478,14 @@ def main() -> int:
     print(("ok   " if ok else "FAIL ") + "dismiss in bulk, and undo" + ("" if ok else f" -> {gone} {left} {log} {refused}"))
     failures += 0 if ok else 1
 
+    # The member picker's provider chips: a proxy's vendor, else the bare host.
+    names = {"nvidia-api": "nvidia", "api.unorouter.com": "unorouter", "router.requesty.ai": "requesty",
+             "cli-proxy-api/codex": "codex", "openrouter": "openrouter", "apihub.agnes-ai.com": "agnes-ai"}
+    got = {t: serve.provider_label(t) for t in names}
+    ok = got == names
+    print(("ok   " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
+    failures += 0 if ok else 1
+
     # The owner's files are as they were. alerts.json can change under a
     # running suite, so only what this test could do counts: the victim's
     # alerts cleared from the real file.

package/scripts/test_server_json.py

@@ -11,7 +11,10 @@ repository URL spells it, since the match is case-sensitive); every version is
 VERSION; package.json carries the name as mcpName (npm's proof) and the README
 PyPI shows carries `mcp-name: <name>` (PyPI's); the identifiers are the real
 package names; the arguments start the MCP server through launcher.py; the
-environment variables are the ones mcp_server.py reads. Standard library
+environment variables are the ones mcp_server.py reads. The root's mcp.json,
+mcp_config.json (Antigravity's name for it), plugin.json, .cursor-plugin and
+skills/ (cursor.directory, Glama and Antigravity read the repo root, not
+plugin/) start the same server and carry the plugin's skill. Standard library
 only: the full schema is validated by mcp-publisher in the release itself.
 """
 
@@ -71,7 +74,24 @@ def main() -> int:
         assert set(names) == read and len(names) == len(read), f"{kind}: lists {names}, mcp_server.py reads {read}"
         for e in p["environmentVariables"]:
             assert e.get("isSecret", False) == (e["name"] == "BENCHWARMERS_KEY"), e
-    print(f"server.json ok: {name} {version}, npm and pypi")
+
+    # The repo root as cursor.directory and Glama read it: mcp.json starts the npm
+    # package's server, and skills/ is the Claude Code plugin's skill, word for word.
+    root_mcp = json.loads((ROOT / "mcp.json").read_text(encoding="utf-8"))["mcpServers"]["benchwarmers"]
+    assert root_mcp["command"] == "npx" and root_mcp["args"][:2] == ["-y", npm["name"]], root_mcp
+    assert launcher.plan(root_mcp["args"][2:]).get("script") == "mcp_server.py", root_mcp["args"]
+    agy = json.loads((ROOT / "mcp_config.json").read_text(encoding="utf-8"))["mcpServers"]["benchwarmers"]
+    assert agy == root_mcp, f"mcp_config.json (Antigravity's) {agy} differs from mcp.json {root_mcp}"
+    assert json.loads((ROOT / "plugin.json").read_text(encoding="utf-8"))["name"] == "benchwarmers"
+    cursor = json.loads((ROOT / ".cursor-plugin" / "plugin.json").read_text(encoding="utf-8"))
+    assert (cursor["skills"], cursor["mcpServers"]) == ("./skills/", "./mcp.json"), cursor
+    lf = lambda p: p.read_text(encoding="utf-8").replace("\r\n", "\n")  # noqa: E731
+    copies = sorted(p.name for p in (ROOT / "skills").iterdir())
+    assert copies == sorted(p.name for p in (ROOT / "plugin" / "skills").iterdir()), copies
+    for skill in copies:
+        assert lf(ROOT / "skills" / skill / "SKILL.md") == lf(ROOT / "plugin" / "skills" / skill / "SKILL.md"), \
+            f"skills/{skill}/SKILL.md differs from plugin/skills/{skill}/SKILL.md: copy one over the other"
+    print(f"server.json ok: {name} {version}, npm and pypi; mcp.json, .cursor-plugin and skills/ match")
     return 0