@wendellresearch-ai/benchwarmers 0.5.5 → 0.5.6

package/metrics/discover.py

@@ -1,6 +1,7 @@
-"""One list of models worth adding, from FreeLLMAPI and models.dev, deduped.
+"""One list of models worth adding, from FreeLLMAPI, models.dev and the providers' own lists, deduped.
 
     python metrics/discover.py           # summary: free models by reach, new this week
+    python metrics/discover.py --refresh # fetch every list now, as Discover's Refresh does
 
 The shopping list: Discover's Add (serve.py) puts a model on the bench by one
 of the routes below. A model offered by the same provider in both sources
@@ -34,7 +35,11 @@ ROUTES
 ------
 The ways this install can reach a provider, each a row's `routes`:
   openwebui    a connection in Open WebUI, known because a bench model on the
-               Open WebUI hub (endpoints.find_hub) is served from it. Adding
+               Open WebUI hub (endpoints.find_hub) is served from it, or from
+               the providers' lists (below). A row is that connection's when
+               the provider's API host is the connection's, or its id
+               (models.dev's or FreeLLMAPI's) is the connection's name or a
+               part of its host: FreeLLMAPI's xkiro is api.xkiro.com. Adding
                goes through metrics/owui.py, so the model also appears in the
                chats. None without an Open WebUI hub.
   direct       an endpoint of its own in config/endpoints.json whose name or
@@ -50,9 +55,28 @@ The ways this install can reach a provider, each a row's `routes`:
 The same model offered by two providers (OpenRouter and NVIDIA) has a route
 on each; the dashboard's Discover popup lists them together by `norm`.
 
-NEW is the date a model was first seen in either list, kept in
+THE PROVIDERS' OWN LISTS
+------------------------
+The catalogs trail the providers by days, so a model a provider started
+serving this morning is in neither. Open WebUI passes each connection's own
+/models through (owui.lists), whatever its Model IDs list allows, so
+refresh() keeps every connection's in runs/cache/provider-lists.json. A model
+there that neither catalog has for that provider is a row of its own, source
+"provider": its id, and what the list says of its name, context, price and
+abilities (each provider says different things; many say nothing of price,
+which is then unknown, not paid). Only through Open WebUI: the providers'
+keys stay where they are, and nothing is sent anywhere new.
+
+LINKS
+-----
+A row's `page` is its freellmapi.co page (where the sitemap has one), and
+`md_page` its models.dev page: /models/<lab>/<id> by models.dev's canonical
+id, else the provider's own page there.
+
+NEW is the date a model was first seen in any list, kept in
 runs/discover-seen.json (gitignored). The first build records everything as a
-baseline, so only models that appear afterwards are badged new.
+baseline, so only models that appear afterwards are badged new; so does the
+first list of each connection.
 """
 
 from __future__ import annotations
@@ -61,6 +85,7 @@ import datetime as dt
 import json
 import pathlib
 import sys
+from urllib.parse import urlsplit
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
@@ -74,6 +99,84 @@ FREELLMAPI_TO_CONNECTION = {"groq": "groq", "nvidia": "nvidia-api", "mistral": "
                             "openrouter": "openrouter", "ollama": "ollama-cloud",
                             "cerebras": "cerebras", "llm7": "llm7-api"}
 NEW_DAYS = 7
+LISTS = "provider-lists"                 # runs/cache/provider-lists.json, written by refresh()
+
+
+def refresh(force: bool = False) -> list[str]:
+    """Fetch the catalogs (sources.FEEDS) and the providers' own lists, each at
+    most once a day unless `force`. What went wrong, as sentences: a list that
+    could not be fetched keeps its cached copy."""
+    import endpoints
+    said = []
+    for name in sources.FEEDS:
+        try:
+            sources.fetch(name, force)
+        except Exception as exc:                       # noqa: BLE001
+            said.append(f"could not fetch {name} ({exc}); kept the cached copy")
+    age = sources.age_s(LISTS)
+    if not endpoints.find_hub("openwebui") or (not force and age is not None and age < sources.MAX_AGE_S):
+        return said
+    import owui
+    try:
+        got, failed = owui.lists()
+    except Exception as exc:                           # noqa: BLE001
+        return said + [f"could not ask Open WebUI for its providers' lists ({exc}); kept the cached copy"]
+    old = {c["names"][0]: c for c in sources.cached(LISTS).get("connections") or []}
+    conns = [{**c, "models": [s for s in map(_slim, c["models"]) if s]} for c in got]
+    sources.store(LISTS, {"at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
+                          "connections": conns + [old[n] for n in failed if n in old]})
+    return said + [f"{n} would not list its models ({why})" + ("; kept its last list" if n in old else "")
+                   for n, why in failed.items()]
+
+
+def _slim(m: dict) -> dict | None:
+    """What Discover keeps of one model in a provider's own list; None when the
+    list says it is not a chat model (images, audio, embeddings, another API).
+    Each provider says it its own way: these are the ways seen 2026-11-07 in
+    OpenRouter's, Requesty's, xKiro's, Mistral's and UnoRouter's lists."""
+    arch = m.get("architecture") if isinstance(m.get("architecture"), dict) else {}
+    caps = m.get("capabilities") if isinstance(m.get("capabilities"), dict) else {}
+    outs = m.get("output_modalities") or arch.get("output_modalities")
+    ins = m.get("input_modalities") or arch.get("input_modalities")
+    apis = m.get("supported_endpoint_types")
+    if ((isinstance(outs, list) and outs and "text" not in outs) or m.get("modality") not in (None, "chat", "text")
+            or m.get("api") not in (None, "chat") or caps.get("completion_chat") is False
+            or (isinstance(apis, list) and apis and "openai" not in apis)):
+        return None
+    tier, price = m.get("access_tier"), m.get("pricing") if isinstance(m.get("pricing"), dict) else {}
+    nums = []
+    for v in (price.get("prompt"), price.get("completion"), price.get("input"), price.get("output"),
+              m.get("input_price"), m.get("output_price")):
+        try:
+            nums.append(float(v))
+        except (TypeError, ValueError):
+            pass
+    vision = caps.get("vision", m.get("supports_vision"))
+    if vision is None and isinstance(ins, list):
+        vision = "image" in ins
+    ctx = m.get("context_length") or m.get("context_window") or m.get("max_context_length")
+    return {"id": str(m["id"]), "name": m.get("display_name") or m.get("name"),
+            "context": ctx if isinstance(ctx, int) and ctx > 0 else None,
+            "free": (True if str(m["id"]).endswith(":free") or tier == "free" else
+                     False if tier in ("paid", "premium") else all(x == 0 for x in nums) if nums else None),
+            "tools": caps.get("tools", caps.get("function_calling", m.get("supports_tool_calling"))),
+            "vision": vision, "reasoning": caps.get("reasoning", m.get("supports_reasoning"))}
+
+
+def _host(base: str) -> str:
+    return urlsplit(base).hostname or "" if "//" in base else base
+
+
+def _conn_for(ids: list[str], api: str | None, hosts: dict[str, str]) -> str | None:
+    """The Open WebUI connection (`hosts`: name -> its API host, the bench's
+    names first) that serves this provider: the provider's API host is its
+    host, or one of the provider's `ids` is its name or a part of its host."""
+    want = _host(api or "")
+    for name, host in hosts.items():
+        parts = host.split(".")[:-1] if "." in host else [host]
+        if (want and host == want) or any(i and (i == name or i in parts) for i in ids):
+            return name
+    return None
 
 
 def _on_bench() -> list[tuple[str, str, str, str, str]]:
@@ -168,14 +271,29 @@ def build() -> list[dict]:
     platform_names = {p.get("id"): p.get("name") for p in fl.get("platforms") or []}
     md_conn = {v: k for k, v in sources.connection_to_modelsdev().items()}
     bench = _on_bench()
-    # Open WebUI's connections, as far as the bench can tell without asking it.
+    # Open WebUI's connections, as far as the bench can tell without asking it:
+    # its models' connections, then the providers' lists, with their API hosts.
     owui = endpoints.find_hub("openwebui")
-    owui_conns = {c for ep, c, *_ in bench if ep == owui} - {"?"}
+    lists = (sources.cached(LISTS).get("connections") or []) if owui else []
+    hosts: dict[str, str] = dict.fromkeys(c for ep, c, *_ in bench if ep == owui and c != "?")
+    for c in lists:
+        for n in c["names"]:
+            hosts[n] = _host(c["base"])
+    hosts = {n: h or n for n, h in hosts.items()}
+    owui_conns = set(hosts)
+
+    def conn_of(ids: list[str], api: str | None, known: str | None) -> str | None:
+        return known if known in hosts else _conn_for(ids, api, hosts) or known
+
     direct = _direct(md)
     rows: dict[tuple[str, str], dict] = {}
+    # models.dev's own page for a model: by its canonical id, which names its lab.
+    labs = {str(m["canonical_model_id"]).split("/")[0] for p in md.values()
+            for m in (p.get("models") or {}).values() if m.get("canonical_model_id")}
 
     for prov, p in md.items():
         openai_like = bool(p.get("api")) or p.get("npm") == "@ai-sdk/openai-compatible"
+        conn = conn_of([prov], p.get("api"), md_conn.get(prov))
         for mid, m in (p.get("models") or {}).items():
             out_mods = (m.get("modalities") or {}).get("output") or ["text"]
             if "text" not in out_mods or m.get("status") == "deprecated":
@@ -193,8 +311,10 @@ def build() -> list[dict]:
                 "vision": "image" in ((m.get("modalities") or {}).get("input") or []),
                 "released": m.get("release_date"), "knowledge": m.get("knowledge"),
                 "open_weights": m.get("open_weights"), "api": p.get("api"),
-                "openai_like": openai_like, "connection": md_conn.get(prov), "limits": None,
-                "rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None}
+                "openai_like": openai_like, "connection": conn, "limits": None,
+                "rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None,
+                "md_page": "https://models.dev/" + (f"models/{m['canonical_model_id']}" if m.get("canonical_model_id")
+                                                    else f"models/{prov}/{mid}" if prov in labs else f"providers/{prov}")}
 
     for m in fl.get("models") or []:
         if (m.get("modality") or "chat") != "chat":
@@ -222,8 +342,30 @@ def build() -> list[dict]:
                 "cost": None, "context": m.get("contextWindow"), "output": None,
                 "tools": m.get("supportsTools"), "reasoning": None,
                 "vision": m.get("supportsVision"), "released": None, "knowledge": None,
-                "open_weights": None, "api": None,
-                "openai_like": False, "connection": FREELLMAPI_TO_CONNECTION.get(plat)}
+                "open_weights": None, "api": None, "md_page": None,
+                "openai_like": False, "connection": conn_of([prov, plat], None, FREELLMAPI_TO_CONNECTION.get(plat))}
+
+    # What each connection's provider lists that neither catalog has for it.
+    listed = set()
+    for c in lists:
+        conn = c["names"][0]                     # the bench's name for it, as conn_of picks
+        provs = sorted({r["provider"] for r in rows.values() if r["connection"] in c["names"]}) or [conn]
+        have = {k[1] for k in rows if k[0] in provs}
+        name = next((r["provider_name"] for r in rows.values() if r["provider"] == provs[0]), c["names"][-1])
+        listed.add(conn)
+        for m in c["models"]:
+            key = (provs[0], sources.norm_model(m["id"]))
+            if key[1] in have:
+                continue
+            have.add(key[1])
+            rows[key] = {
+                "in_mods": ["text", "image"] if m.get("vision") else ["text"], "out_mods": ["text"],
+                "provider": provs[0], "provider_name": name, "model": m["id"], "name": m.get("name") or m["id"],
+                "sources": ["provider"], "free": m.get("free"), "cost": None, "context": m.get("context"),
+                "output": None, "tools": m.get("tools"), "reasoning": m.get("reasoning"), "vision": m.get("vision"),
+                "released": None, "knowledge": None, "open_weights": None, "api": c["base"], "openai_like": True,
+                "connection": conn, "limits": None, "rank": None, "speed": None, "size": None, "budget": None,
+                "quirks": [], "page": None, "md_page": None}
 
     # The same model through ANOTHER provider is fine to add, but worth knowing.
     by_model: dict[str, list[str]] = {}
@@ -234,12 +376,14 @@ def build() -> list[dict]:
     seen = _read_seen()
     today = dt.date.today().isoformat()
     baseline = not seen
+    first_list = {c for c in listed if f"list|{c}" not in seen}   # a connection's first list is its baseline
     walled = {p for p, on in sources.free_in_own_client_only().items() if on}
     out = []
     for key, r in rows.items():
         k = "|".join(key)
         if k not in seen:
-            seen[k] = "baseline" if baseline else today
+            seen[k] = ("baseline" if baseline or (r["sources"] == ["provider"] and r["connection"] in first_list)
+                       else today)
         r["first_seen"] = seen[k]
         r["new"] = seen[k] not in ("baseline",) and seen[k] >= (dt.date.today() - dt.timedelta(days=NEW_DAYS)).isoformat()
         if r["free"] and r["provider"] in walled:
@@ -257,6 +401,7 @@ def build() -> list[dict]:
                       "connected" if r["routes"] else
                       "easy to add" if r["openai_like"] else "other")
         out.append(r)
+    seen.update({f"list|{c}": today for c in first_list})
     _write_seen(seen)
     order = {"connected": 0, "easy to add": 1, "on bench": 2, "other": 3}
     # Within each reach, FreeLLMAPI's smartest first; the models it does not rank
@@ -343,6 +488,38 @@ def _selftest() -> None:
     assert big["page"] == "https://freellmapi.co/model/big-one" and big["quirks"] == [
         {"title": "T", "body": "B", "severity": "info"}] and big["vision"], big
     assert [r["model"] for r in rows] == ["acme/tiny", "acme/big", "acme/small"], [r["model"] for r in rows]
+    # A connection is found by its API host or by the provider's id in its host; the bench's name wins.
+    hosts = {"xk": "api.xkiro.com", "api.xkiro.com": "api.xkiro.com", "router.requesty.ai": "router.requesty.ai",
+             "cli-proxy-api": "203.0.113.7"}
+    assert _conn_for(["xkiro"], None, hosts) == "xk"
+    assert _conn_for(["requesty"], "https://router.requesty.ai/v1", hosts) == "router.requesty.ai"
+    assert _conn_for(["com"], None, hosts) is None and _conn_for(["acme"], "https://acme.example/v1", hosts) is None
+    # A provider's own list: chat models only, price where it says one, unknown where it does not.
+    assert _slim({"id": "a", "architecture": {"output_modalities": ["image"]}}) is None
+    assert _slim({"id": "b", "supported_endpoint_types": ["aihorde"]}) is None
+    assert _slim({"id": "c", "capabilities": {"completion_chat": False}}) is None
+    assert _slim({"id": "d:free"})["free"] is True and _slim({"id": "e"})["free"] is None
+    assert _slim({"id": "f", "pricing": {"prompt": "0", "completion": "0"}})["free"] is True
+    assert _slim({"id": "g", "input_price": 2.5e-07, "output_price": 1e-06})["free"] is False
+    assert _slim({"id": "h", "access_tier": "premium", "capabilities": {"vision": True}, "context_length": 9})[
+        "free"] is False and _slim({"id": "h", "capabilities": {"vision": True}})["vision"] is True
+    # With a list cached: acme's connection is found by its host, the list's new model is a row of its
+    # own (the first list of a connection is a baseline, so not new), and models.dev links each model.
+    feeds["provider-lists"] = {"connections": [{"names": ["acme-conn"], "base": "https://acme.example/v1",
+                                                "models": [_slim({"id": "acme/big"}), _slim({"id": "acme/fresh"})]}]}
+    import endpoints
+    try:
+        sources.cached = lambda name: feeds.get(name, {})
+        globals().update(_on_bench=lambda: [], SEEN=pathlib.Path(tempfile.mkdtemp()) / "seen.json")
+        endpoints_hub, endpoints.find_hub = endpoints.find_hub, lambda which: "owui-ep" if which == "openwebui" else None
+        rows = {r["model"]: r for r in build() if r["provider"] == "acme"}
+    finally:
+        sources.cached, endpoints.find_hub = kept[0], endpoints_hub
+        globals().update(_on_bench=kept[1], SEEN=kept[2])
+    assert set(rows) == {"acme/big", "acme/small", "acme/tiny", "acme/fresh"}, sorted(rows)
+    assert rows["acme/big"]["connection"] == "acme-conn" and rows["acme/big"]["reach"] == "connected", rows["acme/big"]
+    assert rows["acme/fresh"]["sources"] == ["provider"] and not rows["acme/fresh"]["new"], rows["acme/fresh"]
+    assert rows["acme/big"]["md_page"] == "https://models.dev/providers/acme", rows["acme/big"]["md_page"]
     print("discover selftest ok")
 
 
@@ -351,6 +528,9 @@ def main() -> None:
     if "--selftest" in sys.argv:
         _selftest()
         return
+    if "--refresh" in sys.argv:
+        for line in refresh(force=True) or ["every list fetched"]:
+            print(" ", line)
     rows = build()
     free = [r for r in rows if r["free"]]
     by = {}

package/metrics/owui.py

@@ -114,6 +114,32 @@ def connections() -> dict[str, tuple[int, str]]:
     return out
 
 
+def lists() -> tuple[list[dict], dict[str, str]]:
+    """What each connection's provider serves now, by its own /models, which
+    Open WebUI passes through at /openai/models/<index> whatever the
+    connection's Model IDs list allows: [{"names", "base", "models"}], names
+    as connections() has them (the bench's first), and {name: why} for the
+    connections that would not say. Switched-off connections are left out."""
+    names: dict[int, list[str]] = collections.defaultdict(list)
+    for name, (idx, _tag) in connections().items():
+        names[idx].append(name)
+    cfg = _call("/openai/config")
+    bases = cfg.get("OPENAI_API_BASE_URLS") or []
+    out, failed = [], {}
+    for idx, ns in sorted(names.items()):
+        if idx >= len(bases) or (cfg.get("OPENAI_API_CONFIGS") or {}).get(str(idx), {}).get("enable") is False:
+            continue
+        try:
+            r = _call(f"/openai/models/{idx}")
+        except Exception as exc:                       # noqa: BLE001
+            failed[ns[0]] = str(exc)
+            continue
+        data = r.get("data") if isinstance(r, dict) else r
+        out.append({"names": ns, "base": bases[idx],
+                    "models": [m for m in data or [] if isinstance(m, dict) and m.get("id")]})
+    return out, failed
+
+
 def plan(connection: str, model_id: str) -> dict:
     """What adding would change. Raises ValueError when it cannot be done here."""
     release, note = version()

package/metrics/route.py

@@ -273,7 +273,7 @@ def suite_recall() -> dict[str, tuple[int, int, int]]:
 def panel(shape: str | None = None, large: bool = True,
           nominated: list[str] | None = None,
           members: list[str] | None = None, explorers: int = 2,
-          avoid: set[str] | None = None) -> dict:
+          avoid: set[str] | None = None, stand_ins: int = 0) -> dict:
     """`large=True` means the task is a big one — most of ours are.
 
     `members` is a bench-gateway COLLECTION: models the caller always runs. With
@@ -282,7 +282,8 @@ def panel(shape: str | None = None, large: bool = True,
     never with a member or a member's family. `avoid` keeps models out of
     those automatic picks (the gateway passes the ones on a retiring alert, so
     a seat is not spent on a model already known to be failing); a nomination
-    still seats one.
+    still seats one. `stand_ins` more are picked the same way, for members out
+    of quota (the gateway); fewer come back if the pool runs out.
 
     `nominated` names models the operator wants on the panel today, and they
     take EXPLORER seats — never the primary or the second.
@@ -469,6 +470,12 @@ def panel(shape: str | None = None, large: bool = True,
 
     explorer, explorer2 = seats[0], seats[1]
 
+    standing_in = []
+    while len(standing_in) < stand_ins and (pick := stalest()):
+        chosen.add(pick["model"])
+        seated_families.add(pick["family"])
+        standing_in.append(pick)
+
     # RESERVE: the fallback when the panel meets a rate limit.
     #
     # This replaces the hard rule that seats must sit on different
@@ -489,7 +496,7 @@ def panel(shape: str | None = None, large: bool = True,
                       key=lambda r: (r["found"], paid(r)), default=None)
 
     return {"primary": primary, "second": second, "explorer": explorer,
-            "explorer2": explorer2, "reserve": reserve,
+            "explorer2": explorer2, "reserve": reserve, "stand_ins": standing_in,
             "considered": len(rows),
             "untried": sum(1 for r in rows if not r["offloads"]),
             "contenders": len(contenders),

package/metrics/sources.py

@@ -93,17 +93,30 @@ FREELLMAPI_TO_MODELSDEV = {
 def fetch(name: str, force: bool = False) -> dict:
     """A feed, from the cache when it is under a day old."""
     path = CACHE / f"{name}.json"
-    if not force and path.exists() and time.time() - path.stat().st_mtime < MAX_AGE_S:
+    age = age_s(name)
+    if not force and age is not None and age < MAX_AGE_S:
         return json.loads(path.read_text(encoding="utf-8"))
     req = urllib.request.Request(FEEDS[name], headers={"User-Agent": "benchwarmers"})
     with urllib.request.urlopen(req, timeout=90) as r:
         data = (json.load(r) if not FEEDS[name].endswith(".xml") else
                 {"slugs": sorted(set(re.findall(r"/model/([^<\"/]+)<", r.read().decode("utf-8"))))})
+    store(name, data)
+    return data
+
+
+def store(name: str, data: dict) -> None:
+    """Write one cached list whole, never half."""
     CACHE.mkdir(parents=True, exist_ok=True)
+    path = CACHE / f"{name}.json"
     tmp = path.with_suffix(".part")
     tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
     tmp.replace(path)
-    return data
+
+
+def age_s(name: str) -> float | None:
+    """Seconds since a cached list was written; None if it never was."""
+    path = CACHE / f"{name}.json"
+    return time.time() - path.stat().st_mtime if path.exists() else None
 
 
 def cached(name: str) -> dict:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/test_formats.py

@@ -274,7 +274,8 @@ def gateway_routes() -> None:
                                "merge": {"members": ["m1", "m2"], "reply": "merged", "merger": "mg", "description": "x"},
                                "broken": {"members": ["m1", "boom"], "description": "x"},
                                "small": {"members": ["tiny"], "description": "x"},
-                               "other": {"members": ["m2"], "description": "x"}})
+                               "other": {"members": ["m2"], "description": "x"},
+                               "dear": {"members": ["m1"], "description": "held back", "ad_hoc": True}})
     import gateway
     import guard
     import serve
@@ -493,6 +494,65 @@ def gateway_routes() -> None:
         one = json.loads(ok("/v1beta/models/bench/core")[0])
         assert one["name"] == "models/bench/core" and "generateContent" in one["supportedGenerationMethods"]
         assert call("/v1beta/models/bench/nope")[0] == 404
+        # Ad hoc: on no list and in no refusal, so nobody picks it; named, it runs and can be looked up.
+        for listing in (ok("/v1/models")[0], ok("/api/tags")[0], ok("/v1beta/models")[0]):
+            assert "bench/dear" not in listing and "bench/core" in listing, listing
+        assert "bench/dear" not in call("/v1/chat/completions", {"model": "bench/nope", "messages": [
+            {"role": "user", "content": user}]})[2]
+        assert ok("/api/show", {"model": "bench/dear"}) and ok("/v1beta/models/bench/dear")
+        asked = json.loads(ok("/v1/chat/completions", {"model": "bench/dear", "messages": [
+            {"role": "user", "content": user}]})[0])
+        assert asked["model"] == "bench/dear" and [s["alias"] for s in asked["bench"]["seats"]] == ["m1"], asked
+        seats.asked.clear()
+        made = call("/api/collection", {"name": "dear", "members": ["m1"], "ad_hoc": True}, on=dash.server_port)
+        assert json.loads(made[2])["ok"] and serve.bench_collections.load()["dear"]["ad_hoc"] is True, made
+        made = call("/api/collection", {"name": "dear", "members": ["m1"]}, on=dash.server_port)
+        assert json.loads(made[2])["ok"] and "bench/dear" in ok("/v1/models")[0], made     # unticked: on offer again
+        serve.delete_collection("dear")
+
+        # A COLLECTION'S OWN LIMITS: above the gateway's 3 and 0 here, checked, undone by null.
+        def save(name, members, **own):
+            made = call("/api/collection", {"name": name, "members": members, "shape": "general", **own}, on=dash.server_port)
+            return json.loads(made[2])
+        def ask_it(name):
+            return json.loads(ok("/v1/chat/completions", {"model": f"bench/{name}", "messages": [
+                {"role": "user", "content": "x"}], "stream": False})[0])
+        for bad in ({"max_calls": 11}, {"max_calls": 0}, {"max_explorers": 3}, {"max_calls": "4"}, {"max_explorers": True}):
+            assert not save("own", ["m1"], **bad)["ok"], bad
+        assert not save("own", ["m1", "m2", "mg"], max_calls=2)["ok"]           # its own cap, not the gateway's
+        assert save("wide", ["m1", "m2", "mg", "boom"], max_calls=4)["ok"]      # above the gateway's
+        assert "wide" in serve.bench_collections.load() and not serve.save_gateway_settings(1, 0)["errors"][0].count("wide")
+        assert save("own", ["m1", "m2"], max_calls=4, max_explorers=1)["ok"]
+        listed_own = [m for m in json.loads(ok("/v1/models")[0])["data"] if m["id"] == "bench/own"][0]
+        assert listed_own["explorers"] == 1 and listed_own["max_calls"] == 4, listed_own
+        # (boom held throughout: the stub raises for it, and an explorer seat could pick it.)
+        real_refused = gateway.refused_now
+        gateway.refused_now = lambda aliases, *_: {a: "out of quota" for a in aliases if a == "boom"}
+        assert len(ask_it("own")["bench"]["seats"]) == 3                        # two members, its own explorer
+        # STAND-INS: a held member stays on the collection, named, with a seat in its place.
+        gateway.refused_now = lambda aliases, *_: {a: "out of quota" for a in aliases if a in ("m1", "boom")}
+        b = ask_it("own")["bench"]
+        by_seat = {s["seat"]: s["alias"] for s in b["seats"]}
+        assert [k["alias"] for k in b["skipped"]] == ["m1"] and "m1" not in by_seat.values(), b
+        assert len(b["seats"]) == 3 and "stand-in for m1" in by_seat and by_seat["stand-in for m1"] != "m2", b
+        assert "m1" in serve.bench_collections.load()["own"]["members"]
+        # Max explorers 0: only the members, here and when the gateway's own 0 applies.
+        assert save("own", ["m1", "m2"], max_calls=4, max_explorers=0)["ok"]
+        assert [s["alias"] for s in ask_it("own")["bench"]["seats"]] == ["m2"]
+        assert save("own", ["m1", "m2"], max_calls=None, max_explorers=None)["ok"]
+        assert "max_calls" not in serve.bench_collections.load()["own"]
+        assert [s["alias"] for s in ask_it("own")["bench"]["seats"]] == ["m2"]
+        gateway.refused_now = lambda aliases, *_: {a: "out of quota" for a in aliases}
+        assert call("/v1/chat/completions", {"model": "bench/own", "messages": [{"role": "user", "content": "x"}]})[0] == 429
+        gateway.refused_now = real_refused
+        # A connection the alerts say has stopped answering counts as held.
+        import alerts
+        real_active, alerts.active = alerts.active, lambda: {"c": {"kind": "connection", "message": "not answering", "aliases": ["m1"]}}
+        assert gateway.refused_now(["m1", "m2"], 10, 1) == {"m1": "not answering"}
+        alerts.active = real_active
+        serve.delete_collection("own")
+        serve.delete_collection("wide")
+        seats.asked.clear()
         # Token counts are estimates, without a model call; an image counts IMAGE_TOKENS.
         counted = {"model": "bench/core", "messages": [
             {"role": "user", "content": [{"type": "text", "text": "x" * 400},

package/scripts/test_serve_edit.py

@@ -514,6 +514,25 @@ def main() -> int:
     print(("ok   " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
     failures += 0 if ok else 1
 
+    # An answer clears every alert it disproves (2026-11-02: big-pickle answered a
+    # Retry and stayed listed, its unrecognised-failure alert open), never a price;
+    # a held connection loses the model, and goes with its last one.
+    alerts.raise_alert("model:unclassified:t/served", "registration", "t fails oddly")
+    alerts.raise_alert("model:pending:t/served", "retiring", "t retires", alias="t")
+    alerts.raise_alert("model:paid:t/served", "went paid", "t has a price", alias="t")
+    alerts.raise_alert("model:pending:t/served-too", "retiring", "another retires", alias="t2")
+    alerts.raise_alert("connection:held:x", "connection", "all fail", aliases=["t", "t2"])
+    alerts.raise_alert("connection:held:y", "connection", "all fail", aliases=["t"])
+    cleared = alerts.answered("t", "t/served")
+    left = alerts.active()
+    ok = (set(cleared) == {"model:unclassified:t/served", "model:pending:t/served", "connection:held:y"}
+          and "model:paid:t/served" in left and "model:pending:t/served-too" in left
+          and left["connection:held:x"]["aliases"] == ["t2"])
+    for k in ("model:paid:t/served", "model:pending:t/served-too", "connection:held:x"):
+        alerts.clear(k)
+    print(("ok   " if ok else "FAIL ") + "an answer clears what it disproves, not a price" + ("" if ok else f" -> {cleared}"))
+    failures += 0 if ok else 1
+
     # Got it clears a notice (NVIDIA changed its figures), never a state still wrong.
     alerts.raise_alert("test:notice", "limits", "a notice", dismiss=True)
     alerts.raise_alert("test:state", "limits", "a state")