@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.1

package/scripts/auto_retire.py

@@ -49,8 +49,10 @@ from __future__ import annotations
 import csv
 import datetime as dt
 import json
+import os
 import pathlib
 import sys
+import threading
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT / "metrics"))
@@ -201,11 +203,23 @@ def _current(alias: str, history: dict[str, str]) -> str:
     return alias
 
 
-def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
-    """(perf rows, last non-perf answer per alias, restore date per alias)."""
-    history = locked.read_json(DATA / "config" / "alias-history.json")     # {} until a rename
+# evidence() runs on every dashboard render, and re-read 13 MB of offloads and
+# 1,900 saved answers each time: 1.2 s on the owner's data. Each part is read
+# again only when its file changes.
+_perf: tuple = (None, [])               # ((alias history, history.csv's stat), its rows)
+_outputs: dict[str, tuple] = {}         # saved answer's path -> ((mtime, size), day it answered or None)
+
+
+def _stat(path: pathlib.Path) -> tuple:
+    try:
+        st = path.stat()
+    except OSError:
+        return (str(path),)
+    return str(path), st.st_mtime_ns, st.st_size
+
+
+def _perf_rows(path: pathlib.Path, history: dict[str, str]) -> list[dict]:
     rows = []
-    path = DATA / "perf" / "history.csv"
     if path.exists():
         with path.open(encoding="utf-8", newline="") as fh:
             for r in csv.DictReader(fh):
@@ -215,28 +229,74 @@ def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
                              "failure": r.get("failure") or "",
                              "detail": r.get("detail") or ""})
     rows.sort(key=lambda r: r["date"])
+    return rows
+
+
+_useful_lock = threading.Lock()
+_useful_seen: dict = {"mark": {}, "history": None, "days": {}}
+
+
+def _useful(path: pathlib.Path, history: dict[str, str]) -> dict[str, dt.date]:
+    """alias -> the last day a useful offload says it answered. Only the lines
+    added since the last call are read (the gateway adds some on every call),
+    all of them again after a rename."""
+    with _useful_lock:
+        s = _useful_seen
+        if s["history"] != history:
+            s.update(mark={}, history=history)
+        fresh, lines = locked.appended(path, s["mark"])
+        if fresh:
+            s["days"] = {}
+        days = s["days"]
+        for line in lines:
+            try:
+                o = json.loads(line)
+            except ValueError:
+                continue
+            if o.get("useful"):
+                alias, day = _current(o["model"], history), dt.date.fromisoformat(o["date"])
+                days[alias] = max(days.get(alias, day), day)
+        return dict(days)
+
+
+def _saved_answers():
+    """(model, day) for each saved answer that was a real answer. The listing
+    brings each file's stat (free on Windows); a file is read again only when
+    it changes."""
+    from run import outcome
+    for folder in DATA.glob("suite/tasks/*/output"):     # the saved answers are the install's data
+        try:
+            listing = list(os.scandir(folder))
+        except OSError:
+            continue
+        for e in listing:
+            if not e.name.endswith(".md"):
+                continue
+            st = e.stat()
+            hit = _outputs.get(e.path)
+            if hit is None or hit[0] != (st.st_mtime_ns, st.st_size):
+                text = pathlib.Path(e.path).read_text(encoding="utf-8", errors="replace")
+                hit = _outputs[e.path] = ((st.st_mtime_ns, st.st_size),
+                                          dt.date.fromtimestamp(st.st_mtime) if outcome(text) is None else None)
+            if hit[1]:
+                yield e.name[:-3], hit[1]
 
-    answers: dict[str, dt.date] = {}
 
-    def saw(alias: str, day: dt.date) -> None:
+def evidence() -> tuple[list[dict], dict[str, dt.date], dict[str, dt.date]]:
+    """(perf rows, last non-perf answer per alias, restore date per alias).
+    The rows are shared between calls: read them, never change them."""
+    global _perf
+    history = locked.read_json(DATA / "config" / "alias-history.json")     # {} until a rename
+    path = DATA / "perf" / "history.csv"
+    if _perf[0] != (key := (history, _stat(path))):
+        _perf = (key, _perf_rows(path, history))
+    rows = _perf[1]
+    offloads = DATA / "metrics" / "offloads.jsonl"
+    answers = _useful(offloads, history)
+    for alias, day in _saved_answers():
         alias = _current(alias, history)
         answers[alias] = max(answers.get(alias, day), day)
 
-    offloads = DATA / "metrics" / "offloads.jsonl"
-    for line in (offloads.read_text(encoding="utf-8").splitlines()
-                 if offloads.exists() else []):
-        try:
-            o = json.loads(line)
-        except ValueError:
-            continue
-        if o.get("useful"):
-            saw(o["model"], dt.date.fromisoformat(o["date"]))
-
-    from run import outcome
-    for out in DATA.glob("suite/tasks/*/output/*.md"):    # the saved answers are the install's data
-        if outcome(out.read_text(encoding="utf-8", errors="replace")) is None:
-            saw(out.stem, dt.date.fromtimestamp(out.stat().st_mtime))
-
     restored: dict[str, dt.date] = {}
     changes = DATA / "config" / "changes.jsonl"
     for line in (changes.read_text(encoding="utf-8").splitlines()
@@ -403,6 +463,21 @@ def _selfcheck() -> None:
     refused = [row("pk", n, "signin") for n in (9, 5, 0)]
     p = plan(d, {"pk": "v/pk"}, {"v/pk"}, {}, refused, {}, {}, {"v/pk": "v"})
     assert not p["retire"] and not p["confirm"] and not p["pending"], p
+    # evidence() reads a file again when it changes, and only then
+    global DATA
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmp:
+        DATA = pathlib.Path(tmp)
+        out = DATA / "suite" / "tasks" / "t" / "output" / "m.md"
+        out.parent.mkdir(parents=True)
+        out.write_text("a real answer, long enough to count as one: " * 2, encoding="utf-8")
+        log = DATA / "metrics" / "offloads.jsonl"
+        log.parent.mkdir()
+        log.write_text('{"model": "o", "date": "2026-10-01", "useful": true}\n', encoding="utf-8")
+        assert set(evidence()[1]) == {"m", "o"}
+        out.write_text("<<timed out>>", encoding="utf-8")
+        log.write_text("", encoding="utf-8")
+        assert evidence()[1] == {}
     print("auto_retire: rules hold")
 
 

package/scripts/onboard.py

@@ -21,7 +21,9 @@ WHAT IT DOES, AND WHY IN THIS ORDER
 3. **Register and sync**, then, with an Open WebUI hub, refresh the provider
    tags so the connection behind it is known — which is what the router uses
    to keep a panel independent. Without one the endpoint is the connection.
-   Then refresh the catalog, so its details (context, tools, vision) show now.
+   Then refresh the catalog, so its details (context, tools, vision) show now,
+   and ask a decision model which role suits it from them: kept as a
+   suggestion with its confidence (metrics/decide.py suggest_role), never set.
 4. **Probe availability** before spending anything: two prompt sizes, two
    samples. A model that cannot answer "ready" will not answer a 46 KB task,
    and finding that out here costs four calls instead of six large ones.
@@ -185,6 +187,17 @@ def makes_images(alias: str, served_name: str) -> bool:
     return "image" in out if out else bool(IMAGE_NAMES.search(served_name))
 
 
+def role_facts(alias: str, served_name: str, connection: str) -> str:
+    """What a decision model is told when asked for a new chat model's role:
+    its name, connection and catalog facts, the description last."""
+    import catalog
+    rec = catalog.info(alias)
+    facts = [f"{k.replace('_', ' ')}: {', '.join(map(str, v)) if isinstance(v, list) else v}"
+             for k in catalog.FIELDS if k != "description" and (v := rec.get(k)) not in (None, "", [])]
+    return "\n".join([f"model: {served_name}", f"connection: {connection}", *facts,
+                      f"description: {rec.get('description') or 'none known'}"])
+
+
 def run(*args: str, timeout: int = 3600) -> subprocess.CompletedProcess:
     return subprocess.run([str(PY), *args], cwd=str(ROOT), timeout=timeout,
                           capture_output=True, text=True, encoding="utf-8",
@@ -273,7 +286,13 @@ def main() -> None:
             return
         # The passthrough hints are about Open WebUI; anywhere else a 404 is the provider's.
         owui = endpoints.hub(endpoint) == "openwebui"
-        hint = ("  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
+        # ponytail: Cloudflare told apart by its @cf/ ids; its partner models (no @cf/) get the generic hint.
+        hint = ("  Cloudflare answers decisions only on its /ai/run route, which no hub relays: add"
+                " Cloudflare as a direct endpoint (decisions .../accounts/<id>/ai/run/{model},"
+                " models_url .../ai/models/search?per_page=1000) and onboard it from there.\n"
+                if "@cf/" in name and endpoint != "cloudflare" and "{model}" not in
+                (endpoints.load()[endpoint].get("decisions") or "") else
+                "  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
                 " in its environment.\n" if owui and "passthrough" in why.lower() else
                 "  Its connection has no /decisions under its base. An OpenRouter-hosted decision"
                 " model cannot go through Open WebUI (its Decisions API is outside /api/v1);"
@@ -305,7 +324,13 @@ def main() -> None:
                       lambda roles: roles.setdefault("models", {}).__setitem__(alias, "vision"),
                       ensure_ascii=False)
         print("        role vision: a seat only for requests that carry images")
-    run("scripts/sync_config.py", timeout=300)
+    # The llm CLI's copy of the registry, only while it is the client: the
+    # built-in client reads the data folder's, and choosing the llm CLI on
+    # Settings syncs it whole (serve.save_client).
+    import client as model_client
+    synced = model_client.chosen()[0] == "llm"
+    if synced:
+        run("scripts/sync_config.py", timeout=300)
     # Open WebUI's tags name the connection behind the hub; without a hub the
     # endpoint is the connection, and route.connection_of needs no tags.
     if endpoints.find_hub("openwebui"):
@@ -319,7 +344,8 @@ def main() -> None:
     catalog = run("metrics/catalog.py", timeout=300)
     details = ("details from the catalog" if catalog.returncode == 0
                else f"catalog refresh failed (exit {catalog.returncode}); details at its next run")
-    print(f"  [3/5] synced; connection is {connection}; {details}")
+    print(f"  [3/5] {'synced to the llm CLI' if synced else 'the llm CLI not synced (the built-in client is in use)'};"
+          f" connection is {connection}; {details}")
 
     # An image generator: role image, which no chat shape admits, so it never
     # fills a seat or a collection; a chat probe and the suite would only
@@ -341,6 +367,16 @@ def main() -> None:
                      " never a seat, not probed, no suite.\n")
         return
 
+    # A decision model's suggestion for its role, from what the catalog knows:
+    # kept for the model popup (metrics/decided.json), never set from here.
+    try:
+        hint = decide.suggest_role(alias, role_facts(alias, name, connection))
+        print(f"        role suggested by {hint['by']}: {hint['role']} at {hint['confidence']:.2f}"
+              " (a suggestion; the role is yours to set in its model popup)" if "role" in hint
+              else f"        no role suggestion: {hint['error']}")
+    except Exception as exc:                       # noqa: BLE001 -- a suggestion never stops an onboarding
+        print(f"        no role suggestion: {type(exc).__name__}: {exc}")
+
     # 4. availability
     print("  [4/5] probing availability (4 calls)...")
     since = dt.datetime.now().isoformat(timespec="seconds")
@@ -414,6 +450,17 @@ def _selftest() -> None:
     # The name fallback, for a model the catalog says nothing about.
     assert all(IMAGE_NAMES.search(s) for s in ("gpt-image-2.5-flare", "openai/dall-e-3", "black-forest-labs/flux.2-pro"))
     assert not any(IMAGE_NAMES.search(s) for s in ("gpt-5.2", "reflux-7b", "gemini-3-pro"))
+    # What the role suggestion is told: the catalog's facts, the description last.
+    import catalog
+    real = catalog.info
+    catalog.info = lambda a: {"context_window": 131072, "vision": False, "input_modalities": ["text"],
+                              "tools": None, "description": "A coding model."}
+    try:
+        facts = role_facts("m1", "vendor/m1", "groq").splitlines()
+    finally:
+        catalog.info = real
+    assert facts == ["model: vendor/m1", "connection: groq", "context window: 131072", "vision: False",
+                     "input modalities: text", "description: A coding model."], facts
     print("ok")
 
 

package/scripts/test_connect.py

@@ -5,9 +5,11 @@
 metrics/connect.py --selftest covers the flows themselves; this covers what
 serve.py does with them: the callback route on the real handler, saving the
 endpoint by save_endpoint's rules (an owner's headers kept), the refresh token
-beside the key, a device sign-in finished by polling, Ollama's local add, and
-models on a renewing endpoint routed to the built-in client. Endpoints, keys,
-the change log and connect.json all point at a temp folder.
+beside the key, a device sign-in finished by polling, Ollama's local add,
+models on a renewing endpoint routed to the built-in client, and model servers
+found on this machine and what they say of their models for the catalog
+(local_servers). Endpoints, keys, the change log and connect.json all point at
+a temp folder.
 """
 
 from __future__ import annotations
@@ -70,6 +72,151 @@ def serve_on(handler) -> ThreadingHTTPServer:
     return server
 
 
+SHAPES: dict[int, dict] = {}                     # port -> path -> (status, answer): what the server there says
+
+
+class Server(BaseHTTPRequestHandler):
+    """A local model server, answering as SHAPES says the one on its port does."""
+    def do_GET(self):                            # noqa: N802
+        status, body = SHAPES[self.server.server_port].get(self.path, (404, {"error": "not found"}))
+        raw = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", "text/html" if isinstance(body, str) else "application/json")
+        self.send_header("Content-Length", str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+
+    def do_POST(self):                           # noqa: N802 -- Ollama's /api/show: one answer whatever is asked
+        self.rfile.read(int(self.headers.get("Content-Length") or 0))
+        self.do_GET()
+
+    def log_message(self, *_a):
+        pass
+
+
+def local_servers() -> None:
+    """Found on this machine (connect.detect): each server told by its answers,
+    llama.cpp and LocalAI apart on one port, impostors and a hung port ignored,
+    this install's ports and servers already added skipped; then on any port,
+    by the process listening there. Fake servers on free ports stand in for the
+    real ones, a table of their own for this machine's processes (none at
+    first: the usual ports alone); endpoints.json is main()'s temp one."""
+    import os
+    import socket
+    import time
+
+    def listing(*owners):
+        return 200, {"object": "list", "data": [{"id": f"m{i}", "object": "model", **({"owned_by": o} if o else {})}
+                                                for i, o in enumerate(owners)]}
+    refused, version = (401, {"error": "Missing API key"}), (200, {"version": "1.0"})
+    shapes = {
+        "ollama": {"/v1/models": listing("library"), "/api/version": version, "/api/show": (200, {
+            "capabilities": ["completion", "tools"], "model_info": {
+                "general.architecture": "qwen35", "qwen35.context_length": 262144,
+                "qwen35.rope.scaling.original_context_length": 16384}})},
+        "lmstudio": {"/v1/models": listing(None, None, None), "/api/v1/models": (200, {"models": [
+            {"key": "pub/m0", "max_context_length": 32768, "variants": ["pub/m0@q4_k_m"],
+             "capabilities": {"vision": True, "trained_for_tool_use": False},
+             "loaded_instances": [{"id": "pub/m0:2", "config": {"context_length": 4096}}]},
+            {"key": "m1", "max_context_length": 8192, "loaded_instances": [],
+             "capabilities": {"vision": False, "trained_for_tool_use": True, "reasoning": {"default": "on"}}}]})},
+        "jan": {"/v1/models": refused},
+        "llamacpp": {"/v1/models": listing("llamacpp")},
+        "localai": {"/v1/models": listing(None, None), "/version": version},
+        "vllm": {"/v1/models": listing("vllm"), "/version": version},
+        "litellm": {"/v1/models": refused, "/health/liveliness": (200, "I'm alive!")},
+        "cliproxy": {"/v1/models": refused, "/": (200, {"message": "CLI Proxy API Server", "endpoints": []})},
+        "omniroute": {"/v1/models": listing("a", "b")},
+    }
+    servers = {sid: serve_on(Server) for sid in shapes if sid != "localai"}
+    port = {sid: s.server_port for sid, s in servers.items()}
+    port["localai"] = shared = port["llamacpp"]  # as 8080 is both
+    for sid, shape in shapes.items():
+        if sid != "localai":
+            SHAPES[port[sid]] = shape
+    hung = socket.socket()                       # takes the connection, never answers
+    hung.bind(("127.0.0.1", 0))
+    hung.listen()
+    saved = (dict(connect.LOCAL), connect.listeners, os.environ.pop("OLLAMA_HOST", None))
+    connect.listeners = lambda: []
+    try:
+        for sid in shapes:
+            connect.LOCAL[sid] = {**saved[0][sid], "ports": (port[sid],)}
+        connect.LOCAL["hung"] = {"label": "Hung", "ports": (hung.getsockname()[1],), "kind": "local", "docs": "https://x"}
+
+        def found(**kw) -> dict:
+            return {f["id"]: f for f in connect.detect(**kw)}
+
+        start = time.time()
+        got = found()
+        # 0.5 s per probe: a hung port that held it up would take far longer; CI runners are slow.
+        assert time.time() - start < 5, "a hung port held detection up"
+        assert set(got) == set(shapes) - {"localai"}, got           # 8080 answering as llama.cpp
+        assert got["llamacpp"]["port"] == shared and got["lmstudio"]["models"] == 3, got
+        for sid in ("jan", "litellm", "cliproxy"):
+            assert got[sid]["key"] and got[sid]["models"] is None, got[sid]
+        assert (got["litellm"]["kind"], got["litellm"]["hub"]) == ("hub", "litellm")
+        assert (got["cliproxy"]["kind"], got["cliproxy"]["hub"]) == ("hub", "cliproxy")
+        assert got["omniroute"]["kind"] == "aggregator" and got["vllm"]["kind"] == "local"
+        assert got["vllm"]["base"] == f"http://127.0.0.1:{port['vllm']}/v1"
+        assert got["ollama"]["provider"] == "ollama" and got["ollama"]["base"] == connect.PROVIDERS["ollama"]["base"]
+        SHAPES[shared] = shapes["localai"]                          # the same port, answering as LocalAI
+        assert "localai" in found() and "llamacpp" not in found()
+        # Something else on a server's port: a web page, an API without the server's sign.
+        SHAPES[shared] = {"/v1/models": (200, "<html>a web app</html>")}
+        SHAPES[port["vllm"]] = {"/v1/models": listing(None)}
+        assert not {"localai", "llamacpp", "vllm"} & set(found())
+        # This install's own port, and servers already endpoints (by localhost, or by this machine's name).
+        assert "omniroute" not in found(skip={port["omniroute"]})
+        connect.locked.update(connect.endpoints.FILE, lambda d: d.update(
+            mine={"base": f"http://localhost:{port['lmstudio']}/v1", "key": "mine"},
+            lan={"base": f"http://{socket.gethostname()}:{port['jan']}/v1", "key": "lan"}))
+        listed = {f["id"]: f for f in serve.connect_local()["found"]}
+        assert not {"lmstudio", "jan"} & set(listed) and "ollama" in listed, listed
+        assert listed["ollama"]["name"] == "ollama" and listed["litellm"]["name"] == "litellm", listed
+        # What each says of its own models, for the catalog: LM Studio's listing by every id a
+        # request can name, Ollama's /api/show a model at a time, and nothing from anything else.
+        import catalog
+        connect.locked.update(connect.endpoints.FILE, lambda d: d.update(
+            olla={"base": f"http://127.0.0.1:{port['ollama']}/v1", "key": "olla"},
+            vl={"base": f"http://127.0.0.1:{port['vllm']}/v1/", "key": "vl"}))
+        said, facts = catalog._native("mine", ["ignored"])
+        seen = {"context_window": 4096, "vision": True, "tools": False, "reasoning": False}
+        assert said == "lmstudio /api/v1/models" and facts["pub/m0"] == facts["pub/m0@q4_k_m"] == \
+            facts["pub/m0:2"] == seen, facts
+        assert facts["m1"] == {"context_window": 8192, "vision": False, "tools": True, "reasoning": True}, facts
+        assert catalog._native("olla", ["q:2b"]) == ("ollama /api/show", {"q:2b": {
+            "context_window": 262144, "vision": False, "tools": True, "reasoning": False}})
+        assert catalog._native("vl", ["m0"]) == ("", {})
+        # On any port, by the process listening there: LM Studio by its name, vLLM by its command
+        # line under Python, Ollama by OLLAMA_HOST (added at its real port, so not by its own
+        # flow, which knows only 11434); a port whose process is none of them is never asked.
+        extra = {sid: serve_on(Server) for sid in ("lmstudio", "vllm", "ollama", "other")}
+        servers.update({sid + "-moved": s for sid, s in extra.items()})
+        at = {sid: s.server_port for sid, s in extra.items()}
+        for sid in at:
+            SHAPES[at[sid]] = shapes["vllm" if sid == "other" else sid]     # "other" answers as vLLM too
+        connect.listeners = lambda: [(1, at["lmstudio"], "LM Studio.exe", ""),
+                                     (2, at["vllm"], "python3", "/opt/venv/bin/python3 /opt/venv/bin/vllm serve m"),
+                                     (3, at["other"], "notepad.exe", "notepad.exe a.txt")]
+        os.environ["OLLAMA_HOST"] = f"0.0.0.0:{at['ollama']}"
+        moved = {(f["id"], f["port"]): f for f in connect.detect()}
+        assert {("lmstudio", at["lmstudio"]), ("vllm", at["vllm"]), ("ollama", at["ollama"])} <= set(moved), moved
+        assert at["other"] not in {p for _, p in moved}, moved
+        ollama = moved[("ollama", at["ollama"])]
+        assert not ollama["provider"] and ollama["base"] == f"http://127.0.0.1:{at['ollama']}/v1", ollama
+    finally:
+        connect.LOCAL.clear()
+        connect.LOCAL.update(saved[0])
+        connect.listeners = saved[1]
+        os.environ.pop("OLLAMA_HOST", None)
+        if saved[2] is not None:
+            os.environ["OLLAMA_HOST"] = saved[2]
+        hung.close()
+        for s in servers.values():
+            s.shutdown()
+
+
 def main() -> int:
     provider, dash = serve_on(Provider), serve_on(serve.Handler)
     fake = f"http://127.0.0.1:{provider.server_port}"
@@ -207,6 +354,8 @@ def main() -> int:
             assert set(json.loads(serve.DISMISSED_PATH.read_text(encoding="utf-8"))) == {"m2@hub", "m3"}
             assert not serve.remove_endpoint("fk-2")["ok"] and not serve.set_default("nope")["ok"]
             assert sum(1 for e in serve.endpoints.load().values() if e.get("default")) == 1
+
+            local_servers()
         finally:
             connect.PROVIDERS.clear()
             connect.PROVIDERS.update(saved[0])

package/scripts/test_first_run.py

@@ -94,9 +94,11 @@ def main() -> int:
         assert "cannot be read" in get("/"), "a broken endpoints file read as none"
         broken.unlink()
 
-        # 1. No endpoint: the page renders and leads into Add; the APIs answer.
+        # 1. No endpoint: the page renders and leads into Add (on the Endpoints
+        # page, which bench.js fetches when the step opens it); the APIs answer.
         page = get("/")
-        assert "id='first-run' data-step='endpoint'" in page and "id='e-add'" in page, "no first-run step"
+        assert "id='first-run' data-step='endpoint'" in page and "id='e-add'" in get("/api/pane?tab=endpoints")["html"], \
+            "no first-run step"
         assert get("/api/endpoints")["endpoints"] == []
         assert get("/api/connect")["providers"], "no providers to sign in with"
         cand = get("/api/candidates")

package/scripts/test_formats.py

@@ -565,6 +565,45 @@ def gateway_routes() -> None:
         assert n == {"totalTokens": F.estimate("x" * 400)}, n
         assert not seats.asked
 
+        # NO COLLECTION NAMED: a decision model sure enough picks one (not ad hoc, one the key may
+        # call), and the reply says which and why; unsure, or switched off, it is refused as before.
+        real_pick, asked_pick = gateway.decide.pick, []
+
+        def pick(name, state, options, use, answer=None):
+            asked_pick.append((name, state, set(options), use))
+            return answer or {"choice": "other", "confidence": 0.9}, "dm", ""
+        gateway.decide.pick = pick
+        unnamed = {"messages": [{"role": "system", "content": "Be brief."}, {"role": "user", "content": "pick for me"}]}
+        try:
+            text, h = ok("/v1/chat/completions", unnamed)
+            r = json.loads(text)
+            assert r["model"] == "bench/other" and r["bench"]["picked"]["collection"] == "bench/other", r
+            assert h["X-Bench-Picked"] == "bench/other; dm, 0.90", h
+            assert "Collection bench/other chosen for this request: no collection was named, so dm chose it" \
+                in r["choices"][0]["message"]["content"], r
+            name, state, options, use = asked_pick[-1]
+            assert (name, state, use) == ("collection", "Be brief.\n\npick for me", "pick_collection"), asked_pick
+            assert "other" in options and "core" in options and "dear" not in options, options
+            logged = json.loads((TMP / "runs" / "gateway" / "calls.jsonl").read_text(encoding="utf-8").splitlines()[-1])
+            assert logged["picked"]["by"] == "dm", logged                      # the benchmark never learns from it
+            status, h, text, _ = call("/v1/chat/completions", {**unnamed, "stream": True})
+            assert status == 200 and h["X-Bench-Picked"].startswith("bench/other;") and "picked" in text, (h, text)
+            text, h = ok("/v1/messages", {"max_tokens": 8, "messages": [{"role": "user", "content": "pick for me"}]})
+            assert json.loads(text)["model"] == "bench/other" and h["X-Bench-Picked"], text
+            gateway.decide.pick = lambda *a: pick(*a, answer={"choice": "other", "confidence": 0.5})
+            status, h, text, _ = call("/v1/chat/completions", unnamed)
+            assert status == 400 and "No collection was named" in text and "X-Bench-Picked" not in h, text
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0, "pick_confidence": 0.4})   # its own cutoff
+            assert call("/v1/chat/completions", unnamed)[0] == 200
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0, "pick_collection": False})
+            asked_pick.clear()
+            assert call("/v1/chat/completions", unnamed)[0] == 400 and not asked_pick   # off: not even asked
+            assert call("/v1/chat/completions", {**unnamed, "model": "bench/nope"})[0] == 404 and not asked_pick
+        finally:
+            gateway.decide.pick = real_pick
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0})
+        seats.asked.clear()
+
         # The dashboard's port answers Ollama's /api/ routes as the gateway's, beside its own.
         text, _ = ok("/api/tags", on=dash.server_port)
         assert "bench/core" in text

package/scripts/test_quota.py

@@ -20,6 +20,8 @@ not mix. Covered:
   a Gemini 429 body's daily quota learned as a limit;
 - an rpm limit across six threads at once and across two processes (calls in
   flight count), a monthly cap (rpmo), and both refusals classed as capacity;
+- a connection's learned limit (metrics/learn.py) enforced, its refusal not
+  learned from, your own value winning over it, and Forget;
 - the free-tier guard holding a model that went paid, until it is allowed;
 - metrics/cost.py reading the ledger.
 """
@@ -331,6 +333,31 @@ def main() -> int:
     check(not bad["ok"] and serve.save_model_facts("t-ok", {"limits": {"rpmo": 5}}).get("ok") is not None
           and "rpmo" not in ((serve.catalog.overrides().get("t-ok") or {}).get("limits") or {}),
           "a bad rpmo is refused; a model's own limits have none", bad)
+    # A connection's learned limit (learn.py: 429s that named no number) is kept to like one of yours, which wins.
+    import datetime as dt
+    at = dt.datetime.now().astimezone()
+    learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=2))
+    got = learn._learn_connection("monthprov", "rpd", 3, "test", "429", at - dt.timedelta(days=1))
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    h = pacing.held("t-month", "monthprov")
+    check(got == {"rpd": 3} and len(rule) == 1 and rule[0]["scope"] == "connection" and rule[0]["limit"] == 3
+          and rule[0]["source"].startswith("learned ") and h is not None and h[0] > 3600 and "learned" in h[1],
+          "a connection's learned rpd is enforced", (got, rule, h))
+    r = client.chat("t-month", "hi", timeout=5)
+    learn.record("t-month", client.output(r, 5), "test")
+    kept = learn.connection_learned()["monthprov"]["rpd"]
+    check("rate limit (not sent" in r["error"] and "connection-learned.json" in r["error"]
+          and kept["confirmed_at"] == kept["at"] and not learn.connection_learned(pending=True),
+          "its refusal names where the limit is from, and is not learned from", (r["error"], kept))
+    view = {c["name"]: c for c in serve.connections_view()["connections"]}
+    serve.save_connection_limits("monthprov", {"rpm": 20, "rpmo": 1000, "rpd": 100})
+    rule = [x for x in pacing.rules("t-month", "monthprov") if x["axis"] == "rpd"]
+    check(view["monthprov"]["learned"] == {"rpd": {k: kept[k] for k in ("value", "at", "confirmed_at")}}
+          and len(rule) == 1 and rule[0]["limit"] == 100 and rule[0]["source"].startswith("you"),
+          "the Endpoints dialog shows it; your own value for that connection wins", (view["monthprov"], rule))
+    gone = serve.forget_connection_learned("monthprov", "rpd")
+    check(gone["ok"] and learn.connection_learned() == {} and not serve.forget_connection_learned("monthprov", "rpd")["ok"],
+          "Forget drops a learned limit", gone)
     page = serve.quotas_page()
     check("id='quota-data'" in page and "fakeprov" in page and "What the bench sent" in page,
           "the Quotas page carries its tiles and the ledger's counts")