@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.0

package/scripts/test_connect.py

@@ -5,9 +5,11 @@
 metrics/connect.py --selftest covers the flows themselves; this covers what
 serve.py does with them: the callback route on the real handler, saving the
 endpoint by save_endpoint's rules (an owner's headers kept), the refresh token
-beside the key, a device sign-in finished by polling, Ollama's local add, and
-models on a renewing endpoint routed to the built-in client. Endpoints, keys,
-the change log and connect.json all point at a temp folder.
+beside the key, a device sign-in finished by polling, Ollama's local add,
+models on a renewing endpoint routed to the built-in client, and model servers
+found on this machine and what they say of their models for the catalog
+(local_servers). Endpoints, keys, the change log and connect.json all point at
+a temp folder.
 """
 
 from __future__ import annotations
@@ -70,6 +72,151 @@ def serve_on(handler) -> ThreadingHTTPServer:
     return server
 
 
+SHAPES: dict[int, dict] = {}                     # port -> path -> (status, answer): what the server there says
+
+
+class Server(BaseHTTPRequestHandler):
+    """A local model server, answering as SHAPES says the one on its port does."""
+    def do_GET(self):                            # noqa: N802
+        status, body = SHAPES[self.server.server_port].get(self.path, (404, {"error": "not found"}))
+        raw = body.encode() if isinstance(body, str) else json.dumps(body).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", "text/html" if isinstance(body, str) else "application/json")
+        self.send_header("Content-Length", str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+
+    def do_POST(self):                           # noqa: N802 -- Ollama's /api/show: one answer whatever is asked
+        self.rfile.read(int(self.headers.get("Content-Length") or 0))
+        self.do_GET()
+
+    def log_message(self, *_a):
+        pass
+
+
+def local_servers() -> None:
+    """Found on this machine (connect.detect): each server told by its answers,
+    llama.cpp and LocalAI apart on one port, impostors and a hung port ignored,
+    this install's ports and servers already added skipped; then on any port,
+    by the process listening there. Fake servers on free ports stand in for the
+    real ones, a table of their own for this machine's processes (none at
+    first: the usual ports alone); endpoints.json is main()'s temp one."""
+    import os
+    import socket
+    import time
+
+    def listing(*owners):
+        return 200, {"object": "list", "data": [{"id": f"m{i}", "object": "model", **({"owned_by": o} if o else {})}
+                                                for i, o in enumerate(owners)]}
+    refused, version = (401, {"error": "Missing API key"}), (200, {"version": "1.0"})
+    shapes = {
+        "ollama": {"/v1/models": listing("library"), "/api/version": version, "/api/show": (200, {
+            "capabilities": ["completion", "tools"], "model_info": {
+                "general.architecture": "qwen35", "qwen35.context_length": 262144,
+                "qwen35.rope.scaling.original_context_length": 16384}})},
+        "lmstudio": {"/v1/models": listing(None, None, None), "/api/v1/models": (200, {"models": [
+            {"key": "pub/m0", "max_context_length": 32768, "variants": ["pub/m0@q4_k_m"],
+             "capabilities": {"vision": True, "trained_for_tool_use": False},
+             "loaded_instances": [{"id": "pub/m0:2", "config": {"context_length": 4096}}]},
+            {"key": "m1", "max_context_length": 8192, "loaded_instances": [],
+             "capabilities": {"vision": False, "trained_for_tool_use": True, "reasoning": {"default": "on"}}}]})},
+        "jan": {"/v1/models": refused},
+        "llamacpp": {"/v1/models": listing("llamacpp")},
+        "localai": {"/v1/models": listing(None, None), "/version": version},
+        "vllm": {"/v1/models": listing("vllm"), "/version": version},
+        "litellm": {"/v1/models": refused, "/health/liveliness": (200, "I'm alive!")},
+        "cliproxy": {"/v1/models": refused, "/": (200, {"message": "CLI Proxy API Server", "endpoints": []})},
+        "omniroute": {"/v1/models": listing("a", "b")},
+    }
+    servers = {sid: serve_on(Server) for sid in shapes if sid != "localai"}
+    port = {sid: s.server_port for sid, s in servers.items()}
+    port["localai"] = shared = port["llamacpp"]  # as 8080 is both
+    for sid, shape in shapes.items():
+        if sid != "localai":
+            SHAPES[port[sid]] = shape
+    hung = socket.socket()                       # takes the connection, never answers
+    hung.bind(("127.0.0.1", 0))
+    hung.listen()
+    saved = (dict(connect.LOCAL), connect.listeners, os.environ.pop("OLLAMA_HOST", None))
+    connect.listeners = lambda: []
+    try:
+        for sid in shapes:
+            connect.LOCAL[sid] = {**saved[0][sid], "ports": (port[sid],)}
+        connect.LOCAL["hung"] = {"label": "Hung", "ports": (hung.getsockname()[1],), "kind": "local", "docs": "https://x"}
+
+        def found(**kw) -> dict:
+            return {f["id"]: f for f in connect.detect(**kw)}
+
+        start = time.time()
+        got = found()
+        # 0.5 s per probe: a hung port that held it up would take far longer; CI runners are slow.
+        assert time.time() - start < 5, "a hung port held detection up"
+        assert set(got) == set(shapes) - {"localai"}, got           # 8080 answering as llama.cpp
+        assert got["llamacpp"]["port"] == shared and got["lmstudio"]["models"] == 3, got
+        for sid in ("jan", "litellm", "cliproxy"):
+            assert got[sid]["key"] and got[sid]["models"] is None, got[sid]
+        assert (got["litellm"]["kind"], got["litellm"]["hub"]) == ("hub", "litellm")
+        assert (got["cliproxy"]["kind"], got["cliproxy"]["hub"]) == ("hub", "cliproxy")
+        assert got["omniroute"]["kind"] == "aggregator" and got["vllm"]["kind"] == "local"
+        assert got["vllm"]["base"] == f"http://127.0.0.1:{port['vllm']}/v1"
+        assert got["ollama"]["provider"] == "ollama" and got["ollama"]["base"] == connect.PROVIDERS["ollama"]["base"]
+        SHAPES[shared] = shapes["localai"]                          # the same port, answering as LocalAI
+        assert "localai" in found() and "llamacpp" not in found()
+        # Something else on a server's port: a web page, an API without the server's sign.
+        SHAPES[shared] = {"/v1/models": (200, "<html>a web app</html>")}
+        SHAPES[port["vllm"]] = {"/v1/models": listing(None)}
+        assert not {"localai", "llamacpp", "vllm"} & set(found())
+        # This install's own port, and servers already endpoints (by localhost, or by this machine's name).
+        assert "omniroute" not in found(skip={port["omniroute"]})
+        connect.locked.update(connect.endpoints.FILE, lambda d: d.update(
+            mine={"base": f"http://localhost:{port['lmstudio']}/v1", "key": "mine"},
+            lan={"base": f"http://{socket.gethostname()}:{port['jan']}/v1", "key": "lan"}))
+        listed = {f["id"]: f for f in serve.connect_local()["found"]}
+        assert not {"lmstudio", "jan"} & set(listed) and "ollama" in listed, listed
+        assert listed["ollama"]["name"] == "ollama" and listed["litellm"]["name"] == "litellm", listed
+        # What each says of its own models, for the catalog: LM Studio's listing by every id a
+        # request can name, Ollama's /api/show a model at a time, and nothing from anything else.
+        import catalog
+        connect.locked.update(connect.endpoints.FILE, lambda d: d.update(
+            olla={"base": f"http://127.0.0.1:{port['ollama']}/v1", "key": "olla"},
+            vl={"base": f"http://127.0.0.1:{port['vllm']}/v1/", "key": "vl"}))
+        said, facts = catalog._native("mine", ["ignored"])
+        seen = {"context_window": 4096, "vision": True, "tools": False, "reasoning": False}
+        assert said == "lmstudio /api/v1/models" and facts["pub/m0"] == facts["pub/m0@q4_k_m"] == \
+            facts["pub/m0:2"] == seen, facts
+        assert facts["m1"] == {"context_window": 8192, "vision": False, "tools": True, "reasoning": True}, facts
+        assert catalog._native("olla", ["q:2b"]) == ("ollama /api/show", {"q:2b": {
+            "context_window": 262144, "vision": False, "tools": True, "reasoning": False}})
+        assert catalog._native("vl", ["m0"]) == ("", {})
+        # On any port, by the process listening there: LM Studio by its name, vLLM by its command
+        # line under Python, Ollama by OLLAMA_HOST (added at its real port, so not by its own
+        # flow, which knows only 11434); a port whose process is none of them is never asked.
+        extra = {sid: serve_on(Server) for sid in ("lmstudio", "vllm", "ollama", "other")}
+        servers.update({sid + "-moved": s for sid, s in extra.items()})
+        at = {sid: s.server_port for sid, s in extra.items()}
+        for sid in at:
+            SHAPES[at[sid]] = shapes["vllm" if sid == "other" else sid]     # "other" answers as vLLM too
+        connect.listeners = lambda: [(1, at["lmstudio"], "LM Studio.exe", ""),
+                                     (2, at["vllm"], "python3", "/opt/venv/bin/python3 /opt/venv/bin/vllm serve m"),
+                                     (3, at["other"], "notepad.exe", "notepad.exe a.txt")]
+        os.environ["OLLAMA_HOST"] = f"0.0.0.0:{at['ollama']}"
+        moved = {(f["id"], f["port"]): f for f in connect.detect()}
+        assert {("lmstudio", at["lmstudio"]), ("vllm", at["vllm"]), ("ollama", at["ollama"])} <= set(moved), moved
+        assert at["other"] not in {p for _, p in moved}, moved
+        ollama = moved[("ollama", at["ollama"])]
+        assert not ollama["provider"] and ollama["base"] == f"http://127.0.0.1:{at['ollama']}/v1", ollama
+    finally:
+        connect.LOCAL.clear()
+        connect.LOCAL.update(saved[0])
+        connect.listeners = saved[1]
+        os.environ.pop("OLLAMA_HOST", None)
+        if saved[2] is not None:
+            os.environ["OLLAMA_HOST"] = saved[2]
+        hung.close()
+        for s in servers.values():
+            s.shutdown()
+
+
 def main() -> int:
     provider, dash = serve_on(Provider), serve_on(serve.Handler)
     fake = f"http://127.0.0.1:{provider.server_port}"
@@ -207,6 +354,8 @@ def main() -> int:
             assert set(json.loads(serve.DISMISSED_PATH.read_text(encoding="utf-8"))) == {"m2@hub", "m3"}
             assert not serve.remove_endpoint("fk-2")["ok"] and not serve.set_default("nope")["ok"]
             assert sum(1 for e in serve.endpoints.load().values() if e.get("default")) == 1
+
+            local_servers()
         finally:
             connect.PROVIDERS.clear()
             connect.PROVIDERS.update(saved[0])

package/scripts/test_first_run.py

@@ -94,9 +94,11 @@ def main() -> int:
         assert "cannot be read" in get("/"), "a broken endpoints file read as none"
         broken.unlink()
 
-        # 1. No endpoint: the page renders and leads into Add; the APIs answer.
+        # 1. No endpoint: the page renders and leads into Add (on the Endpoints
+        # page, which bench.js fetches when the step opens it); the APIs answer.
         page = get("/")
-        assert "id='first-run' data-step='endpoint'" in page and "id='e-add'" in page, "no first-run step"
+        assert "id='first-run' data-step='endpoint'" in page and "id='e-add'" in get("/api/pane?tab=endpoints")["html"], \
+            "no first-run step"
         assert get("/api/endpoints")["endpoints"] == []
         assert get("/api/connect")["providers"], "no providers to sign in with"
         cand = get("/api/candidates")

package/scripts/test_formats.py

@@ -565,6 +565,45 @@ def gateway_routes() -> None:
         assert n == {"totalTokens": F.estimate("x" * 400)}, n
         assert not seats.asked
 
+        # NO COLLECTION NAMED: a decision model sure enough picks one (not ad hoc, one the key may
+        # call), and the reply says which and why; unsure, or switched off, it is refused as before.
+        real_pick, asked_pick = gateway.decide.pick, []
+
+        def pick(name, state, options, use, answer=None):
+            asked_pick.append((name, state, set(options), use))
+            return answer or {"choice": "other", "confidence": 0.9}, "dm", ""
+        gateway.decide.pick = pick
+        unnamed = {"messages": [{"role": "system", "content": "Be brief."}, {"role": "user", "content": "pick for me"}]}
+        try:
+            text, h = ok("/v1/chat/completions", unnamed)
+            r = json.loads(text)
+            assert r["model"] == "bench/other" and r["bench"]["picked"]["collection"] == "bench/other", r
+            assert h["X-Bench-Picked"] == "bench/other; dm, 0.90", h
+            assert "Collection bench/other chosen for this request: no collection was named, so dm chose it" \
+                in r["choices"][0]["message"]["content"], r
+            name, state, options, use = asked_pick[-1]
+            assert (name, state, use) == ("collection", "Be brief.\n\npick for me", "pick_collection"), asked_pick
+            assert "other" in options and "core" in options and "dear" not in options, options
+            logged = json.loads((TMP / "runs" / "gateway" / "calls.jsonl").read_text(encoding="utf-8").splitlines()[-1])
+            assert logged["picked"]["by"] == "dm", logged                      # the benchmark never learns from it
+            status, h, text, _ = call("/v1/chat/completions", {**unnamed, "stream": True})
+            assert status == 200 and h["X-Bench-Picked"].startswith("bench/other;") and "picked" in text, (h, text)
+            text, h = ok("/v1/messages", {"max_tokens": 8, "messages": [{"role": "user", "content": "pick for me"}]})
+            assert json.loads(text)["model"] == "bench/other" and h["X-Bench-Picked"], text
+            gateway.decide.pick = lambda *a: pick(*a, answer={"choice": "other", "confidence": 0.5})
+            status, h, text, _ = call("/v1/chat/completions", unnamed)
+            assert status == 400 and "No collection was named" in text and "X-Bench-Picked" not in h, text
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0, "pick_confidence": 0.4})   # its own cutoff
+            assert call("/v1/chat/completions", unnamed)[0] == 200
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0, "pick_collection": False})
+            asked_pick.clear()
+            assert call("/v1/chat/completions", unnamed)[0] == 400 and not asked_pick   # off: not even asked
+            assert call("/v1/chat/completions", {**unnamed, "model": "bench/nope"})[0] == 404 and not asked_pick
+        finally:
+            gateway.decide.pick = real_pick
+            write("gateway.json", {"max_calls": 3, "max_explorers": 0})
+        seats.asked.clear()
+
         # The dashboard's port answers Ollama's /api/ routes as the gateway's, beside its own.
         text, _ = ok("/api/tags", on=dash.server_port)
         assert "bench/core" in text

package/scripts/test_serve_edit.py

@@ -4,8 +4,9 @@
 
 Each rule the popup enforces is tried once and must refuse; then a role change
 and a rename are applied to the copy and checked. The llm sync is not run:
-serve.ROOT points at the copy, which has no scripts/sync_config.py, so the
-sync reports that it could not run. Alerts and endpoints point at copies too,
+with the built-in client it is skipped, and with the llm CLI serve.ROOT points
+at the copy, which has no scripts/sync_config.py, so the sync reports that it
+could not run. Alerts and endpoints point at copies too,
 because removing a model clears its alerts and a regressed endpoint check
 would write the endpoint file.
 """
@@ -170,6 +171,50 @@ def main() -> int:
     ok = (set(view["served"]) == set(view["models"]) and victim not in view["served"]
           and view["served"].get(other) == d3[other]["served"])
     print(("ok   " if ok else "FAIL ") + "collection view: served name for every member choice"); failures += 0 if ok else 1
+    # Clients: every snippet parses in its format, names each collection, sets the
+    # timeouts a panel needs, and a made key goes only into a shell line or a field.
+    import tomllib
+    try:
+        import yaml                                     # comes with the llm CLI; the built-in client's venv may lack it
+        load_yaml = yaml.safe_load
+    except ImportError:
+        load_yaml = None
+    bad = []
+    for keys in (False, True):
+        setups = {c["id"]: c for c in serve.client_setups("http://127.0.0.1:8122", [("core", 200_000, False),
+                                                                                    ("fast", None, True)], keys)}
+        parsed = {}
+        for c in setups.values():
+            for b in c["blocks"]:
+                parse = {"json": json.loads, "toml": tomllib.loads, "yaml": load_yaml}.get(b["lang"])
+                try:
+                    parsed[c["id"], b["label"]] = parse(b["text"]) if parse else b["text"]
+                except Exception as exc:                # noqa: BLE001
+                    bad.append(f"{c['id']} {b['label']} does not parse: {exc}")
+                if b["key"] and b["lang"] not in ("sh", "bat", "text"):
+                    bad.append(f"{c['id']} {b['label']}: a key would go into a file")
+            alltext = "\n".join(b["text"] for b in c["blocks"])
+            want = ("bench/core:merged", "bench/fast:merged") if c["id"] == "chat" else \
+                ("bench/core:route", "bench/fast:route") if c["id"] != "codex" else ("bench/core:route",)
+            bad += [f"{c['id']} lacks {w}" for w in want if w not in alltext]
+        oc, cx = parsed.get(("opencode", "opencode.json")), parsed.get(("codex", "config.toml"))
+        if oc and cx:
+            p = oc["provider"]["benchwarmers"]
+            if (oc["mcp"]["benchwarmers"]["timeout"] != 900_000 or cx["mcp_servers"]["benchwarmers"]["tool_timeout_sec"] != 900
+                    or cx["model_providers"]["benchwarmers"]["wire_api"] != "responses" or cx["web_search"] != "disabled"
+                    or p["models"]["bench/fast:route"]["limit"]["context"] != serve.UNKNOWN_CONTEXT
+                    or ("apiKey" in p["options"]) != keys or ("env_key" in cx["model_providers"]["benchwarmers"]) != keys):
+                bad.append(f"opencode or Codex settings wrong (keys={keys})")
+        if parsed.get(("cline", "Cline: cline_mcp_settings.json"), {}).get("mcpServers", {}).get("benchwarmers", {}).get("timeout") != 900:
+            bad.append("Cline's MCP timeout is not 900 s")
+        if not all(m["toolCalling"] for m in parsed.get(("vscode", "chatLanguageModels.json"), [{}])[0].get("models", [{}])):
+            bad.append("a VS Code model without toolCalling is hidden from the agent picker")
+        if load_yaml and parsed.get(("goose", "config.yaml"), {}).get("extensions", {}).get("benchwarmers", {}).get("timeout") != 900:
+            bad.append("goose's extension timeout is not 900 s")
+    ok = not bad
+    print(("ok   " if ok else "FAIL ") + "client set-ups: JSON, TOML" + (", YAML" if load_yaml else " (YAML unread: no PyYAML)")
+          + " parse, collections named, timeouts set, keys only in shell lines and fields" + ("" if ok else f" -> {bad}"))
+    failures += 0 if ok else 1
     # restore: same alias back, out of retired.json, role and ad hoc kept, and
     # back into the collections that still exist (one deleted, one changed since)
     serve.delete_collection("doomed")
@@ -255,7 +300,7 @@ def main() -> int:
 
     # Call options: max_tokens set, refused above the model's max output or as
     # a fraction or for no model, then cleared; the llm CLI's copy is stubbed.
-    told, real_info = [], serve.catalog.info
+    told, real_info, llm_option = [], serve.catalog.info, serve._llm_option
     serve._llm_option = lambda alias, name, value: told.append((alias, name, value)) or "stub"
     serve.catalog.info = lambda a: {**real_info(a), "max_output": 50_000}
     chat = next(a for a, m in sorted(serve.model_directory().items()) if m.get("kind") != "decision")
@@ -290,6 +335,7 @@ def main() -> int:
     import client as model_client
     model_client.CHOICE = cfg / "client.json"
     env, real_which = os.environ.pop("BENCH_CLIENT", None), shutil.which
+    ran, real_run = [], serve.subprocess.run
     try:
         shutil.which = lambda name: "/usr/bin/" + name                 # llm installed: still built-in
         first = model_client.chosen()
@@ -305,8 +351,14 @@ def main() -> int:
         no_llm = serve.save_client("llm")["ok"]
         shutil.which = real_which
         unknown = serve.save_client("curl")["ok"]
+        # With the built-in client nothing is mirrored into the llm CLI (5-6 s
+        # a save); choosing the llm CLI syncs it once.
+        serve.subprocess.run = lambda args, **kw: ran.append(args) or serve.subprocess.CompletedProcess(args, 0, "", "")
+        quiet = (serve._sync_cli(), llm_option(chat, "max_tokens", 5), list(ran))
+        shutil.which = lambda name: "/usr/bin/" + name
+        switched, again = serve.save_client("llm"), serve.save_client("llm")
     finally:
-        shutil.which = real_which
+        serve.subprocess.run, shutil.which = real_run, real_which
         os.environ.pop("BENCH_CLIENT", None)
         if env is not None:
             os.environ["BENCH_CLIENT"] = env
@@ -314,6 +366,12 @@ def main() -> int:
           and pinned == (("llm", "environment"), False) and not no_llm and not unknown)
     print(("ok   " if ok else "FAIL ") + "model client choice" + ("" if ok else f" -> {first} {made} {picked} {pinned}"))
     failures += 0 if ok else 1
+    # Saving it again syncs again: the way to retry a failed sync.
+    ok = (quiet == (serve.NOT_SYNCED, serve.NOT_SYNCED, []) and switched["sync"] == "synced to the llm CLI"
+          and again["sync"] == "synced to the llm CLI" and len(ran) == 2
+          and all(str(r[-1]).endswith("sync_config.py") for r in ran))
+    print(("ok   " if ok else "FAIL ") + "llm CLI synced only when it is the client" + ("" if ok else f" -> {quiet} {switched} {ran}"))
+    failures += 0 if ok else 1
 
     # Free only in their own app: only what differs from the built-in is written; bad input refused.
     import sources
@@ -553,6 +611,252 @@ def main() -> int:
                                    if (DATA / "config" / "collections.json").exists() else None))
     print(("ok   " if ok else "FAIL ") + "the real alerts, endpoints and collections were not touched")
     failures += 0 if ok else 1
+
+    # The Logs page reads a log from its end, and still gets whole lines.
+    log = cfg / "big.log"
+    lines = [f"{i} " + "x" * 2000 for i in range(400)]
+    log.write_text("\n".join(lines) + "\n\n", encoding="utf-8")
+    ok = (serve._log_rows(log) == lines[::-1][:200] and serve._log_rows(log, 100) == lines[::-1][:100]
+          and serve._log_rows(log.with_name("none.log")) == [] and list(serve._from_end(log)) == lines[::-1])
+    print(("ok   " if ok else "FAIL ") + "a log's last lines, read from its end")
+    failures += 0 if ok else 1
+
+    # Confidently wrong is scored again when a task or a saved answer changes.
+    answer = cfg / "tasks" / "t" / "output" / "m.md"
+    answer.parent.mkdir(parents=True)
+    (cfg / "tasks" / "t" / "task.json").write_text("{}", encoding="utf-8")
+    answer.write_text("a", encoding="utf-8")
+    before = serve._tasks_key(cfg / "tasks")
+    answer.write_text("ab", encoding="utf-8")
+    ok = len(before) == 2 and serve._tasks_key(cfg / "tasks") != before and serve._tasks_key(cfg / "none") == ()
+    print(("ok   " if ok else "FAIL ") + "a changed answer changes the scoring's key")
+    failures += 0 if ok else 1
+
+    # Suite scores and statuses are worked out once a render; what one pane
+    # changes in its copy, the next does not see.
+    (serve.DATA / "runs" / "suite-2026-01-01.md").write_text("| t1 | `m1` | 1/2 | — | 3.0 |\n", encoding="utf-8")
+    with serve.locked.snapshot():
+        serve.model_status().clear()
+        for v in serve.suite_scores().values():
+            v.clear()
+        ok = (serve.model_status() == serve._model_status() and serve._suite_scores()
+              and serve.suite_scores() == serve._suite_scores())
+    print(("ok   " if ok else "FAIL ") + "suite scores and statuses once a render, a copy each")
+    failures += 0 if ok else 1
+
+    # Trend: a model against itself, task by task. A drop in two runs is a trend;
+    # the same drop in one run, or against one earlier run, is not. A retried row
+    # ("2.0 (x2)") is a score, and a renamed alias goes by its current name.
+    import datetime as dt
+    hist = json.loads(serve.HISTORY_PATH.read_text(encoding="utf-8"))
+    serve.HISTORY_PATH.write_text(json.dumps({**hist, "tr-old": "tr-down"}), encoding="utf-8")
+    aliases.forget()
+    today = dt.date.today()
+
+    def note(ago, rows):
+        day = (today - dt.timedelta(days=ago)).isoformat()
+        with (serve.DATA / "runs" / f"suite-{day}.md").open("a", encoding="utf-8") as fh:
+            fh.write(f"\n## Suite run {day}\n\n| Task | Model | Found | Invented | Seconds |\n| --- | --- | --- | --- | --- |\n"
+                     + "".join(f"| {t} | `{m}` | {f} | — | 2.0 (x2) |\n" for t, m, f in rows))
+    three = ("ta", "tb", "tc")
+    note(50, [(t, m, "5/5") for m in ("tr-old", "tr-once", "tr-thin") for t in three] + [("ta", "tr-twice", "0/1")])
+    note(50, [("ta", "tr-twice", "1/1")])               # the same task again that day: one score, the mean
+    note(40, [(t, m, "5/5") for m in ("tr-old", "tr-once") for t in three] + [("ta", "tr-down", "timeout")])
+    note(10, [("ta", "tr-down", "3/5"), ("tb", "tr-down", "2/5"), ("ta", "tr-thin", "2/5"), ("tb", "tr-thin", "2/5")]
+         + [(t, "tr-once", "5/5") for t in three])
+    note(3, [("tc", "tr-down", "3/5"), ("tc", "tr-thin", "2/5")] + [(t, "tr-once", "1/5") for t in three])
+    note(10, [("ta", "tr-twice", "1/1")])
+    # A note appended in the middle of a render is read from disk, not from that render's copy.
+    with serve.locked.snapshot():
+        serve.note_rows()
+        note(0, [("ta", "tr-late", "1/1")])
+        late = any(r[3] == "tr-late" for r in serve.note_rows())
+    t = serve.trend_rows()["models"]
+    w = {m: t[m]["windows"][serve.TREND_DAYS[0]] for m in ("tr-down", "tr-once", "tr-thin")}
+    twice = t["tr-twice"]
+    late = late and "tr-late" in t and not t["tr-late"]["compared"]
+    # The chart's and the table's order: by flag, the biggest move first; too little by name.
+    flags = {"x": ("higher", 0.1), "y": ("higher", 0.3), "z": ("lower", -0.1), "b": ("too little", 0.1), "a": ("too little", 0.9)}
+    order = sorted(flags, key=lambda m: serve._trend_order(m, dict(zip(("state", "change"), flags[m]))))
+    ok = ({m: s["state"] for m, s in w.items()} == {"tr-down": "lower", "tr-once": "steady", "tr-thin": "too little"}
+          and [[k, round(v, 9)] for k, v in w["tr-down"]["per"]] == [["tb", -0.6], ["ta", -0.4], ["tc", -0.4]]
+          and order == ["z", "y", "x", "a", "b"]
+          and (w["tr-down"]["tasks"], w["tr-down"]["down"], w["tr-down"]["runs"], w["tr-down"]["before"]) == (3, 3, 2, 2)
+          and abs(w["tr-down"]["change"] + 1.4 / 3) < 1e-9 and w["tr-once"]["runsDown"] == 1
+          and "tr-old" not in t and t["tr-down"]["last"] == (today - dt.timedelta(days=3)).isoformat()
+          and ("ta", "tr-down") in serve._suite_scores() and serve.trend_rows() is serve.trend_rows()
+          and late and twice["compared"] and twice["last"] == (today - dt.timedelta(days=10)).isoformat()
+          and abs(twice["windows"][serve.TREND_DAYS[0]]["change"] - 0.5) < 1e-9)   # day 50's two scores: one mean
+    print(("ok   " if ok else "FAIL ") + "trend: a model against itself, two runs to be a trend"
+          + ("" if ok else f" -> {w} {twice} late {late}"))
+    failures += 0 if ok else 1
+
+    # gzip for a caller off this machine that takes it; never over loopback.
+    import gzip
+
+    class Reply:
+        def __init__(self, address, accept):
+            self.headers, self.client_address, self.sent = {"Accept-Encoding": accept}, (address, 1), {}
+
+        def send_header(self, k, v):
+            self.sent[k] = v
+    body = b"<p>a page</p>" * 500
+    far, near, plain = Reply("203.0.113.20", "gzip, deflate, br"), Reply("::ffff:127.0.0.1", "gzip"), Reply("10.0.0.2", "")
+    refused, other = Reply("10.0.0.3", "gzip;q=0, identity"), Reply("10.0.0.4", "x-gzip-not")
+    ok = (gzip.decompress(serve.Handler._length(far, body)) == body and far.sent["Content-Encoding"] == "gzip"
+          and int(far.sent["Content-Length"]) < len(body) and serve.Handler._length(near, body) == body
+          and serve.Handler._length(plain, body) == body and serve.Handler._length(refused, body) == body
+          and serve.Handler._length(other, body) == body
+          and "Content-Encoding" not in {**near.sent, **plain.sent, **refused.sent, **other.sent}
+          and plain.sent["Content-Length"] == str(len(body)))
+    print(("ok   " if ok else "FAIL ") + "gzip off this machine only" + ("" if ok else f" -> {far.sent} {near.sent}"))
+    failures += 0 if ok else 1
+
+    # Uptime counts each call once: a gateway seat from its log, a probe from its
+    # row, the suite's calls from the ledger only after its last failure kept
+    # without text; the ledger's own lines for the gateway and the probe are skipped.
+    import datetime as dt
+    import ledger
+    import time
+    home = serve.DATA                                   # `tmp` names another folder by now
+    ledger.DIR = home / "usage"
+    ledger.DIR.mkdir()
+    now = time.time()
+    at = dt.datetime.fromtimestamp(now).astimezone().isoformat(timespec="seconds")
+    (home / "runs" / "gateway").mkdir(parents=True, exist_ok=True)
+    seat = {"call": "p.member1", "seat": "member", "alias": "m1", "status": "ok", "answer": "yes", "error": "", "seconds": 2}
+    panel = {"at": at, "shape": "design", "seats": [
+        seat, {**seat, "status": "error", "answer": "", "error": "Error: Error code: 503 - high demand"},
+        {**seat, "status": "error", "answer": "", "error": "Error: Error code: 429 - Daily token quota exceeded."}]}
+    calls = home / "runs" / "gateway" / "calls.jsonl"
+    calls.write_text(json.dumps(panel) + "\n", encoding="utf-8")
+    (home / "perf").mkdir(exist_ok=True)
+    (home / "perf" / "history.csv").write_text(
+        "timestamp,model,prompt,n,p50_s,p95_s,min_s,max_s,chars_per_s,answered,refused,error,timeout,failure,detail\n"
+        f"{at[:19]},m1,tiny,1,1.5,1.5,1.5,1.5,0,1,0,0,0,,\n", encoding="utf-8")
+
+    def line(dt_s, caller, status, **more):
+        return json.dumps({"t": now + dt_s, "at": at, "alias": "m1", "connection": "c1", "endpoint": "e",
+                           "served": "m1", "status": status, "attempts": 1, "seconds": 3.0, "caller": caller,
+                           "purpose": "", **more}, separators=(",", ":")) + "\n"
+    ledger._file(now).write_text(
+        line(-30, "serve", "ok") + line(-20, "suite/run", "ok") + line(-10, "suite/run", "error")
+        + line(10, "suite/run", "ok") + line(20, "suite/run", "error", error="Error: Error code: 503 - overloaded"),
+        encoding="utf-8")
+    first = serve._uptime_rows()["models"]["m1"][-1]
+    with calls.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps({**panel, "seats": [seat]}) + "\n")
+    again = serve._uptime_rows()
+    ok = (first[:3] == [3, 2, 1] and again["models"]["m1"][-1][:3] == [4, 2, 1]
+          and again["providers"]["c1"][-1][:3] == [4, 2, 1] and first[3] == {"busy": 2, "our quota": 1})
+    print(("ok   " if ok else "FAIL ") + "uptime counts each call once, the new ones only"
+          + ("" if ok else f" -> {first} {again['models']['m1'][-1]}"))
+    failures += 0 if ok else 1
+
+    # Decision models in Uptime, from their own log; and an error no rule knows, counted as a
+    # decision model said, marked so in the hover and the list (never asked here).
+    real_decide = decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS
+    decide.FILE, decide.LOG = home / "config" / "decision-models.json", home / "metrics" / "decisions.jsonl"
+    decide.DECIDED, decide.RESULTS = home / "metrics" / "decided.json", home / "metrics" / "decide-bench.json"
+    decide.FILE.write_text(json.dumps({"dmA": {"served": "dmA", "endpoint": "e", "via": "e",
+                                               "question_types": ["choice", "noul"]},
+                                       "dmB": {"served": "v/dmB:free", "endpoint": "e", "via": "e",
+                                               "question_types": ["choice"]}}),
+                           encoding="utf-8")
+    decide.LOG.write_text(
+        json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": {"q": {}}, "error": "",
+                    "seconds": 0.4}) + "\n"
+        + json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": None,
+                      "error": "404 No endpoints found for v/dmB:free.", "seconds": 0.1}) + "\n", encoding="utf-8")
+    odd_error = "Error: the gremlins ate it"
+    decide.DECIDED.write_text(json.dumps({"errors": {decide.shape(odd_error): {"class": "busy", "confidence": 0.81,
+                                                                                "by": "dmA"}},
+                                          "roles": {"m1": {"role": "code", "confidence": 0.7, "by": "dmA"}}}),
+                              encoding="utf-8")
+    with calls.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps({**panel, "seats": [{**seat, "status": "error", "answer": "", "error": odd_error}]}) + "\n")
+    u = serve._uptime_rows()
+    marked = "busy (decided by dmA at 0.81)"
+    ok = (u["models"]["dmB"][-1][:4] == [1, 0, 1, {"setup: not found": 1}]
+          and u["models"]["m1"][-1][3].get(marked) == 1 and serve._down(marked)
+          and [o[4:] for o in u["odd"]] == [[serve._shape(odd_error), "busy, decided by dmA at 0.81"]]
+          and any(r["name"] == "dmB" for r in serve.uptime_data()["models"])
+          and "decided by dmA at 0.81" in serve.uptime_card())
+    print(("ok   " if ok else "FAIL ") + "uptime: decision models' calls, and decided shapes marked"
+          + ("" if ok else f" -> {u['models'].get('dmB')} {u['models']['m1'][-1]} {u['odd']}"))
+    failures += 0 if ok else 1
+
+    # The Decision models card: ranked from MIN_ITEMS items, said so below that; the role
+    # suggestion reaches the popup's data, and the collection pick is saved beside the limits.
+    cal = [{"from": 0.8, "n": 8, "said": 0.9, "right": 0.875}]
+    decide.RESULTS.write_text(json.dumps({"updated": at, "models": {
+        "dmA": {"errors": {"asked": 8, "answered": 8, "correct": 7, "accuracy": 0.875, "macro_f1": 0.8, "brier": 0.1,
+                           "ece": 0.03, "calibration": cal, "confident": {"cutoff": 0.6, "coverage": 1.0, "accuracy": 0.875},
+                           "seconds": 0.3, "failures": {}, "last": at}},
+        "dmB": {"errors": {"asked": 1, "answered": 0, "correct": 0, "failures": {"setup": 1}, "seconds": 0.1,
+                           "last": at}}}}), encoding="utf-8")
+    html = serve.decision_card()
+    data = serve.decision_data()
+    ok = ("1 of 1" in html and "below 8 items" in html and "7/8" in html and "88% <span class='muted'>of 8"
+          in html and data["points"] == [{"model": "dmA", "set": "errors", "said": 0.9, "right": 0.875, "n": 8}]
+          and "best of 1 ranked on the errors set: 7 right of 8 asked" in html
+          and json.loads(serve.directory_json())["suggested"]["m1"]["role"] == "code")
+    print(("ok   " if ok else "FAIL ") + "decision card: ranks, below the floor, uses, role suggestion"
+          + ("" if ok else f" -> {data}"))
+    failures += 0 if ok else 1
+    gw = serve.bench_collections.SETTINGS_FILE
+    saved = [serve.save_gateway_settings(5, 2, pick_collection=False, pick_confidence=0.8),
+             serve.save_gateway_settings(5, 2)]                       # not sent: left as they are
+    kept = serve.bench_collections.pick_settings()
+    saved.append(serve.save_gateway_settings(5, 2, pick_collection=True, pick_confidence=None))
+    bad = [serve.save_gateway_settings(5, 2, pick_collection="yes"), serve.save_gateway_settings(5, 2, pick_confidence=0),
+           serve.save_gateway_settings(5, 2, pick_confidence=True)]
+    ok = (all(r["ok"] for r in saved) and not any(r["ok"] for r in bad)
+          and kept == {"pick_collection": False, "pick_confidence": 0.8}
+          and json.loads(gw.read_text(encoding="utf-8")).get("pick_collection") is True
+          and "pick_confidence" not in json.loads(gw.read_text(encoding="utf-8")))
+    print(("ok   " if ok else "FAIL ") + "gateway settings: the collection pick, on, off and its confidence"
+          + ("" if ok else f" -> {saved} {bad} {kept}"))
+    failures += 0 if ok else 1
+    # Export and import: the settings and the endpoints' keys out of this copy, into
+    # an empty install; a preview writes nothing; a name outside the scope is refused whole.
+    import io
+    import zipfile
+    real_llm, there = serve.endpoints.LLM_DIR, serve.DATA
+    serve.endpoints.LLM_DIR = there
+    ep = next(e for e in serve.endpoints.load().values() if e.get("key"))
+    (there / "keys.json").write_text(json.dumps({ep["key"]: "k-1", "unrelated": "k-2"}), encoding="utf-8")
+    (cfg / "x.json.lock").write_text("", encoding="utf-8")
+    plain, with_keys, evil = io.BytesIO(), io.BytesIO(), io.BytesIO()
+    serve.export_zip("settings", False, plain)
+    made = serve.export_zip("settings", True, with_keys)
+    names = zipfile.ZipFile(plain).namelist()
+    carried = json.loads(zipfile.ZipFile(with_keys).read("keys.json"))
+    other = there / "other"
+    other.mkdir()
+    serve.DATA = serve.endpoints.LLM_DIR = other
+    with_keys.seek(0)
+    look = serve.import_zip(with_keys, apply=False)
+    wrote_nothing = not (other / "config").exists()
+    with_keys.seek(0)
+    done = serve.import_zip(with_keys, apply=True)
+    with zipfile.ZipFile(evil, "w") as z:
+        z.writestr("manifest.json", json.dumps({"benchwarmers_export": 1, "scope": "settings"}))
+        z.writestr("config/../../outside.txt", "x")
+    evil.seek(0)
+    refused = serve.import_zip(evil, apply=True)
+    ok = ("manifest.json" in names and "keys.json" not in names and "config/endpoints.json" in names
+          and not any(n.endswith(".lock") for n in names) and carried == {ep["key"]: "k-1"} and made["keys"] == 1
+          and look["ok"] and look["replaces"] == 0 and look["keys"] == 1 and wrote_nothing
+          and done["ok"] and (other / done["backup"]).is_file()
+          and (other / "config" / "endpoints.json").read_bytes() == (cfg / "endpoints.json").read_bytes()
+          and json.loads((other / "keys.json").read_text(encoding="utf-8")) == {ep["key"]: "k-1"}
+          and not refused["ok"] and not (there / "outside.txt").exists() and not (there.parent / "outside.txt").exists())
+    serve.DATA, serve.endpoints.LLM_DIR = there, real_llm
+    print(("ok   " if ok else "FAIL ") + "export and import: settings and keys out, into an empty install, nothing outside"
+          + ("" if ok else f" -> {names[:5]} {carried} {look} {done} {refused}"))
+    failures += 0 if ok else 1
+    decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS = real_decide
     shutil.rmtree(tmp, ignore_errors=True)
     print("\nall rules hold" if not failures else f"\n{failures} FAILED")
     return 1 if failures else 0