@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.0

package/scripts/test_llm_log.py

@@ -0,0 +1,81 @@
+"""An answer stays an answer when the `llm` CLI fails to write its own log.
+
+    python scripts/test_llm_log.py
+
+2026-10-22, panel p20261022-160401-6ca2: seat inkling-small-free printed
+"pong" and was recorded as an error, because `llm` then raised
+"sqlite3.OperationalError: database is locked" writing logs.db, which the
+other seats' `llm` processes held. A fake `llm` here prints an answer and
+fails that way, through the two places the bench runs `llm -m`: the gateway's
+seat call and suite/run.py's ask_once (the perf probe, the suite, repoqa,
+retries, auto-retire). The same lock before any answer, and a stream cut
+mid-answer, stay errors. BENCHWARMERS_HOME and LLM_USER_PATH name temp
+folders before anything is imported.
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+import tempfile
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+TMP = pathlib.Path(tempfile.mkdtemp(prefix="bench-llmlog-"))
+(TMP / "config").mkdir()
+(TMP / "llm").mkdir()
+os.environ.update(BENCHWARMERS_HOME=str(TMP), LLM_USER_PATH=str(TMP / "llm"), BENCH_CLIENT="llm")
+sys.path[:0] = [str(ROOT), str(ROOT / "metrics"), str(ROOT / "suite")]
+
+FAKE = '''import os, sys
+sys.stdin.read()
+mode = os.environ["FAKE_LLM_MODE"]
+if mode in ("locked", "cut"):
+    sys.stdout.write("pong\\n" if mode == "locked" else "half an ans")
+    sys.stdout.flush()
+if mode == "cut":
+    sys.stderr.write("Error: Connection error.\\n")
+    sys.exit(1)
+if mode != "ok":
+    sys.stderr.write("Traceback (most recent call last):\\n"
+                     "  File \\"llm/models.py\\", line 1, in log_to_db\\n"
+                     "sqlite3.OperationalError: database is locked\\n")
+    sys.exit(1)
+sys.stdout.write("pong\\n")
+'''
+
+
+def fake_llm() -> str:
+    """A path that runs like an `llm` executable on Windows, macOS and Linux."""
+    script = TMP / "fake_llm.py"
+    script.write_text(FAKE, encoding="utf-8")
+    if os.name == "nt":
+        shim = TMP / "llm.cmd"
+        shim.write_text(f'@"{sys.executable}" "{script}" %*\r\n@exit /b %ERRORLEVEL%\r\n', encoding="utf-8")
+    else:
+        shim = TMP / "llm"
+        shim.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8")
+        shim.chmod(0o755)
+    return str(shim)
+
+
+def main() -> int:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    import gateway
+    import run
+    binary = fake_llm()
+    for mode, answered in (("ok", True), ("locked", True), ("locked-early", False), ("cut", False)):
+        os.environ["FAKE_LLM_MODE"] = mode
+        answer, error = gateway._ask_llm([binary, "-m", "m1"], "ping")
+        assert (answer, error == "") == (("pong", True) if answered else (answer, False)), (mode, answer, error)
+        body, _ = run.ask_once(binary, "m1", "ping", 60)
+        kind = (run.outcome(body, min_length=1) or ("answered", ""))[0]
+        assert (kind == "answered") == answered and (body.strip() == "pong") == answered, (mode, kind, body)
+        if mode == "locked-early":
+            assert "database is locked" in error and "database is locked" in body, (error, body)
+    print("ok")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/test_mcp.py

@@ -0,0 +1,178 @@
+"""The MCP server (mcp_server.py) as a client starts it: a subprocess on stdio, against a stub server.
+
+    python scripts/test_mcp.py
+
+initialize and its protocol version negotiation, ping, tools/list, a call of
+each tool (what reaches the server, gateway key included, and what comes
+back), a refusal passed through as a tool error, a ping answered while a
+panel is still running, protocol errors, and the error when no Benchwarmers
+is running. Nothing but protocol messages may reach stdout.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import socket
+import subprocess
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+SEEN: list[tuple[str, str, dict, str]] = []          # (method, path, body, Authorization)
+
+MODELS = {"object": "list", "data": [
+    {"id": "bench/core", "description": "second opinions", "tags": ["general"], "members": ["m1", "m2"],
+     "explorers": 2, "shape": "", "reply": "all", "merger": "",
+     "together": {"context_window": 128000, "capabilities": ["tools"], "connections": ["x"], "limits": {}}},
+    {"id": "bench/core:merged", "description": "second opinions", "reply": "merged"}]}
+SEATS = [{"call": "p1.member1", "seat": "member", "alias": "m1", "status": "ok", "seconds": 1.0,
+          "answer": "answer one", "error": ""},
+         {"call": "p1.explorer", "seat": "explorer", "alias": "x1", "status": "empty", "seconds": 2.0,
+          "answer": "", "error": ""}]
+
+
+class Stub(BaseHTTPRequestHandler):
+    def do_GET(self):                                  # noqa: N802
+        SEEN.append(("GET", self.path, {}, self.headers.get("Authorization", "")))
+        self._send(200, MODELS)
+
+    def do_POST(self):                                 # noqa: N802
+        body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+        SEEN.append(("POST", self.path, body, self.headers.get("Authorization", "")))
+        if self.path == "/bench/verdict":
+            return self._send(200, {"ok": True, "recorded": {"call": body["call"]}})
+        if body["model"] == "bench/nope":
+            return self._send(404, {"error": {"message": "no collection nope; choose one of bench/core"}})
+        time.sleep(1)                                  # a panel takes a while
+        merged = (body.get("bench") or {}).get("reply") == "merged"
+        self._send(200, {"choices": [{"message": {"role": "assistant", "content": "merged answer" if merged
+                                                  else "## member — m1\nanswer one"}}],
+                         "bench": {"panel": "p1", "collection": body["model"], "shape": "general",
+                                   "seats": SEATS, "reply": "merged" if merged else "all"}})
+
+    def _send(self, status, data):
+        raw = json.dumps(data).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+
+    def log_message(self, *_a):
+        pass
+
+
+class Client:
+    def __init__(self, url: str, key: str = ""):
+        env = {**os.environ, "BENCHWARMERS_URL": url, "BENCHWARMERS_KEY": key, "BENCHWARMERS_TIMEOUT": "30"}
+        self.p = subprocess.Popen([sys.executable, str(ROOT / "mcp_server.py")], stdin=subprocess.PIPE,
+                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+        self.next = 0
+
+    def send(self, msg) -> None:
+        self.p.stdin.write((msg if isinstance(msg, str) else json.dumps(msg)).encode() + b"\n")
+        self.p.stdin.flush()
+
+    def read(self) -> dict:
+        line = self.p.stdout.readline()
+        assert line, "the server closed stdout: " + self.p.stderr.read().decode()
+        return json.loads(line)                        # every stdout line must be one JSON-RPC message
+
+    def request(self, method: str, params: dict | None = None) -> dict:
+        self.next += 1
+        self.send({"jsonrpc": "2.0", "id": self.next, "method": method, **({"params": params} if params else {})})
+        reply = self.read()
+        assert reply["id"] == self.next and reply["jsonrpc"] == "2.0", reply
+        return reply
+
+    def tool(self, name: str, **arguments) -> dict:
+        return self.request("tools/call", {"name": name, "arguments": arguments})["result"]
+
+    def close(self) -> None:
+        self.p.stdin.close()
+        self.p.wait(timeout=10)
+
+
+def main() -> int:
+    stub = ThreadingHTTPServer(("127.0.0.1", 0), Stub)
+    threading.Thread(target=stub.serve_forever, daemon=True).start()
+    url = f"http://127.0.0.1:{stub.server_port}"
+    c = Client(url + "/v1", key="bwk_test")            # a pasted /v1 is fine
+
+    init = c.request("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
+                                    "clientInfo": {"name": "test", "version": "0"}})["result"]
+    assert init["protocolVersion"] == "2025-06-18", init
+    assert init["capabilities"] == {"tools": {"listChanged": False}}, init
+    assert init["serverInfo"]["name"] == "benchwarmers" and "verdict" in init["instructions"], init
+    c.send({"jsonrpc": "2.0", "method": "notifications/initialized"})     # no answer to a notification
+    assert c.request("ping")["result"] == {}
+
+    tools = {t["name"]: t for t in c.request("tools/list")["result"]["tools"]}
+    assert set(tools) == {"list_collections", "ask", "verdict"}, tools
+    assert tools["ask"]["inputSchema"]["required"] == ["collection", "prompt"]
+
+    r = c.tool("list_collections")
+    assert not r["isError"], r
+    rows = json.loads(r["content"][0]["text"])
+    assert [x["name"] for x in rows] == ["bench/core"], rows        # the :merged twin is folded in
+    assert rows[0]["context_window"] == 128000 and rows[0]["members"] == ["m1", "m2"], rows
+    assert SEEN[-1][:2] == ("GET", "/v1/models") and SEEN[-1][3] == "Bearer bwk_test", SEEN[-1]
+
+    # A ping sent while the panel runs is answered first.
+    c.send({"jsonrpc": "2.0", "id": "a", "method": "tools/call",
+            "params": {"name": "ask", "arguments": {"collection": "core", "prompt": "hi", "system": "be brief",
+                                                    "with": ["x1"]}}})
+    c.send({"jsonrpc": "2.0", "id": "p", "method": "ping"})
+    first, second = c.read(), c.read()
+    assert (first["id"], second["id"]) == ("p", "a"), (first, second)
+    r = second["result"]
+    assert not r["isError"] and "answer one" in r["content"][0]["text"], r
+    seats = json.loads(r["content"][1]["text"].split("\n", 1)[1])["seats"]
+    assert [s["call"] for s in seats] == ["p1.member1", "p1.explorer"] and "answer" not in seats[0], seats
+    method, path, body, _ = SEEN[-1]
+    assert (method, path) == ("POST", "/v1/chat/completions"), SEEN[-1]
+    assert body["model"] == "bench/core" and body["with"] == ["x1"] and "bench" not in body, body
+    assert body["messages"] == [{"role": "system", "content": "be brief"}, {"role": "user", "content": "hi"}]
+
+    r = c.tool("ask", collection="bench/core", prompt="hi", merged=True)
+    assert SEEN[-1][2]["bench"] == {"reply": "merged"}, SEEN[-1]
+    assert r["content"][0]["text"] == "merged answer" and "answer one" in r["content"][1]["text"], r
+
+    r = c.tool("ask", collection="nope", prompt="hi")              # the server's refusal, passed on
+    assert r["isError"] and "(404)" in r["content"][0]["text"] and "choose one" in r["content"][0]["text"], r
+    r = c.tool("ask", collection="core")
+    assert r["isError"] and "prompt" in r["content"][0]["text"], r
+
+    r = c.tool("verdict", call="p1.member1", useful=True, note="checked", real=["a true claim"], false=["x"])
+    assert not r["isError"] and "1 real, 1 false" in r["content"][0]["text"], r
+    assert SEEN[-1][1:3] == ("/bench/verdict", {"call": "p1.member1", "useful": True, "note": "checked",
+                                                "real": ["a true claim"], "false": ["x"]}), SEEN[-1]
+    assert c.tool("verdict", call="p1.member1")["isError"]          # useful is required
+
+    assert c.request("tools/call", {"name": "nope"})["error"]["code"] == -32602
+    assert c.request("resources/list")["error"]["code"] == -32601
+    c.send("{not json")
+    assert c.read()["error"]["code"] == -32700
+    c.close()
+
+    # An unknown protocol version gets the newest; no server running is a clear tool error.
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        dead = s.getsockname()[1]
+    c = Client(f"http://127.0.0.1:{dead}")
+    init = c.request("initialize", {"protocolVersion": "1999-01-01", "capabilities": {}})["result"]
+    assert init["protocolVersion"] == "2025-11-25", init
+    r = c.tool("list_collections")
+    assert r["isError"] and "not running" in r["content"][0]["text"], r
+    c.close()
+    stub.shutdown()
+    print("mcp server ok: initialize, ping, tools/list, each tool, refusals, a ping mid-panel, not running")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/test_merged.py

@@ -181,6 +181,27 @@ def main() -> int:
         assert serve.save_collection("chat", ["m1"], "", "", [], "merged", "mg")["ok"]
         saved = json.loads((TMP / "config" / "collections.json").read_text(encoding="utf-8"))["chat"]
         assert (saved["reply"], saved["merger"]) == ("merged", "mg"), saved
+
+        # 9. A retired model a collection still names (member, `with`, merger) is
+        # not called: skipped with why, and the merge goes to a merger chosen by verdicts.
+        write("retired.json", {"gone": {"served": "gone", "at": "2026-10-20T05:00:00+02:00", "why": "withdrawn"}})
+        collection(members=["m1", "gone"], merger="gone")
+        ASKED.clear()
+        r, b, content = ask("bench/core:merged", **{"with": ["gone"]})
+        assert [s["alias"] for s in b["seats"]] == ["m1"] and not any(q["model"] == "gone" for q in ASKED), b
+        assert {(k["seat"], k["alias"]) for k in b["skipped"]} == {("member", "gone"), ("merger", "gone")}, b
+        assert all("retired from the bench on 2026-10-20" in k["why"] for k in b["skipped"]), b["skipped"]
+        assert b["reply"] == "merged" and b["merger"]["alias"] != "gone", b
+        # Asked for every answer, the reply says who was not asked; `with` a retired model is skipped too.
+        collection(members=["m1"])
+        r, b, content = ask("bench/core", **{"with": ["gone"]})
+        assert b["skipped"] == [{"seat": "with", "alias": "gone", "why": b["skipped"][0]["why"]}], b
+        assert "Not asked: gone (retired from the bench" in content and not any(q["model"] == "gone" for q in ASKED)
+        # Skipped before the max_calls cap (2 here), so the member after it still sits.
+        collection(members=["gone", "m1", "m2"])
+        r, b, content = ask("bench/core")
+        assert [s["alias"] for s in b["seats"]] == ["m1", "m2"] and [k["alias"] for k in b["skipped"]] == ["gone"], b
+        (TMP / "config" / "retired.json").unlink()
     finally:
         upstream.shutdown()
         gw.shutdown()

package/scripts/test_serve_edit.py

@@ -59,7 +59,12 @@ def main() -> int:
     shutil.copy(DATA / "config" / "endpoints.json", cfg / "endpoints.json")
     serve.endpoints.FILE = cfg / "endpoints.json"
     serve.endpoints.YAML = serve.YAML_PATH
+    # Collections: removing a model takes it out of them, and Restore puts it back.
+    serve.bench_collections.FILE = cfg / "collections.json"
+    serve.bench_collections.SETTINGS_FILE = cfg / "gateway.json"
     real = {p: p.read_bytes() for p in (DATA / "metrics" / "alerts.json", DATA / "config" / "endpoints.json")}
+    real_collections = (DATA / "config" / "collections.json").read_bytes() \
+        if (DATA / "config" / "collections.json").exists() else None
 
     d = serve.model_directory()
     some = sorted(d)[0]
@@ -130,6 +135,13 @@ def main() -> int:
         r = serve.remove_model("no-such" if "unknown" in label else victim, **kw)
         ok = not r["ok"] and not r["changed"]
         print(("ok   " if ok else "FAIL ") + label); failures += 0 if ok else 1
+    # In three collections: a member of two, the merger of one; a fourth without it.
+    colls = {"with-it": {"members": [other, victim], "merger": victim, "reply": "merged"},
+             "member": {"members": [victim, other], "merger": ""},
+             "doomed": {"members": [victim], "merger": ""},
+             "without": {"members": [other], "merger": other}}
+    (cfg / "collections.json").write_text(json.dumps(colls), encoding="utf-8")
+    logged = len(serve.CHANGES_PATH.read_text(encoding="utf-8").splitlines())
     before = len(serve.registered())
     r = serve.remove_model(victim, victim, "retired by provider on 2026-09-15")
     d3 = serve.model_directory()
@@ -142,8 +154,30 @@ def main() -> int:
           and served not in serve.registered())
     print(("ok   " if ok else "FAIL ") + f"removed {victim}; registry {before} -> {len(serve.registered())}")
     failures += 0 if ok else 1
+    after = json.loads((cfg / "collections.json").read_text(encoding="utf-8"))
+    log = [json.loads(x) for x in serve.CHANGES_PATH.read_text(encoding="utf-8").splitlines()[logged:]]
+    ok = (after == {"with-it": {"members": [other], "merger": "", "reply": "merged"},
+                    "member": {"members": [other], "merger": ""},
+                    "doomed": {"members": [], "merger": ""},
+                    "without": colls["without"]}
+          and retired[victim]["collections"] == {"with-it": ["members", "merger"], "member": ["members"],
+                                                 "doomed": ["members"]}
+          and "config/collections.json" in r["changed"]
+          and sorted(e["field"] for e in log if e["field"].startswith("collection "))
+          == ["collection doomed", "collection member", "collection with-it"])
+    print(("ok   " if ok else "FAIL ") + "removed from its collections, recorded and logged"
+          + ("" if ok else f" -> {after} {retired[victim]} {r['changed']} {log}")); failures += 0 if ok else 1
     expect_refused("retired name never reused", alias=other, new_alias=victim, new_role=None)
-    # restore: same alias back, out of retired.json, role and ad hoc kept
+    # The collection dialog's member search matches the served name too, so the view carries it.
+    view = serve.collections_view()
+    ok = (set(view["served"]) == set(view["models"]) and victim not in view["served"]
+          and view["served"].get(other) == d3[other]["served"])
+    print(("ok   " if ok else "FAIL ") + "collection view: served name for every member choice"); failures += 0 if ok else 1
+    # restore: same alias back, out of retired.json, role and ad hoc kept, and
+    # back into the collections that still exist (one deleted, one changed since)
+    serve.delete_collection("doomed")
+    serve.bench_collections.FILE.write_text(json.dumps({**json.loads(serve.bench_collections.FILE.read_text(
+        encoding="utf-8")), "member": {"members": [other, "someone-new"], "merger": ""}}), encoding="utf-8")
     r = serve.restore_model(victim)
     retired = json.loads((cfg / "retired.json").read_text(encoding="utf-8"))
     roles = json.loads((cfg / "model-roles.json").read_text(encoding="utf-8"))
@@ -151,6 +185,22 @@ def main() -> int:
           and roles["models"].get(victim) == new_role and victim in roles["ad_hoc"])
     print(("ok   " if ok else "FAIL ") + f"restored {victim}" + ("" if ok else f" -> {r}"))
     failures += 0 if ok else 1
+    back = json.loads((cfg / "collections.json").read_text(encoding="utf-8"))
+    ok = (back == {"with-it": {"members": [other, victim], "merger": victim, "reply": "merged"},
+                   "member": {"members": [other, "someone-new", victim], "merger": ""},
+                   "without": colls["without"]}
+          and "config/collections.json" in r["changed"])
+    print(("ok   " if ok else "FAIL ") + "restored into the collections that still exist"
+          + ("" if ok else f" -> {back} {r['changed']}")); failures += 0 if ok else 1
+    # One that was in no collection: collections.json untouched and not reported.
+    (cfg / "collections.json").write_text(json.dumps({"without": colls["without"]}), encoding="utf-8")
+    r = serve.remove_model(victim, victim, "retired for the no-collection check")
+    ok = (r["ok"] and "config/collections.json" not in r["changed"]
+          and "collections" not in json.loads((cfg / "retired.json").read_text(encoding="utf-8"))[victim])
+    r2 = serve.restore_model(victim)
+    ok = ok and r2["ok"] and "config/collections.json" not in r2["changed"]
+    print(("ok   " if ok else "FAIL ") + "in no collection: collections.json not touched"
+          + ("" if ok else f" -> {r} {r2}")); failures += 0 if ok else 1
     ok = not serve.restore_model(victim)["ok"]
     print(("ok   " if ok else "FAIL ") + "restore refused when not retired"); failures += 0 if ok else 1
     serve.remove_model(victim, victim, "retired again for the onboarding checks")
@@ -374,8 +424,10 @@ def main() -> int:
     real_alerts = DATA / "metrics" / "alerts.json"
     victim_keys = {k for k in json.loads(real[real_alerts] or b"{}") if k.endswith(f":{served}")}
     ok = (real[DATA / "config" / "endpoints.json"] == (DATA / "config" / "endpoints.json").read_bytes()
-          and victim_keys <= set(json.loads(real_alerts.read_bytes() or b"{}")))
-    print(("ok   " if ok else "FAIL ") + "the real alerts and endpoints were not touched")
+          and victim_keys <= set(json.loads(real_alerts.read_bytes() or b"{}"))
+          and real_collections == ((DATA / "config" / "collections.json").read_bytes()
+                                   if (DATA / "config" / "collections.json").exists() else None))
+    print(("ok   " if ok else "FAIL ") + "the real alerts, endpoints and collections were not touched")
     failures += 0 if ok else 1
     shutil.rmtree(tmp, ignore_errors=True)
     print("\nall rules hold" if not failures else f"\n{failures} FAILED")