@wendellresearch-ai/benchwarmers 0.3.0 → 0.3.2

package/mcp_server.py

@@ -15,7 +15,8 @@ the verdicts are that server's, whichever machine it is on:
     BENCHWARMERS_TIMEOUT   seconds to wait for a panel, default 900
 
 Tools: list_collections (GET /v1/models), ask (POST /v1/chat/completions),
-verdict (POST /bench/verdict). A refusal from the server, or no server at
+verdict (POST /bench/verdict), and pending_verdicts (POST /bench/pending) for
+the plugin's Stop hook. A refusal from the server, or no server at
 all, comes back as a tool error carrying its message. Standard library only,
 and nothing from metrics/: the data folder is the server's, not this one's.
 """
@@ -24,6 +25,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import sys
 import threading
 import urllib.error
@@ -90,6 +92,16 @@ TOOLS = [
                                    "description": "Claims it made that are untrue, one sentence each."}}},
      "annotations": {"readOnlyHint": False, "destructiveHint": False, "idempotentHint": True,
                      "openWorldHint": False}},
+    {"name": "pending_verdicts",
+     "title": "Seats still to judge",
+     "description": "For the Benchwarmers Claude Code plugin's Stop hook, which calls it at the end of a"
+                    " turn: the seats of this session's panels that have no verdict yet. You do not need to"
+                    " call it; send verdicts with `verdict`.",
+     "inputSchema": {"type": "object", "additionalProperties": False,
+                     "properties": {"session": {"type": "string", "description": "The Claude Code session id."},
+                                    "transcript": {"type": "string", "description": "The session's transcript_path."},
+                                    "active": {"description": "The hook's stop_hook_active: true says nothing."}}},
+     "annotations": {"readOnlyHint": True, "openWorldHint": False}},
 ]
 
 
@@ -102,11 +114,32 @@ def base_url() -> str:
     return url.removesuffix("/v1")
 
 
-def http(method: str, path: str, body: dict | None = None) -> dict:
-    url = base_url() + path
-    headers = {"Accept": "application/json"}
-    if key := (os.environ.get("BENCHWARMERS_KEY") or "").strip():
-        headers["Authorization"] = f"Bearer {key}"
+# (server, headers for it): where a tool's calls go.
+Conn = tuple[str, dict]
+
+
+def env_conn() -> Conn:
+    """The stdio server's: BENCHWARMERS_URL and BENCHWARMERS_KEY. Served over HTTP
+    (gateway.py's /mcp), the server's own address and the caller's credential instead."""
+    key = (os.environ.get("BENCHWARMERS_KEY") or "").strip()
+    return base_url(), {"Authorization": f"Bearer {key}"} if key else {}
+
+
+# A seat's call id, as ask returns it (gateway.py: p<date>-<time>-<hex>.<seat>); only ids the
+# call log knows count, so a loose match costs nothing.
+CALL_ID = re.compile(r"\bp\d{8}-\d{6}-[0-9a-f]{4}\.[a-z]+\d*\b")
+TRANSCRIPT_TAIL = 8 * 1024 * 1024                    # bytes of a transcript read: its recent panels
+# Per Claude Code session, the seats pending_verdicts already reminded it of. ponytail: in memory
+# and capped, so a restart reminds once more; keep it in the data folder if that ever matters.
+REMINDED: dict[str, set] = {}
+REMINDED_LOCK = threading.Lock()
+MAX_SESSIONS = 1000
+
+
+def http(conn: Conn, method: str, path: str, body: dict | None = None) -> dict:
+    base, extra = conn
+    url = base + path
+    headers = {"Accept": "application/json", **extra}
     data = None
     if body is not None:
         data = json.dumps(body).encode()
@@ -126,21 +159,21 @@ def http(method: str, path: str, body: dict | None = None) -> dict:
     except (urllib.error.URLError, ConnectionError) as e:
         reason = getattr(e, "reason", e)
         if isinstance(reason, TimeoutError):
-            raise ToolError(f"no answer from Benchwarmers at {base_url()} in {timeout:.0f} s;"
+            raise ToolError(f"no answer from Benchwarmers at {base} in {timeout:.0f} s;"
                             " set BENCHWARMERS_TIMEOUT for longer panels") from None
-        raise ToolError(f"Benchwarmers is not running at {base_url()} ({reason}). Start it (`benchwarmers`,"
+        raise ToolError(f"Benchwarmers is not running at {base} ({reason}). Start it (`benchwarmers`,"
                         " or its service), or point BENCHWARMERS_URL at the server you use.") from None
     except TimeoutError:
-        raise ToolError(f"no answer from Benchwarmers at {base_url()} in {timeout:.0f} s;"
+        raise ToolError(f"no answer from Benchwarmers at {base} in {timeout:.0f} s;"
                         " set BENCHWARMERS_TIMEOUT for longer panels") from None
     except ValueError:
-        raise ToolError(f"{base_url()} answered {method} {path} with something that is not JSON;"
+        raise ToolError(f"{base} answered {method} {path} with something that is not JSON;"
                         " is BENCHWARMERS_URL a Benchwarmers server?") from None
 
 
-def list_collections(_args: dict) -> list[str]:
+def list_collections(_args: dict, conn: Conn) -> list[str]:
     rows = []
-    for m in http("GET", "/v1/models").get("data") or []:
+    for m in http(conn, "GET", "/v1/models").get("data") or []:
         if ":" in m.get("id", ""):                     # the same collection in its other reply mode
             continue
         together = m.get("together") or {}
@@ -153,7 +186,7 @@ def list_collections(_args: dict) -> list[str]:
     return [json.dumps(rows, indent=1, ensure_ascii=False)]
 
 
-def ask(args: dict) -> list[str]:
+def ask(args: dict, conn: Conn) -> list[str]:
     collection, prompt = args.get("collection"), args.get("prompt")
     if not (isinstance(collection, str) and collection.strip()):
         raise ToolError("`collection` is required: a name from list_collections, e.g. core")
@@ -168,7 +201,7 @@ def ask(args: dict) -> list[str]:
         body["bench"] = {"reply": "merged" if args["merged"] else "all"}
     if args.get("with"):
         body["with"] = args["with"]
-    reply = http("POST", "/v1/chat/completions", body)
+    reply = http(conn, "POST", "/v1/chat/completions", body)
     content = reply["choices"][0]["message"]["content"]
     bench = reply.get("bench") or {}
     merged = bench.get("reply") == "merged"
@@ -185,19 +218,59 @@ def ask(args: dict) -> list[str]:
             + json.dumps(data, indent=1, ensure_ascii=False)]
 
 
-def verdict(args: dict) -> list[str]:
+def pending_verdicts(args: dict, conn: Conn) -> list[str]:
+    """For the plugin's Stop hook: the seats of this session's panels with no verdict yet, as the
+    hook's feedback for Claude (Claude Code reads it as a command hook's stdout), once per seat;
+    nothing when all are judged, already reminded of, or the hook is already continuing a turn.
+
+    The session's panels are the call ids in its transcript (the hook's transcript_path): Claude
+    Code keeps no Mcp-Session-Id, so the transcript is the one record of which panels were whose,
+    a subagent's included once its report names them. Read by whichever process runs this: the
+    stdio server always can, a server on another machine cannot, and then this says nothing.
+    Which of those ids are unjudged seats that answered is the server's call log, so a verdict
+    sent any way counts."""
+    session, path = args.get("session"), args.get("transcript")
+    if str(args.get("active")).lower() == "true" or not (isinstance(session, str) and isinstance(path, str)) \
+            or not path.endswith(".jsonl"):
+        return [""]
+    try:
+        with open(path, "rb") as fh:
+            fh.seek(max(0, os.fstat(fh.fileno()).st_size - TRANSCRIPT_TAIL))
+            calls = list(dict.fromkeys(CALL_ID.findall(fh.read().decode("utf-8", "replace"))))
+    except OSError:
+        return [""]
+    if not calls:
+        return [""]
+    pending = http(conn, "POST", "/bench/pending", {"calls": calls}).get("pending") or []
+    with REMINDED_LOCK:
+        if session not in REMINDED and len(REMINDED) >= MAX_SESSIONS:
+            REMINDED.pop(next(iter(REMINDED)))         # the oldest session
+        reminded = REMINDED.setdefault(session, set())
+        due = [c for c in pending if c not in reminded]
+        reminded.update(due)
+    if not due:
+        return [""]
+    return [json.dumps({"hookSpecificOutput": {"hookEventName": "Stop", "additionalContext": (
+        f"Benchwarmers: {len(due)} seat{'s' * (len(due) != 1)} from this session's panels"
+        f" {'have' if len(due) != 1 else 'has'} no verdict yet: {', '.join(due)}. Send a verdict for each"
+        " seat you judged (the verdict tool, with its true claims in `real` and untrue ones in `false`),"
+        " or say in one line why not.")}})]
+
+
+def verdict(args: dict, conn: Conn) -> list[str]:
     if not (isinstance(args.get("call"), str) and args["call"].strip()):
         raise ToolError("`call` is required: the seat's call id from ask")
     if not isinstance(args.get("useful"), bool):
         raise ToolError("`useful` must be true or false")
     body = {"call": args["call"].strip(), "useful": args["useful"], "note": args.get("note") or "",
             "real": args.get("real") or [], "false": args.get("false") or []}
-    http("POST", "/bench/verdict", body)
+    http(conn, "POST", "/bench/verdict", body)
     return [f"Recorded {body['call']}: {'useful' if body['useful'] else 'not useful'},"
             f" {len(body['real'])} real, {len(body['false'])} false."]
 
 
-HANDLERS = {"list_collections": list_collections, "ask": ask, "verdict": verdict}
+HANDLERS = {"list_collections": list_collections, "ask": ask, "verdict": verdict,
+            "pending_verdicts": pending_verdicts}
 OUT = threading.Lock()
 
 
@@ -208,46 +281,70 @@ def send(message: dict) -> None:
         sys.stdout.buffer.flush()
 
 
-def call_tool(rid, params: dict) -> None:
+def call_tool(rid, params: dict, conn: Conn) -> dict:
     name, args = params.get("name"), params.get("arguments") or {}
-    if name not in HANDLERS:
-        return send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"unknown tool {name!r}"}})
+    if not isinstance(name, str) or name not in HANDLERS:
+        return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"unknown tool {name!r}"}}
     try:
         if not isinstance(args, dict):
             raise ToolError("arguments must be an object")
-        texts, failed = HANDLERS[name](args), False
+        texts, failed = HANDLERS[name](args, conn), False
     except ToolError as e:
         texts, failed = [str(e)], True
     except Exception as e:                             # noqa: BLE001 -- a bad reply must not kill the server
         texts, failed = [f"{type(e).__name__}: {e}"], True
-    send({"jsonrpc": "2.0", "id": rid,
-          "result": {"content": [{"type": "text", "text": t} for t in texts], "isError": failed}})
+    return {"jsonrpc": "2.0", "id": rid,
+            "result": {"content": [{"type": "text", "text": t} for t in texts], "isError": failed}}
 
 
-def handle(msg) -> None:
+def answer(msg, conn: Conn) -> dict | None:
+    """The reply to one JSON-RPC message, or None for a notification. Both
+    transports: the stdio loop below and the HTTP route (gateway.py, /mcp).
+    Never raises: a request left unanswered would hang its client."""
+    try:
+        return _answer(msg, conn)
+    except Exception as e:                             # noqa: BLE001
+        rid = msg.get("id") if isinstance(msg, dict) else None
+        return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32603, "message": f"{type(e).__name__}: {e}"}}
+
+
+def _answer(msg, conn: Conn) -> dict | None:
     if not isinstance(msg, dict) or msg.get("jsonrpc") != "2.0" or not isinstance(msg.get("method"), str):
         rid = msg.get("id") if isinstance(msg, dict) else None
-        return send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32600, "message": "invalid request"}})
+        return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32600, "message": "invalid request"}}
     if "id" not in msg:                                # a notification (initialized, cancelled): nothing to answer
-        return
+        return None
     rid, method, params = msg["id"], msg["method"], msg.get("params") or {}
+    if not isinstance(params, dict):
+        return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": "params must be an object"}}
     if method == "initialize":
         asked = params.get("protocolVersion")
-        send({"jsonrpc": "2.0", "id": rid, "result": {
+        return {"jsonrpc": "2.0", "id": rid, "result": {
             "protocolVersion": asked if asked in PROTOCOLS else PROTOCOLS[0],
             "capabilities": {"tools": {"listChanged": False}},
             "serverInfo": {"name": "benchwarmers", "title": "Benchwarmers", "version": VERSION},
-            "instructions": RULES}})
-    elif method == "ping":
-        send({"jsonrpc": "2.0", "id": rid, "result": {}})
-    elif method == "tools/list":
-        send({"jsonrpc": "2.0", "id": rid, "result": {"tools": TOOLS}})
-    elif method == "tools/call":
+            "instructions": RULES}}
+    if method == "ping":
+        return {"jsonrpc": "2.0", "id": rid, "result": {}}
+    if method == "tools/list":
+        return {"jsonrpc": "2.0", "id": rid, "result": {"tools": TOOLS}}
+    if method == "tools/call":
+        return call_tool(rid, params, conn)
+    return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"no method {method!r}"}}
+
+
+def reply_to(msg) -> None:
+    if (reply := answer(msg, env_conn())) is not None:
+        send(reply)
+
+
+def handle(msg) -> None:
+    if isinstance(msg, dict) and msg.get("method") == "tools/call":
         # A panel takes minutes: on its own thread, so a ping or a second call is answered meanwhile.
         # Not a daemon: when stdin closes, a call already sent (a verdict) still finishes.
-        threading.Thread(target=call_tool, args=(rid, params)).start()
+        threading.Thread(target=reply_to, args=(msg,)).start()
     else:
-        send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"no method {method!r}"}})
+        reply_to(msg)
 
 
 def main() -> None:
@@ -256,7 +353,7 @@ def main() -> None:
             continue
         try:
             msg = json.loads(raw)
-        except ValueError:
+        except (ValueError, RecursionError):           # RecursionError: nesting deeper than json can parse
             send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error"}})
             continue
         handle(msg)

package/metrics/client.py

@@ -2,8 +2,9 @@
 
     python metrics/client.py <alias> "prompt"     # one call; prints what ask_once would return
 
-RELEASE-PLAN.md Phases 2a and 2b. A drop-in for `llm -m <alias>`, chosen on
-Settings or by BENCH_CLIENT=builtin (see chosen()), used by suite/run.py's
+RELEASE-PLAN.md Phases 2a and 2b. A drop-in for `llm -m <alias>`, and the
+default since v0.3 (`llm` stays a choice on Settings or BENCH_CLIENT=llm; see
+chosen()), used by suite/run.py's
 ask_once, and so the perf probe, the suite, retries and auto-retire's
 confirmation call, and by the gateway's seats. It reads the same
 registration (the yaml entry's model name, base, key name and headers), the
@@ -118,7 +119,6 @@ import json
 import os
 import pathlib
 import random
-import shutil
 import sys
 import threading
 import time
@@ -146,10 +146,11 @@ CHOICE = DATA / "config" / "client.json"           # the Settings page's choice
 
 def chosen() -> tuple[str, str]:
     """The configured client and what set it: BENCH_CLIENT ("environment"),
-    else the Settings page's choice ("settings"), else ("default") `llm` where
-    the llm CLI is installed and the built-in client where it is not, as in
-    an install from npm or a container. Read on every call, so a change
-    applies to the next one."""
+    else the Settings page's choice ("settings"), else ("default") the
+    built-in client, installed `llm` or not: since v0.3 the `llm` CLI is an
+    alternative chosen on Settings, no longer the default (its trial week:
+    RELEASE-PLAN Phase 2a). Read on every call, so a change applies to the
+    next one."""
     env = (os.environ.get("BENCH_CLIENT") or "").strip().lower()
     if env:
         return env, "environment"
@@ -157,7 +158,7 @@ def chosen() -> tuple[str, str]:
         saved = json.loads(CHOICE.read_text(encoding="utf-8")).get("client")
     except (OSError, ValueError, AttributeError):
         saved = None
-    return (saved, "settings") if saved in CLIENTS else ("llm" if shutil.which("llm") else "builtin", "default")
+    return (saved, "settings") if saved in CLIENTS else ("builtin", "default")
 
 
 def backend(alias: str = "") -> str:

package/metrics/guard.py

@@ -220,7 +220,14 @@ def startup(address: str) -> None:
 
 def gateway_route(path: str) -> bool:
     """A route the gateway answers, on its own port or the dashboard's."""
-    return path.startswith(GATEWAY_ROUTES) or path.rstrip("/") in OLLAMA_ROUTES
+    return path.startswith(GATEWAY_ROUTES) or path.rstrip("/") in OLLAMA_ROUTES or mcp_route(path)
+
+
+def mcp_route(path: str) -> bool:
+    """MCP over HTTP (gateway.py): /mcp, or /v1/mcp for a BENCHWARMERS_URL
+    pasted with its /v1; doubled and trailing slashes from joining a URL
+    that ends in one are the same route."""
+    return re.sub("/{2,}", "/", path.split("?", 1)[0]).rstrip("/") in ("/mcp", "/v1/mcp")
 
 
 def _sign(expires: int) -> bytes:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/test_mcp.py

@@ -23,6 +23,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 SEEN: list[tuple[str, str, dict, str]] = []          # (method, path, body, Authorization)
+JUDGED: set[str] = set()                             # call ids the stub has a verdict for
 
 MODELS = {"object": "list", "data": [
     {"id": "bench/core", "description": "second opinions", "tags": ["general"], "members": ["m1", "m2"],
@@ -44,7 +45,10 @@ class Stub(BaseHTTPRequestHandler):
         body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
         SEEN.append(("POST", self.path, body, self.headers.get("Authorization", "")))
         if self.path == "/bench/verdict":
+            JUDGED.add(body["call"])
             return self._send(200, {"ok": True, "recorded": {"call": body["call"]}})
+        if self.path == "/bench/pending":
+            return self._send(200, {"pending": [c for c in body["calls"] if c not in JUDGED]})
         if body["model"] == "bench/nope":
             return self._send(404, {"error": {"message": "no collection nope; choose one of bench/core"}})
         time.sleep(1)                                  # a panel takes a while
@@ -112,7 +116,8 @@ def main() -> int:
     assert c.request("ping")["result"] == {}
 
     tools = {t["name"]: t for t in c.request("tools/list")["result"]["tools"]}
-    assert set(tools) == {"list_collections", "ask", "verdict"}, tools
+    assert set(tools) == {"list_collections", "ask", "verdict", "pending_verdicts"}, tools
+    assert c.tool("pending_verdicts", active="false")["content"][0]["text"] == ""     # no panel yet: nothing
     assert tools["ask"]["inputSchema"]["required"] == ["collection", "prompt"]
 
     r = c.tool("list_collections")
@@ -153,10 +158,40 @@ def main() -> int:
                                                 "real": ["a true claim"], "false": ["x"]}), SEEN[-1]
     assert c.tool("verdict", call="p1.member1")["isError"]          # useful is required
 
+    # The Stop hook's tool: the call ids in the session's transcript (each once), the server says which
+    # are unjudged, and those come back as feedback for Claude, once; nothing while the hook is already
+    # continuing a turn, nor for a transcript it cannot read.
+    import tempfile
+    a, b = "p20261025-120000-abcd.member1", "p20261025-120000-abcd.explorer2"
+    with tempfile.TemporaryDirectory() as tmp:
+        transcript = pathlib.Path(tmp) / "session.jsonl"
+        transcript.write_text(f'{{"tool_result": "`{a}` ok"}}\n{{"tool_result": "`{b}` ok, `{a}` again"}}\n',
+                              encoding="utf-8")
+        hook = {"session": "s", "transcript": str(transcript)}
+        assert not c.tool("verdict", call=a, useful=True)["isError"]
+        assert c.tool("pending_verdicts", active="true", **hook)["content"][0]["text"] == ""
+        out = json.loads(c.tool("pending_verdicts", active="false", **hook)["content"][0]["text"])
+        assert SEEN[-1][1:3] == ("/bench/pending", {"calls": [a, b]}), SEEN[-1]
+        ctx = out["hookSpecificOutput"]
+        assert ctx["hookEventName"] == "Stop" and "1 seat " in ctx["additionalContext"], ctx
+        assert b in ctx["additionalContext"] and a not in ctx["additionalContext"], ctx
+        assert c.tool("pending_verdicts", active="false", **hook)["content"][0]["text"] == ""        # reminded
+        assert c.tool("pending_verdicts", session="t", transcript=str(transcript) + ".missing.jsonl",
+                      active="false")["content"][0]["text"] == ""
+        assert c.tool("pending_verdicts", session="t", transcript=str(pathlib.Path(tmp) / "x.txt"),
+                      active="false")["content"][0]["text"] == ""                                 # not a transcript
+
     assert c.request("tools/call", {"name": "nope"})["error"]["code"] == -32602
+    assert c.request("tools/call", {"name": ["x"]})["error"]["code"] == -32602         # unhashable: answered
     assert c.request("resources/list")["error"]["code"] == -32601
     c.send("{not json")
     assert c.read()["error"]["code"] == -32700
+    c.send("[" * 100_000)                                          # deeper than json parses
+    assert c.read()["error"]["code"] == -32700
+    for method in ("initialize", "tools/call"):                    # params not an object: answered, never raised
+        c.send({"jsonrpc": "2.0", "id": method, "method": method, "params": [1]})
+        assert c.read() == {"jsonrpc": "2.0", "id": method,
+                            "error": {"code": -32602, "message": "params must be an object"}}
     c.close()
 
     # An unknown protocol version gets the newest; no server running is a clear tool error.
@@ -170,9 +205,179 @@ def main() -> int:
     assert r["isError"] and "not running" in r["content"][0]["text"], r
     c.close()
     stub.shutdown()
-    print("mcp server ok: initialize, ping, tools/list, each tool, refusals, a ping mid-panel, not running")
+    over_http()
+    print("mcp server ok: initialize, ping, tools/list, each tool, refusals, a ping mid-panel, not running;"
+          " /mcp on both ports, its paths, the key and its scope through the loopback, the guard;"
+          " pending_verdicts and /bench/pending for the Stop hook")
     return 0
 
 
+def over_http() -> None:
+    """POST /mcp on the real handlers, both ports: the tools reach the server's own routes with the caller's key."""
+    import tempfile
+    import urllib.error
+    import urllib.request
+    sys.path.insert(0, str(ROOT))
+    import serve
+    import gateway
+    import guard
+
+    def post(port, path, msg, **headers):
+        headers = {"Host": f"127.0.0.1:{port}", "Content-Type": "application/json", **headers}
+        data = msg.encode() if isinstance(msg, str) else json.dumps(msg).encode()
+        req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", data, headers,
+                                     method="POST" if msg is not None else "GET")
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                raw = r.read()
+                return r.status, json.loads(raw) if raw else None
+        except urllib.error.HTTPError as e:
+            return e.code, json.loads(e.read() or b"null")
+
+    def tool(port, name, path="/mcp", **headers_and_args):
+        headers = {k: headers_and_args.pop(k) for k in ("Authorization", "Origin") if k in headers_and_args}
+        status, reply = post(port, path, {"jsonrpc": "2.0", "id": 7, "method": "tools/call",
+                                          "params": {"name": name, "arguments": headers_and_args}}, **headers)
+        assert status == 200 and reply["id"] == 7, (status, reply)
+        return reply["result"]
+
+    servers = [ThreadingHTTPServer(("127.0.0.1", 0), h) for h in (serve.Handler, gateway.Handler)]
+    for s in servers:
+        threading.Thread(target=s.serve_forever, daemon=True).start()
+    saved = (guard.KEYS, gateway.bench_collections.FILE, gateway.run_collection, sys.stderr,
+             os.environ.pop("BENCHWARMERS_ADMIN_TOKEN", None), os.environ.pop("BENCHWARMERS_REQUIRE_KEYS", None),
+             gateway.offload.LOG)
+    ran = []
+    with tempfile.TemporaryDirectory() as tmp:
+        try:
+            t = pathlib.Path(tmp)
+            guard.KEYS, gateway.bench_collections.FILE = t / "gateway-keys.json", t / "collections.json"
+            gateway.bench_collections.FILE.write_text(json.dumps({"core": {"members": [], "description": "x"},
+                                                                  "vision": {"members": [], "description": "y"}}),
+                                                      encoding="utf-8")
+            gateway.offload.LOG = t / "offloads.jsonl"
+            gateway.run_collection = lambda name, messages, nominated, reply, who: (
+                ran.append((name, who)) or f"answer from {name}",
+                {"seats": [{"call": f"{name}.member1", "seat": "member", "alias": "m1", "status": "ok"},
+                           {"call": f"{name}.explorer", "seat": "explorer", "alias": "x1", "status": "error"}]})
+            sys.stderr = open(os.devnull, "w")         # the access log
+            for port in (s.server_port for s in servers):
+                init = {"jsonrpc": "2.0", "id": 1, "method": "initialize",
+                        "params": {"protocolVersion": "2025-06-18", "capabilities": {}}}
+                # A BENCHWARMERS_URL with /v1 or a trailing slash, joined to /mcp, is the same route.
+                for path in ("/mcp", "/mcp/", "//mcp", "/v1/mcp", "/v1//mcp"):
+                    status, reply = post(port, path, init)
+                    assert status == 200 and reply["result"]["serverInfo"]["name"] == "benchwarmers", (path, reply)
+                assert not guard.mcp_route("/mcpx") and not guard.mcp_route("/v1/mcp/x")
+                assert post(port, "/mcp", {"jsonrpc": "2.0", "method": "notifications/initialized"}) == (202, None)
+                assert post(port, "/mcp", None)[0] == 405                     # no server stream
+                assert post(port, "/mcp", "{not json") == (400, {"jsonrpc": "2.0", "id": None,
+                                                                  "error": {"code": -32700, "message": "parse error"}})
+                status, reply = post(port, "/mcp", {"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
+                assert {x["name"] for x in reply["result"]["tools"]} == {"list_collections", "ask", "verdict",
+                                                                          "pending_verdicts"}
+                r = tool(port, "list_collections", path="//mcp")
+                assert not r["isError"] and [x["name"] for x in json.loads(r["content"][0]["text"])] == \
+                    ["bench/core", "bench/vision"], r
+                r = tool(port, "ask", collection="core", prompt="hi")
+                assert not r["isError"] and "answer from core" in r["content"][0]["text"], r
+                # A client that takes SSE (every streamable-HTTP client must) gets a tool call as a stream,
+                # keep-alives while it runs and then the reply, so its idle timeouts never fire mid-panel.
+                saved_keepalive, gateway.KEEPALIVE = gateway.KEEPALIVE, 0.2
+                slow, gateway.run_collection = gateway.run_collection, lambda *a, **k: (time.sleep(0.7), slow(*a, **k))[1]
+                req = urllib.request.Request(
+                    f"http://127.0.0.1:{port}/mcp", json.dumps({"jsonrpc": "2.0", "id": 9, "method": "tools/call",
+                                                                "params": {"name": "ask", "arguments": {
+                                                                    "collection": "core", "prompt": "hi"}}}).encode(),
+                    {"Host": f"127.0.0.1:{port}", "Content-Type": "application/json",
+                     "Accept": "application/json, text/event-stream"})
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    assert resp.headers["Content-Type"] == "text/event-stream", resp.headers
+                    events = resp.read().decode()
+                assert events.startswith(": keep-alive\n\n") and events.endswith("\n\n"), events
+                reply = json.loads(events.rsplit("data: ", 1)[1])
+                assert reply["id"] == 9 and "answer from core" in reply["result"]["content"][0]["text"], reply
+                # With a progressToken, progress notifications instead: what resets a client's idle timeout.
+                req = urllib.request.Request(
+                    f"http://127.0.0.1:{port}/mcp", json.dumps({"jsonrpc": "2.0", "id": 11, "method": "tools/call",
+                                                                "params": {"name": "ask", "_meta": {"progressToken": "t1"},
+                                                                           "arguments": {"collection": "core",
+                                                                                         "prompt": "hi"}}}).encode(),
+                    {"Host": f"127.0.0.1:{port}", "Content-Type": "application/json",
+                     "Accept": "application/json, text/event-stream"})
+                with urllib.request.urlopen(req, timeout=30) as resp:
+                    messages = [json.loads(line[6:]) for line in resp.read().decode().splitlines()
+                                if line.startswith("data: ")]
+                gateway.KEEPALIVE, gateway.run_collection = saved_keepalive, slow
+                assert ": keep-alive" not in json.dumps(messages) and len(messages) >= 2, messages
+                assert all(m["method"] == "notifications/progress" and m["params"]["progressToken"] == "t1"
+                           for m in messages[:-1]), messages
+                assert messages[-1]["id"] == 11 and not messages[-1]["result"]["isError"], messages[-1]
+                # A tool call that cannot be answered still ends its stream (it once kept it open forever).
+                req = urllib.request.Request(
+                    f"http://127.0.0.1:{port}/mcp", json.dumps({"jsonrpc": "2.0", "id": 10, "method": "tools/call",
+                                                                "params": [1]}).encode(),
+                    {"Host": f"127.0.0.1:{port}", "Content-Type": "application/json",
+                     "Accept": "application/json, text/event-stream"})
+                with urllib.request.urlopen(req, timeout=5) as resp:
+                    reply = json.loads(resp.read().decode().rsplit("data: ", 1)[1])
+                assert reply["id"] == 10 and reply["error"]["code"] == -32602, reply
+                assert post(port, "/mcp", "[" * 100_000)[1]["error"]["code"] == -32700
+                # The guard on /mcp itself: Host and Origin, as on /v1.
+                assert post(port, "/mcp", init, Host="evil.example")[0] == 403
+                assert post(port, "/mcp", init, Origin="http://evil.example")[0] == 403
+            assert ran == [("core", "local")] * 6, ran
+
+            # The Stop hook over HTTP: the call ids in the transcript, of which the call log says which are
+            # seats that answered (ok or empty, not errored) and have no verdict; reminded of once; a
+            # verdict sent any way clears one; an id the log does not know is not a seat.
+            g = servers[1].server_port
+            ok, empty, failed, judged, unknown = (f"p20261025-130000-beef.{s}" for s in
+                                                  ("member1", "member2", "explorer", "explorer2", "member9"))
+            for call, note in ((ok, "member; ok in 1.0s"), (empty, "member; empty in 2.0s"),
+                               (failed, "explorer; error in 0.4s: Error: 429"), (judged, "explorer 2; ok in 1.0s")):
+                gateway.offload.add("m", "t", None, note, call)
+            gateway.offload.add("m", "t", True, "checked", judged)
+            status, reply = post(g, "/bench/pending", {"calls": [ok, empty, failed, judged, unknown]})
+            assert (status, reply) == (200, {"pending": [ok, empty]}), reply
+            assert post(g, "/bench/pending", {"calls": "x"})[0] == 400
+            transcript = t / "claude-1.jsonl"
+            transcript.write_text(" ".join((ok, empty, failed, judged, unknown, ok)), encoding="utf-8")
+
+            def pending(session="claude-1"):
+                return tool(g, "pending_verdicts", session=session, transcript=str(transcript),
+                            active="false")["content"][0]["text"]
+            ctx = json.loads(pending())["hookSpecificOutput"]["additionalContext"]
+            assert "2 seats " in ctx and ok in ctx and empty in ctx and failed not in ctx and judged not in ctx, ctx
+            assert pending() == ""                                         # once per session
+            assert ok in pending("claude-2")                               # another session: its own reminder
+            assert not tool(g, "verdict", call=ok, useful=True)["isError"]
+            assert post(g, "/bench/pending", {"calls": [ok, empty]})[1] == {"pending": [empty]}
+
+            # Keys required: /mcp asks for one, and the loopback carries it, scope and all.
+            os.environ["BENCHWARMERS_REQUIRE_KEYS"] = "1"
+            vision = {"Authorization": "Bearer " + guard.make_key("vision-only", ["vision"])}
+            ran.clear()
+            for port in (s.server_port for s in servers):
+                assert post(port, "/mcp", init)[0] == 401
+                assert post(port, "/mcp", init, Authorization="Bearer ")[0] == 401     # the plugin with no key set
+                r = tool(port, "list_collections", **vision)
+                assert [x["name"] for x in json.loads(r["content"][0]["text"])] == ["bench/vision"], r
+                r = tool(port, "ask", collection="core", prompt="hi", **vision)
+                assert r["isError"] and "may call bench/vision" in r["content"][0]["text"], r
+                assert not tool(port, "ask", collection="vision", prompt="hi", **vision)["isError"]
+            assert ran == [("vision", "vision-only")] * 2, ran
+        finally:
+            sys.stderr.close()
+            guard.KEYS, gateway.bench_collections.FILE, gateway.run_collection, sys.stderr = saved[:4]
+            gateway.offload.LOG = saved[6]
+            for name, value in (("BENCHWARMERS_ADMIN_TOKEN", saved[4]), ("BENCHWARMERS_REQUIRE_KEYS", saved[5])):
+                os.environ.pop(name, None)
+                if value is not None:
+                    os.environ[name] = value
+            for s in servers:
+                s.shutdown()
+
+
 if __name__ == "__main__":
     sys.exit(main())

package/scripts/test_serve_edit.py

@@ -291,7 +291,7 @@ def main() -> int:
     model_client.CHOICE = cfg / "client.json"
     env, real_which = os.environ.pop("BENCH_CLIENT", None), shutil.which
     try:
-        shutil.which = lambda name: "/usr/bin/" + name                 # llm installed: the default is llm
+        shutil.which = lambda name: "/usr/bin/" + name                 # llm installed: still built-in
         first = model_client.chosen()
         shutil.which = lambda name: None                               # not installed (npm, a container)
         bare = model_client.chosen()
@@ -310,7 +310,7 @@ def main() -> int:
         os.environ.pop("BENCH_CLIENT", None)
         if env is not None:
             os.environ["BENCH_CLIENT"] = env
-    ok = (first == ("llm", "default") and bare == ("builtin", "default") and made["ok"] and picked == (("builtin", "settings"), "builtin")
+    ok = (first == ("builtin", "default") and bare == ("builtin", "default") and made["ok"] and picked == (("builtin", "settings"), "builtin")
           and pinned == (("llm", "environment"), False) and not no_llm and not unknown)
     print(("ok   " if ok else "FAIL ") + "model client choice" + ("" if ok else f" -> {first} {made} {picked} {pinned}"))
     failures += 0 if ok else 1