@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.2

package/scripts/test_server_json.py

@@ -0,0 +1,79 @@
+"""server.json, the MCP Registry's entry: it says what the packages really are.
+
+    python scripts/test_server_json.py
+
+A release publishes server.json to the registry after npm and PyPI
+(.github/workflows/release.yml), and the registry refuses it, after the
+packages are already out, when the entry and the packages disagree. So this
+checks what the registry checks, and what it cannot: the name is in the
+io.github.<owner> namespace GitHub's OIDC token proves (the owner as the
+repository URL spells it, since the match is case-sensitive); every version is
+VERSION; package.json carries the name as mcpName (npm's proof) and the README
+PyPI shows carries `mcp-name: <name>` (PyPI's); the identifiers are the real
+package names; the arguments start the MCP server through launcher.py; the
+environment variables are the ones mcp_server.py reads. Standard library
+only: the full schema is validated by mcp-publisher in the release itself.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import re
+import sys
+import tomllib
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import launcher  # noqa: E402
+
+SCHEMA = "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json"
+
+
+def main() -> int:
+    server = json.loads((ROOT / "server.json").read_text(encoding="utf-8"))
+    npm = json.loads((ROOT / "package.json").read_text(encoding="utf-8"))
+    pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]
+    version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
+
+    assert server["$schema"] == SCHEMA, server["$schema"]
+    name = server["name"]
+    owner = re.fullmatch(r"git\+https://github\.com/([^/]+)/[^/]+?\.git", npm["repository"]["url"]).group(1)
+    assert name == f"io.github.{owner}/benchwarmers", f"{name}: not in io.github.{owner}, the OIDC namespace"
+    assert re.fullmatch(r"[a-zA-Z0-9.-]+/[a-zA-Z0-9._-]+", name) and 3 <= len(name) <= 200, name
+    assert 1 <= len(server["description"]) <= 100, f"description is {len(server['description'])} chars, max 100"
+    assert 1 <= len(server.get("title", "x")) <= 100, server["title"]
+    assert server["repository"] == {"url": f"https://github.com/{owner}/benchwarmers", "source": "github"}
+
+    assert npm["version"] == version, f"package.json says {npm['version']}, VERSION says {version}"
+    assert server["version"] == version, f"server.json says {server['version']}, VERSION says {version}"
+    for p in server["packages"]:
+        assert p["version"] == version, f"server.json's {p['registryType']} package says {p['version']}"
+
+    # Ownership: npm's package.json and PyPI's README name the server.
+    assert npm.get("mcpName") == name, f"package.json mcpName is {npm.get('mcpName')!r}, not {name}"
+    assert pyproject["readme"] == "README.md", pyproject["readme"]
+    readme = (ROOT / "README.md").read_text(encoding="utf-8")
+    assert re.search(r"mcp-name: " + re.escape(name) + r"(?=\s|<|-->|$)", readme), \
+        f"README.md has no 'mcp-name: {name}' followed by a space, a newline or -->"
+
+    kinds = {p["registryType"]: p for p in server["packages"]}
+    assert sorted(kinds) == ["npm", "pypi"] and len(kinds) == len(server["packages"]), list(kinds)
+    assert kinds["npm"]["identifier"] == npm["name"], kinds["npm"]["identifier"]
+    assert kinds["pypi"]["identifier"] == pyproject["name"], kinds["pypi"]["identifier"]
+
+    read = set(re.findall(r'environ\.get\("(\w+)"', (ROOT / "mcp_server.py").read_text(encoding="utf-8")))
+    for kind, p in kinds.items():
+        assert p["transport"] == {"type": "stdio"}, (kind, p["transport"])
+        args = [a["value"] for a in p["packageArguments"] if a["type"] == "positional"]
+        assert launcher.plan(args).get("script") == "mcp_server.py", f"{kind}: {args} does not start mcp_server.py"
+        names = [e["name"] for e in p["environmentVariables"]]
+        assert set(names) == read and len(names) == len(read), f"{kind}: lists {names}, mcp_server.py reads {read}"
+        for e in p["environmentVariables"]:
+            assert e.get("isSecret", False) == (e["name"] == "BENCHWARMERS_KEY"), e
+    print(f"server.json ok: {name} {version}, npm and pypi")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/test_verdict.py

@@ -1,98 +1,102 @@
-"""Verdicts and the gateway's `bench` field, against a throwaway log. No model or
-endpoint is called and the real offloads.jsonl and runs/gateway/ are never
-written. The collection comes from the data folder (in CI, tests/fixtures/data).
-
-    python scripts/test_verdict.py
-"""
-
-from __future__ import annotations
-
-import json
-import pathlib
-import sys
-import tempfile
-import threading
-import urllib.request
-from http.server import ThreadingHTTPServer
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-sys.path[:0] = [str(ROOT), str(ROOT / "metrics"), str(ROOT / "suite")]
-import gateway  # noqa: E402
-import offload  # noqa: E402
-import run  # noqa: E402
-import serve  # noqa: E402
-
-tmp = pathlib.Path(tempfile.mkdtemp(prefix="bench-verdict-"))
-offload.LOG = tmp / "offloads.jsonl"
-gateway.LOG_DIR = tmp
-
-
-def verdicts() -> None:
-    offload.add("m1", "task", None, "member; ok in 3s", "pX.member1")
-    assert not serve.record_verdict("pX.member1", None, "", [], [])["ok"]        # no pick
-    assert not serve.record_verdict("pX.member1", True, "", "x", [])["ok"]       # claims not a list
-    assert not serve.record_verdict("nope.member1", True, "", [], [])["ok"]      # unknown call
-    assert serve.record_verdict("pX.member1", True, "  found   it ", ["real one", " "], ["wrong one"])["ok"]
-    e = offload.entries()[-1]
-    assert (e["useful"], e["note"], e["real"], e["false"]) == (True, "found it", ["real one"], ["wrong one"]), e
-    n = len(offload.LOG.read_text(encoding="utf-8").splitlines())
-    assert serve.record_verdict("pX.member1", True, "found it", ["real one"], ["wrong one"])["ok"]
-    assert len(offload.LOG.read_text(encoding="utf-8").splitlines()) == n, "the same verdict again wrote a line"
-    # The dashboard form shows note and claims for editing: blank means cleared.
-    assert serve.record_verdict("pX.member1", False, "", [], [])["ok"]
-    e = offload.entries()[-1]
-    assert e["useful"] is False and e["note"] == "" and "real" not in e and "false" not in e, e
-    # The CLI: a blank note or claim list keeps the earlier one.
-    offload.verdict("pX.member1", False, "", ["cli claim"])
-    offload.verdict("pX.member1", True, "")
-    assert offload.entries()[-1]["real"] == ["cli claim"], offload.entries()[-1]
-    assert len(offload.entries()) == 1                                           # superseded by call id
-
-
-def bench_field() -> None:
-    gateway.pick_shape = lambda text: ("general", "test")
-    gateway.ask_seat = lambda alias, system, prompt, messages=None: {
-        "status": "ok", "answer": f"answer from {alias}", "error": "", "seconds": 0.1}
-    # The explorer seats ask every endpoint's /models; None is its own "cannot
-    # be asked, do not filter", so no request leaves the machine.
-    run._served_model_names = lambda cache=None: None
-    srv = ThreadingHTTPServer(("127.0.0.1", 0), gateway.Handler)
-    gateway.PORT = srv.server_address[1]          # the Host check expects the port it runs on
-    threading.Thread(target=srv.serve_forever, daemon=True).start()
-    url = f"http://127.0.0.1:{srv.server_address[1]}/v1/chat/completions"
-
-    def post(body, **headers):
-        req = urllib.request.Request(url, json.dumps(body).encode(),
-                                     {"Content-Type": "application/json", **headers})
-        with urllib.request.urlopen(req) as r:
-            return r.read().decode()
-
-    # metrics/guard.py, end to end: a rebinding Host, a page's text/plain
-    # form and a foreign Origin are refused before any seat is called.
-    for headers, code in (({"Host": "evil.example"}, 403), ({"Content-Type": "text/plain"}, 415),
-                          ({"Origin": "http://evil.example"}, 403)):
-        try:
-            post({}, **headers)
-            raise AssertionError(f"{headers} was served")
-        except urllib.error.HTTPError as exc:
-            assert exc.code == code, (headers, exc.code)
-
-    name = next(iter(gateway.bench_collections.load()))
-    msg = {"model": f"bench/{name}", "messages": [{"role": "user", "content": "say hi"}]}
-    r = json.loads(post(msg))
-    b = r["bench"]
-    assert b["collection"] == f"bench/{name}" and b["seats"], b
-    assert all(s["answer"] == f"answer from {s['alias']}" and s["call"].startswith(b["panel"] + ".")
-               for s in b["seats"])
-    assert all(s["call"] in r["choices"][0]["message"]["content"] for s in b["seats"])
-    assert "prompt" not in b and "system" not in b
-    chunks = [json.loads(line[6:]) for line in post({**msg, "stream": True}).splitlines()
-              if line.startswith("data: {")]
-    assert "bench" in chunks[0] and "bench" not in chunks[1], chunks
-    srv.shutdown()
-
-
-if __name__ == "__main__":
-    verdicts()
-    bench_field()
-    print("verdict and bench-field checks pass")
+"""Verdicts and the gateway's `bench` field, against a throwaway log. No model or
+endpoint is called and the real offloads.jsonl and runs/gateway/ are never
+written. The collection comes from the data folder (in CI, tests/fixtures/data).
+
+    python scripts/test_verdict.py
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import sys
+import tempfile
+import threading
+import urllib.request
+from http.server import ThreadingHTTPServer
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path[:0] = [str(ROOT), str(ROOT / "metrics"), str(ROOT / "suite")]
+import gateway  # noqa: E402
+import offload  # noqa: E402
+import run  # noqa: E402
+import serve  # noqa: E402
+
+tmp = pathlib.Path(tempfile.mkdtemp(prefix="bench-verdict-"))
+offload.LOG = tmp / "offloads.jsonl"
+gateway.LOG_DIR = tmp
+
+
+def verdicts() -> None:
+    offload.add("m1", "task", None, "member; ok in 3s", "pX.member1")
+    assert not serve.record_verdict("pX.member1", None, "", [], [])["ok"]        # no pick
+    assert not serve.record_verdict("pX.member1", True, "", "x", [])["ok"]       # claims not a list
+    assert not serve.record_verdict("nope.member1", True, "", [], [])["ok"]      # unknown call
+    assert serve.record_verdict("pX.member1", True, "  found   it ", ["real one", " "], ["wrong one"])["ok"]
+    e = offload.entries()[-1]
+    assert (e["useful"], e["note"], e["real"], e["false"]) == (True, "found it", ["real one"], ["wrong one"]), e
+    n = len(offload.LOG.read_text(encoding="utf-8").splitlines())
+    assert serve.record_verdict("pX.member1", True, "found it", ["real one"], ["wrong one"])["ok"]
+    assert len(offload.LOG.read_text(encoding="utf-8").splitlines()) == n, "the same verdict again wrote a line"
+    # The dashboard form shows note and claims for editing: blank means cleared.
+    assert serve.record_verdict("pX.member1", False, "", [], [])["ok"]
+    e = offload.entries()[-1]
+    assert e["useful"] is False and e["note"] == "" and "real" not in e and "false" not in e, e
+    # The CLI: a blank note or claim list keeps the earlier one.
+    offload.verdict("pX.member1", False, "", ["cli claim"])
+    offload.verdict("pX.member1", True, "")
+    assert offload.entries()[-1]["real"] == ["cli claim"], offload.entries()[-1]
+    assert len(offload.entries()) == 1                                           # superseded by call id
+
+
+def bench_field() -> None:
+    gateway.pick_shape = lambda text: ("general", "test")
+    gateway.ask_seat = lambda alias, system, prompt, messages=None: {
+        "status": "ok", "answer": f"answer from {alias}", "error": "", "seconds": 0.1}
+    # The explorer seats ask every endpoint's /models; None is its own "cannot
+    # be asked, do not filter", so no request leaves the machine.
+    run._served_model_names = lambda cache=None: None
+    srv = ThreadingHTTPServer(("127.0.0.1", 0), gateway.Handler)
+    gateway.PORT = srv.server_address[1]          # the Host check expects the port it runs on
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    url = f"http://127.0.0.1:{srv.server_address[1]}/v1/chat/completions"
+
+    def post(body, **headers):
+        req = urllib.request.Request(url, json.dumps(body).encode(),
+                                     {"Content-Type": "application/json", **headers})
+        with urllib.request.urlopen(req) as r:
+            return r.read().decode()
+
+    # metrics/guard.py, end to end: a rebinding Host, a page's text/plain
+    # form and a foreign Origin are refused before any seat is called.
+    for headers, code in (({"Host": "evil.example"}, 403), ({"Content-Type": "text/plain"}, 415),
+                          ({"Origin": "http://evil.example"}, 403)):
+        try:
+            post({}, **headers)
+            raise AssertionError(f"{headers} was served")
+        except urllib.error.HTTPError as exc:
+            assert exc.code == code, (headers, exc.code)
+
+    name = next(iter(gateway.bench_collections.load()))
+    msg = {"model": f"bench/{name}", "messages": [{"role": "user", "content": "say hi"}]}
+    r = json.loads(post(msg))
+    b = r["bench"]
+    assert b["collection"] == f"bench/{name}" and b["seats"], b
+    assert all(s["answer"] == f"answer from {s['alias']}" and s["call"].startswith(b["panel"] + ".")
+               for s in b["seats"])
+    assert all(s["call"] in r["choices"][0]["message"]["content"] for s in b["seats"])
+    assert "prompt" not in b and "system" not in b
+    chunks = [json.loads(line[6:]) for line in post({**msg, "stream": True}).splitlines()
+              if line.startswith("data: {")]
+    # Seat by seat as each finishes; the bench rides on the last chunk, with the finish_reason.
+    assert "bench" in chunks[-1] and chunks[-1]["choices"][0]["finish_reason"] == "stop", chunks
+    assert not any("bench" in c for c in chunks[:-1]), chunks
+    said = "".join(c["choices"][0]["delta"].get("content", "") for c in chunks)
+    assert all(s["call"] in said for s in chunks[-1]["bench"]["seats"]), said
+    srv.shutdown()
+
+
+if __name__ == "__main__":
+    verdicts()
+    bench_field()
+    print("verdict and bench-field checks pass")