@wendellresearch-ai/benchwarmers 0.6.7 → 0.6.8

package/VERSION

@@ -1 +1 @@
-0.6.7
+0.6.8

package/metrics/decide.py

@@ -19,12 +19,17 @@ model answers a typed question.
 
 WHERE THE CALL GOES
 -------------------
-Each endpoint in `config/endpoints.json` may carry a `decisions` URL. Open
+Each endpoint in `config/endpoints.json` may carry a `decisions` URL. One
+that is not a hub and has none is asked under its own base (`decisions_url`:
+`<base>/decisions`, then `<base>/systemone`, as AMD Token Lab names it). Open
 WebUI has one only through its `/openai/{path}` passthrough, which forwards to
 `{connection base}/decisions` and answers only when the server runs with
 ENABLE_OPENAI_API_PASSTHROUGH=True. That reaches Inception
 (`api.inceptionlabs.ai/v1/decisions`) but never OpenRouter, whose Decisions API
-is `/api/alpha/decisions`, outside its `/api/v1` base.
+is `/api/alpha/decisions`, outside its `/api/v1` base. Nor a connection with
+a prefix in Open WebUI: the passthrough sends the provider the prefixed id
+(`amd.d3-flash`), which it does not know; such a model is onboarded from the
+provider's own endpoint.
 
 A model is called on the endpoint it was picked from when onboarding, recorded
 as `via`: picked from Open WebUI, it goes through Open WebUI; picked from
@@ -226,6 +231,16 @@ def load() -> dict[str, dict]:
         return {}
 
 
+def decisions_url(endpoint: str) -> str:
+    """Where `endpoint`'s Decisions API is: its own `decisions` URL, else `{path}`
+    under its base, which is where a provider that has one serves it (AMD Token
+    Lab: /v1/systemone), so a direct endpoint needs nothing written by hand. ""
+    for a hub without one: its relay is not under its base (Open WebUI's is
+    /openai/{path}, outside /api) and is given by hand."""
+    url = endpoints.load()[endpoint].get("decisions") or ""
+    return url or ("" if endpoints.hub(endpoint) else endpoints.base(endpoint) + "/{path}")
+
+
 def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
          path: str = "decisions", alias: str = "", use: str = "", meta: dict | None = None,
          retry: bool = True) -> dict:
@@ -237,7 +252,7 @@ def call(endpoint: str, served: str, state, questions: dict, timeout: int = 60,
     try:
         # Inside the try, so an endpoint gone from the config is logged too.
         entry = endpoints.load()[endpoint]
-        url = (entry.get("decisions") or "").replace("{path}", path)
+        url = decisions_url(endpoint).replace("{path}", path)
         if not url:
             raise ValueError(f"{endpoint} has no decisions URL in config/endpoints.json")
         headers = {"Content-Type": "application/json", **(entry.get("headers") or {})}
@@ -343,7 +358,7 @@ def error_text(exc: Exception) -> str:
 def probe(endpoint: str, served: str) -> tuple[str | None, str]:
     """(None, path) if `endpoint`'s decisions URL answers a yes/no question for
     `served`, else (why not, ""). Each of PATHS is tried where the URL has `{path}`."""
-    url = endpoints.load()[endpoint].get("decisions")
+    url = decisions_url(endpoint)
     if not url:
         return f"{endpoint} has no decisions URL in config/endpoints.json", ""
     why = []

package/metrics/endpoints.py

@@ -20,7 +20,8 @@ that asks "what is served" asks every endpoint here.
 
 `decisions` (optional) is the full URL of the endpoint's Decisions API, for
 decision models that cannot be reached through chat completions; see
-`metrics/decide.py`. Open WebUI's is its `/openai/{path}` passthrough, which
+`metrics/decide.py`. Without one, an endpoint that is not a hub is asked under
+its own base (`decide.decisions_url`). Open WebUI's is its `/openai/{path}` passthrough, which
 answers only when the server runs with ENABLE_OPENAI_API_PASSTHROUGH=True;
 `{path}` is filled per model (`decisions` for Inception, `systemone` for
 Ollama), because the passthrough appends it to the connection's own base.

package/metrics/test_decide.py

@@ -31,7 +31,9 @@ def jsonl(path: str, rows: list[dict]) -> None:
 write("config/endpoints.json", {"hub": {"base": "http://127.0.0.1:9/api", "key": "hub", "default": True,
                                         "decisions": "http://127.0.0.1:9/openai/{path}"},
                                 "cf": {"base": "http://127.0.0.1:9/ai/v1", "key": "hub",
-                                       "decisions": "http://127.0.0.1:9/ai/run/{model}"}})
+                                       "decisions": "http://127.0.0.1:9/ai/run/{model}"},
+                                "bare": {"base": "http://127.0.0.1:9/v1", "key": "hub", "kind": "provider"},
+                                "barehub": {"base": "http://127.0.0.1:8/api", "key": "hub"}})
 # dm2 first: before any result it is the one asked (the first that takes choice, as before),
 # except per request, where its :free budget is spared while dm1 can answer.
 MODELS = {
@@ -148,6 +150,20 @@ decide.urllib.request.urlopen = cf_urlopen
 assert decide.probe("cf", "@cf/cloudflare/clef-flash") == (None, "decisions")
 assert decide.log_rows()[-1]["answers"]["probe"]["noul"] == 0.98
 decide.LOG.unlink()
+
+
+# An endpoint with no decisions URL is asked under its own base, each path in turn (AMD Token
+# Lab answers only /v1/systemone); a hub without one is not asked: its relay is given by hand.
+def bare_urlopen(request, timeout=0):
+    if request.full_url != "http://127.0.0.1:9/v1/systemone":
+        raise http_error(b'{"error": {"message": "Unknown endpoint"}}', 404)
+    return io.BytesIO(json.dumps({"answers": {"probe": {"type": "noul", "noul": 0.99}}}).encode())
+
+
+decide.urllib.request.urlopen = bare_urlopen
+assert decide.probe("bare", "d3") == (None, "systemone")
+assert decide.probe("barehub", "d3") == ("barehub has no decisions URL in config/endpoints.json", "")
+decide.LOG.unlink()
 decide.urllib.request.urlopen = urlopen
 
 # ---- every call is logged, and an error's body survives the log -------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/onboard.py

@@ -292,8 +292,16 @@ def main() -> None:
                 " models_url .../ai/models/search?per_page=1000) and onboard it from there.\n"
                 if "@cf/" in name and endpoint != "cloudflare" and "{model}" not in
                 (endpoints.load()[endpoint].get("decisions") or "") else
+                # Only a hub gets here: any other endpoint is asked under its own base.
+                "  Give this hub a decisions URL in config/endpoints.json (Open WebUI's is"
+                " http://<host>/openai/{path}), or onboard the model from its provider's own endpoint.\n"
+                if "has no decisions URL" in why else
                 "  Open WebUI relays decisions only with ENABLE_OPENAI_API_PASSTHROUGH=True"
                 " in its environment.\n" if owui and "passthrough" in why.lower() else
+                # The relay sends the id as Open WebUI lists it, connection prefix and all.
+                f"  Open WebUI sends the provider the prefixed id ({name}), which it does not know."
+                " Add the provider as an endpoint of its own and onboard the model from there.\n"
+                if owui and " 404" in why and connection_prefix(live[name]) else
                 "  Its connection has no /decisions under its base. An OpenRouter-hosted decision"
                 " model cannot go through Open WebUI (its Decisions API is outside /api/v1);"
                 " pick it from openrouter (direct) instead.\n" if owui and " 404" in why and " 402" not in why else