@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

package/scripts/provider_tags.py

@@ -1,129 +1,129 @@
-"""Record which connection each model arrives through.
-
-    python scripts/provider_tags.py
-
-Open WebUI fronts several upstreams, and which one a model comes through
-decides things the model itself does not: whether it shares a rate limit with
-another model, whether it is a free tier, and who to ask when it disappears.
-`kimi-k3` and `gemini-3.8-flash-high` are different vendors but the same
-connection, so they cap together.
-
-Writes `config/provider-tags.json` — a plain map of model id to a short tag —
-which the dashboard reads. A file rather than a live call so the dashboard
-stays a view of the repository and works with the network down.
-
-**THE KEYS ARE NEVER READ.** `/openai/config` returns `OPENAI_API_KEYS`
-alongside the base URLs, in plaintext. This takes `OPENAI_API_BASE_URLS` and
-the host name off each URL, and nothing else, so no secret can reach a file
-that is committed. Worth knowing that endpoint behaves that way.
-
-A REFINEMENT, NOT A PREREQUISITE. Only an Open WebUI hub (endpoints.find_hub)
-has connections to name; without one this writes nothing and exits cleanly,
-and every model's connection is the endpoint it is registered on
-(route.connection_of), which needs no file at all.
-"""
-
-from __future__ import annotations
-
-import json
-import pathlib
-import sys
-import urllib.request
-
-REPO = pathlib.Path(__file__).resolve().parent.parent
-sys.path.insert(0, str(REPO / "metrics"))
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-OUT = DATA / "config" / "provider-tags.json"
-import endpoints  # noqa: E402
-TIMEOUT = 30
-
-# Host -> the name a person would use for it. Anything unlisted falls back to
-# its own hostname, which is still more use than an index.
-# Connections on a local network are named by hand, because the API will not
-# say: `hosts` in config/connections.json. The suffix that follows (e.g.
-# `/vendor-a`) comes from the MODEL's own `owned_by` and is worth keeping: one
-# proxy fronts several upstreams, so `my-proxy/vendor-a` and `my-proxy/vendor-b`
-# share a host and a rate limit while being entirely different vendors.
-
-NAMES = {
-    "ollama.com": "ollama-cloud",
-    "openrouter.ai": "openrouter",
-    "api.groq.com": "groq",
-    "api.cerebras.ai": "cerebras",
-    "integrate.api.nvidia.com": "nvidia-api",
-    "api.mistral.ai": "mistral",
-    "api.llm7.io": "llm7-api",
-    "api.openai.com": "openai",
-    "api.anthropic.com": "anthropic",
-}
-
-
-def get(hub: str, path: str) -> dict:
-    """GET off the Open WebUI server's root, where `/openai/config` and
-    `/api/models` both hang, with the hub's own key."""
-    request = urllib.request.Request(
-        endpoints.base(hub).removesuffix("/api") + path,
-        headers={"Authorization": f"Bearer {endpoints.key(hub)}"})
-    with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
-        return json.load(response)
-
-
-def connection_names(hub: str) -> list[str]:
-    """One short name per connection index. URLs only — never the keys."""
-    urls = get(hub, "/openai/config").get("OPENAI_API_BASE_URLS", [])
-    names = []
-    for url in urls:
-        host = url.split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0]
-        if host.replace(".", "").isdigit():
-            # A BARE IP NAMES NOTHING, and guessing from it was worse than
-            # useless: a proxy labelled `lan-proxy` is described by where it
-            # sits rather than what it is, and Open WebUI does not expose
-            # connection names over the API to find that out —
-            # `OPENAI_API_CONFIGS` has a `tags` field and it is empty. So it
-            # is written down in config/connections.json `hosts`.
-            names.append((endpoints.connections().get("hosts") or {}).get(host, "lan-proxy"))
-        else:
-            names.append(NAMES.get(host, host))
-    return names
-
-
-def main() -> None:
-    hub = endpoints.find_hub("openwebui")
-    if not hub:
-        print("no Open WebUI hub: nothing to tag; each model's connection is its endpoint")
-        return
-    names = connection_names(hub)
-    tags = {}
-    for entry in get(hub, "/api/models").get("data", []):
-        model_id = entry.get("id")
-        if not model_id:
-            continue
-        if entry.get("connection_type") == "local":
-            tags[model_id] = "local"
-            continue
-        index = entry.get("urlIdx")
-        tag = names[index] if isinstance(index, int) and index < len(names) else "?"
-        # The upstream owner, when it says something the connection does not.
-        owner = (entry.get("openai") or {}).get("owned_by")
-        if owner and owner not in ("openai", tag) and tag != "?":
-            tag = f"{tag}/{owner}"
-        tags[model_id] = tag
-
-    # Models registered on another endpoint arrive through THAT endpoint,
-    # whatever Open WebUI would have called them.
-    for r in endpoints.registry():
-        if r["endpoint"] not in ("?", hub):
-            tags[r["served"]] = r["endpoint"]
-
-    OUT.write_text(json.dumps(tags, indent=2, sort_keys=True) + "\n",
-                   encoding="utf-8")
-    print(f"wrote {OUT}  ({len(tags)} models)")
-    seen: dict[str, int] = {}
-    for tag in tags.values():
-        seen[tag] = seen.get(tag, 0) + 1
-    for tag, count in sorted(seen.items(), key=lambda kv: -kv[1]):
-        print(f"  {tag:24} {count}")
-
-
-if __name__ == "__main__":
-    main()
+"""Record which connection each model arrives through.
+
+    python scripts/provider_tags.py
+
+Open WebUI fronts several upstreams, and which one a model comes through
+decides things the model itself does not: whether it shares a rate limit with
+another model, whether it is a free tier, and who to ask when it disappears.
+`kimi-k3` and `gemini-3.8-flash-high` are different vendors but the same
+connection, so they cap together.
+
+Writes `config/provider-tags.json` — a plain map of model id to a short tag —
+which the dashboard reads. A file rather than a live call so the dashboard
+stays a view of the repository and works with the network down.
+
+**THE KEYS ARE NEVER READ.** `/openai/config` returns `OPENAI_API_KEYS`
+alongside the base URLs, in plaintext. This takes `OPENAI_API_BASE_URLS` and
+the host name off each URL, and nothing else, so no secret can reach a file
+that is committed. Worth knowing that endpoint behaves that way.
+
+A REFINEMENT, NOT A PREREQUISITE. Only an Open WebUI hub (endpoints.find_hub)
+has connections to name; without one this writes nothing and exits cleanly,
+and every model's connection is the endpoint it is registered on
+(route.connection_of), which needs no file at all.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import sys
+import urllib.request
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO / "metrics"))
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+OUT = DATA / "config" / "provider-tags.json"
+import endpoints  # noqa: E402
+TIMEOUT = 30
+
+# Host -> the name a person would use for it. Anything unlisted falls back to
+# its own hostname, which is still more use than an index.
+# Connections on a local network are named by hand, because the API will not
+# say: `hosts` in config/connections.json. The suffix that follows (e.g.
+# `/vendor-a`) comes from the MODEL's own `owned_by` and is worth keeping: one
+# proxy fronts several upstreams, so `my-proxy/vendor-a` and `my-proxy/vendor-b`
+# share a host and a rate limit while being entirely different vendors.
+
+NAMES = {
+    "ollama.com": "ollama-cloud",
+    "openrouter.ai": "openrouter",
+    "api.groq.com": "groq",
+    "api.cerebras.ai": "cerebras",
+    "integrate.api.nvidia.com": "nvidia-api",
+    "api.mistral.ai": "mistral",
+    "api.llm7.io": "llm7-api",
+    "api.openai.com": "openai",
+    "api.anthropic.com": "anthropic",
+}
+
+
+def get(hub: str, path: str) -> dict:
+    """GET off the Open WebUI server's root, where `/openai/config` and
+    `/api/models` both hang, with the hub's own key."""
+    request = urllib.request.Request(
+        endpoints.base(hub).removesuffix("/api") + path,
+        headers={"Authorization": f"Bearer {endpoints.key(hub)}"})
+    with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
+        return json.load(response)
+
+
+def connection_names(hub: str) -> list[str]:
+    """One short name per connection index. URLs only — never the keys."""
+    urls = get(hub, "/openai/config").get("OPENAI_API_BASE_URLS", [])
+    names = []
+    for url in urls:
+        host = url.split("//", 1)[-1].split("/", 1)[0].split(":", 1)[0]
+        if host.replace(".", "").isdigit():
+            # A BARE IP NAMES NOTHING, and guessing from it was worse than
+            # useless: a proxy labelled `lan-proxy` is described by where it
+            # sits rather than what it is, and Open WebUI does not expose
+            # connection names over the API to find that out —
+            # `OPENAI_API_CONFIGS` has a `tags` field and it is empty. So it
+            # is written down in config/connections.json `hosts`.
+            names.append((endpoints.connections().get("hosts") or {}).get(host, "lan-proxy"))
+        else:
+            names.append(NAMES.get(host, host))
+    return names
+
+
+def main() -> None:
+    hub = endpoints.find_hub("openwebui")
+    if not hub:
+        print("no Open WebUI hub: nothing to tag; each model's connection is its endpoint")
+        return
+    names = connection_names(hub)
+    tags = {}
+    for entry in get(hub, "/api/models").get("data", []):
+        model_id = entry.get("id")
+        if not model_id:
+            continue
+        if entry.get("connection_type") == "local":
+            tags[model_id] = "local"
+            continue
+        index = entry.get("urlIdx")
+        tag = names[index] if isinstance(index, int) and index < len(names) else "?"
+        # The upstream owner, when it says something the connection does not.
+        owner = (entry.get("openai") or {}).get("owned_by")
+        if owner and owner not in ("openai", tag) and tag != "?":
+            tag = f"{tag}/{owner}"
+        tags[model_id] = tag
+
+    # Models registered on another endpoint arrive through THAT endpoint,
+    # whatever Open WebUI would have called them.
+    for r in endpoints.registry():
+        if r["endpoint"] not in ("?", hub):
+            tags[r["served"]] = r["endpoint"]
+
+    OUT.write_text(json.dumps(tags, indent=2, sort_keys=True) + "\n",
+                   encoding="utf-8")
+    print(f"wrote {OUT}  ({len(tags)} models)")
+    seen: dict[str, int] = {}
+    for tag in tags.values():
+        seen[tag] = seen.get(tag, 0) + 1
+    for tag, count in sorted(seen.items(), key=lambda kv: -kv[1]):
+        print(f"  {tag:24} {count}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/reprefix.py

@@ -1,136 +1,136 @@
-"""Repoint a connection's models after Open WebUI gains or changes a prefix.
-
-    python scripts/reprefix.py llm7-api --prefix "llm7."
-    python scripts/reprefix.py llm7-api --prefix "llm7." --apply
-
-Open WebUI can prefix a connection's model ids, which is the fix for two
-connections serving the same name — `codestral-latest` exists on both mistral
-and llm7, and Open WebUI showed only one of them until llm7 got a prefix.
-
-**THE RENAME IS CHEAP BECAUSE NOTHING HERE IS KEYED BY THE SERVED NAME.**
-Every score, every perf row, every offload and every floor result is keyed by
-the ALIAS. So repointing a connection is one line per model in the yaml, and
-all history survives untouched — which is the answer to "will I have to redo
-everything connected to the models". No.
-
-Two things do need care, and this handles both:
-
-1. **The availability snapshot.** It records served names, so after a prefix
-   the old ones simply vanish and the daily check reports every model on the
-   connection as retired. The snapshot is migrated rather than left to raise a
-   page of false alarms.
-2. **Aliases stay put.** It is tempting to rename `minimax-m2.7` to
-   `llm7-minimax-m2.7` for tidiness. Do not: that orphans its perf history and
-   its suite outputs, and the router would treat it as a model it has never
-   seen. The alias is the identity; the served name is just an address.
-
-Dry-run by default. `--apply` writes. Prefixes are an Open WebUI feature, so
-without an Open WebUI hub (endpoints.find_hub) this refuses before reading
-anything.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import pathlib
-import re
-import sys
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-sys.path.insert(0, str(ROOT / "metrics"))
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-YAML = DATA / "config" / "extra-openai-models.yaml"
-SNAPSHOT = DATA / "config" / "served-snapshot.json"
-TAGS = DATA / "config" / "provider-tags.json"
-
-
-def registered() -> list[tuple[str, str]]:
-    """(alias, served_name) in file order."""
-    out, current = [], None
-    for line in YAML.read_text(encoding="utf-8").splitlines():
-        if (m := re.match(r"\s*-?\s*model_id:\s*(.+)", line)):
-            current = m.group(1).strip().strip("\"'")
-        elif (m := re.match(r"\s*model_name:\s*(.+)", line)):
-            out.append((current, m.group(1).strip().strip("\"'")))
-    return out
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("connection", help="tag as shown by provider_tags.py")
-    ap.add_argument("--prefix", required=True,
-                    help='what Open WebUI now prepends, e.g. "llm7."')
-    ap.add_argument("--apply", action="store_true", help="write the changes")
-    args = ap.parse_args()
-    import endpoints
-    hub = endpoints.find_hub("openwebui")
-    if not hub:
-        sys.exit("no endpoint is an Open WebUI hub, and only Open WebUI prefixes connections:"
-                 " nothing to repoint")
-
-    tags = json.loads(TAGS.read_text(encoding="utf-8"))
-    served_now = set(tags)
-
-    changes = []
-    for alias, served in registered():
-        if tags.get(served, "").split("/")[0] == args.connection:
-            continue                               # already correct
-        candidate = args.prefix + served
-        # Only repoint when the PREFIXED name is what the endpoint now serves
-        # and the bare one is not. Anything else is a different problem and
-        # guessing at it would break a working registration.
-        if candidate in served_now and served not in served_now:
-            changes.append((alias, served, candidate))
-
-    if not changes:
-        print(f"Nothing to repoint for {args.connection}.")
-        print("Either the prefix is already applied, or the endpoint does not "
-              "serve the prefixed names.")
-        return
-
-    print(f"{len(changes)} registration(s) to repoint "
-          f"({'applying' if args.apply else 'dry run'}):\n")
-    for alias, old, new in changes:
-        print(f"  {alias:<32} {old}  ->  {new}")
-    print("\nAliases are NOT changed, so every score, perf row, offload and "
-          "floor result\nkeeps pointing at the same model. That is why this "
-          "is cheap.")
-
-    if not args.apply:
-        print("\nRe-run with --apply to write.")
-        return
-
-    text = YAML.read_text(encoding="utf-8")
-    for _alias, old, new in changes:
-        quoted_new = f'"{new}"' if any(c in new for c in ":/") else new
-        text = re.sub(rf"^(  model_name: ){re.escape(old)}\"?$",
-                      rf"\g<1>{quoted_new}", text, count=1, flags=re.M)
-        text = re.sub(rf'^(  model_name: )"{re.escape(old)}"$',
-                      rf"\g<1>{quoted_new}", text, count=1, flags=re.M)
-    YAML.write_text(text, encoding="utf-8")
-
-    # The snapshot records served NAMES, so without this the daily check
-    # reports the whole connection as retired tomorrow morning.
-    if SNAPSHOT.exists():
-        snap = json.loads(SNAPSHOT.read_text(encoding="utf-8"))
-        moved = 0
-        for _alias, old, new in changes:
-            # Keyed as the snapshot keys the hub's models: bare when it is the default.
-            old, new = endpoints.snapshot_key(old, hub), endpoints.snapshot_key(new, hub)
-            if old in snap.get("last_seen", {}):
-                snap["last_seen"][new] = snap["last_seen"].pop(old)
-                moved += 1
-            snap.get("first_missing", {}).pop(old, None)
-        SNAPSHOT.write_text(json.dumps(snap, indent=2, sort_keys=True) + "\n",
-                            encoding="utf-8")
-        print(f"\nmigrated {moved} snapshot entr"
-              f"{'ies' if moved != 1 else 'y'} so the daily check does not "
-              f"report them retired")
-
-    print("\nNow run:  python scripts/sync_config.py && "
-          "python scripts/provider_tags.py")
-
-
-if __name__ == "__main__":
-    main()
+"""Repoint a connection's models after Open WebUI gains or changes a prefix.
+
+    python scripts/reprefix.py llm7-api --prefix "llm7."
+    python scripts/reprefix.py llm7-api --prefix "llm7." --apply
+
+Open WebUI can prefix a connection's model ids, which is the fix for two
+connections serving the same name — `codestral-latest` exists on both mistral
+and llm7, and Open WebUI showed only one of them until llm7 got a prefix.
+
+**THE RENAME IS CHEAP BECAUSE NOTHING HERE IS KEYED BY THE SERVED NAME.**
+Every score, every perf row, every offload and every floor result is keyed by
+the ALIAS. So repointing a connection is one line per model in the yaml, and
+all history survives untouched — which is the answer to "will I have to redo
+everything connected to the models". No.
+
+Two things do need care, and this handles both:
+
+1. **The availability snapshot.** It records served names, so after a prefix
+   the old ones simply vanish and the daily check reports every model on the
+   connection as retired. The snapshot is migrated rather than left to raise a
+   page of false alarms.
+2. **Aliases stay put.** It is tempting to rename `minimax-m2.7` to
+   `llm7-minimax-m2.7` for tidiness. Do not: that orphans its perf history and
+   its suite outputs, and the router would treat it as a model it has never
+   seen. The alias is the identity; the served name is just an address.
+
+Dry-run by default. `--apply` writes. Prefixes are an Open WebUI feature, so
+without an Open WebUI hub (endpoints.find_hub) this refuses before reading
+anything.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "metrics"))
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+YAML = DATA / "config" / "extra-openai-models.yaml"
+SNAPSHOT = DATA / "config" / "served-snapshot.json"
+TAGS = DATA / "config" / "provider-tags.json"
+
+
+def registered() -> list[tuple[str, str]]:
+    """(alias, served_name) in file order."""
+    out, current = [], None
+    for line in YAML.read_text(encoding="utf-8").splitlines():
+        if (m := re.match(r"\s*-?\s*model_id:\s*(.+)", line)):
+            current = m.group(1).strip().strip("\"'")
+        elif (m := re.match(r"\s*model_name:\s*(.+)", line)):
+            out.append((current, m.group(1).strip().strip("\"'")))
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("connection", help="tag as shown by provider_tags.py")
+    ap.add_argument("--prefix", required=True,
+                    help='what Open WebUI now prepends, e.g. "llm7."')
+    ap.add_argument("--apply", action="store_true", help="write the changes")
+    args = ap.parse_args()
+    import endpoints
+    hub = endpoints.find_hub("openwebui")
+    if not hub:
+        sys.exit("no endpoint is an Open WebUI hub, and only Open WebUI prefixes connections:"
+                 " nothing to repoint")
+
+    tags = json.loads(TAGS.read_text(encoding="utf-8"))
+    served_now = set(tags)
+
+    changes = []
+    for alias, served in registered():
+        if tags.get(served, "").split("/")[0] == args.connection:
+            continue                               # already correct
+        candidate = args.prefix + served
+        # Only repoint when the PREFIXED name is what the endpoint now serves
+        # and the bare one is not. Anything else is a different problem and
+        # guessing at it would break a working registration.
+        if candidate in served_now and served not in served_now:
+            changes.append((alias, served, candidate))
+
+    if not changes:
+        print(f"Nothing to repoint for {args.connection}.")
+        print("Either the prefix is already applied, or the endpoint does not "
+              "serve the prefixed names.")
+        return
+
+    print(f"{len(changes)} registration(s) to repoint "
+          f"({'applying' if args.apply else 'dry run'}):\n")
+    for alias, old, new in changes:
+        print(f"  {alias:<32} {old}  ->  {new}")
+    print("\nAliases are NOT changed, so every score, perf row, offload and "
+          "floor result\nkeeps pointing at the same model. That is why this "
+          "is cheap.")
+
+    if not args.apply:
+        print("\nRe-run with --apply to write.")
+        return
+
+    text = YAML.read_text(encoding="utf-8")
+    for _alias, old, new in changes:
+        quoted_new = f'"{new}"' if any(c in new for c in ":/") else new
+        text = re.sub(rf"^(  model_name: ){re.escape(old)}\"?$",
+                      rf"\g<1>{quoted_new}", text, count=1, flags=re.M)
+        text = re.sub(rf'^(  model_name: )"{re.escape(old)}"$',
+                      rf"\g<1>{quoted_new}", text, count=1, flags=re.M)
+    YAML.write_text(text, encoding="utf-8")
+
+    # The snapshot records served NAMES, so without this the daily check
+    # reports the whole connection as retired tomorrow morning.
+    if SNAPSHOT.exists():
+        snap = json.loads(SNAPSHOT.read_text(encoding="utf-8"))
+        moved = 0
+        for _alias, old, new in changes:
+            # Keyed as the snapshot keys the hub's models: bare when it is the default.
+            old, new = endpoints.snapshot_key(old, hub), endpoints.snapshot_key(new, hub)
+            if old in snap.get("last_seen", {}):
+                snap["last_seen"][new] = snap["last_seen"].pop(old)
+                moved += 1
+            snap.get("first_missing", {}).pop(old, None)
+        SNAPSHOT.write_text(json.dumps(snap, indent=2, sort_keys=True) + "\n",
+                            encoding="utf-8")
+        print(f"\nmigrated {moved} snapshot entr"
+              f"{'ies' if moved != 1 else 'y'} so the daily check does not "
+              f"report them retired")
+
+    print("\nNow run:  python scripts/sync_config.py && "
+          "python scripts/provider_tags.py")
+
+
+if __name__ == "__main__":
+    main()