@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/logo-dark.svg +49 -0
- package/.github/logo.svg +49 -0
- package/AGENTS.md +4 -1
- package/BENCHMARKS.md +160 -0
- package/GATEWAY.md +59 -12
- package/PRIVACY.md +25 -6
- package/README.md +62 -15
- package/SECURITY.md +20 -9
- package/USAGE.md +87 -14
- package/VERSION +1 -1
- package/cli.py +12 -1
- package/gateway.py +122 -91
- package/mcp_server.py +2 -1
- package/metrics/alerts.py +25 -0
- package/metrics/bench_collections.py +58 -9
- package/metrics/catalog.py +54 -9
- package/metrics/client.py +51 -1
- package/metrics/connect.py +225 -0
- package/metrics/decide.py +690 -27
- package/metrics/discover.py +191 -11
- package/metrics/endpoints.py +4 -6
- package/metrics/guard.py +6 -4
- package/metrics/ledger.py +3 -1
- package/metrics/locked.py +102 -1
- package/metrics/offload.py +36 -19
- package/metrics/owui.py +26 -0
- package/metrics/route.py +19 -4
- package/metrics/scheduler.py +11 -1
- package/metrics/sources.py +15 -2
- package/metrics/test_decide.py +275 -7
- package/metrics/wrong.py +21 -9
- package/package.json +3 -1
- package/scripts/auto_retire.py +96 -21
- package/scripts/onboard.py +44 -3
- package/scripts/test_connect.py +152 -3
- package/scripts/test_first_run.py +4 -2
- package/scripts/test_formats.py +100 -1
- package/scripts/test_serve_edit.py +327 -4
- package/serve.py +1746 -170
- package/suite/README.md +5 -2
- package/suite/run.py +226 -6
- package/suite/test_pick.py +15 -0
- package/suite/test_score.py +82 -1
- package/ui/bench.css +44 -0
- package/ui/bench.js +940 -207
package/metrics/discover.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
"""One list of models worth adding, from FreeLLMAPI
|
|
1
|
+
"""One list of models worth adding, from FreeLLMAPI, models.dev and the providers' own lists, deduped.
|
|
2
2
|
|
|
3
3
|
python metrics/discover.py # summary: free models by reach, new this week
|
|
4
|
+
python metrics/discover.py --refresh # fetch every list now, as Discover's Refresh does
|
|
4
5
|
|
|
5
6
|
The shopping list: Discover's Add (serve.py) puts a model on the bench by one
|
|
6
7
|
of the routes below. A model offered by the same provider in both sources
|
|
@@ -34,7 +35,11 @@ ROUTES
|
|
|
34
35
|
------
|
|
35
36
|
The ways this install can reach a provider, each a row's `routes`:
|
|
36
37
|
openwebui a connection in Open WebUI, known because a bench model on the
|
|
37
|
-
Open WebUI hub (endpoints.find_hub) is served from it
|
|
38
|
+
Open WebUI hub (endpoints.find_hub) is served from it, or from
|
|
39
|
+
the providers' lists (below). A row is that connection's when
|
|
40
|
+
the provider's API host is the connection's, or its id
|
|
41
|
+
(models.dev's or FreeLLMAPI's) is the connection's name or a
|
|
42
|
+
part of its host: FreeLLMAPI's xkiro is api.xkiro.com. Adding
|
|
38
43
|
goes through metrics/owui.py, so the model also appears in the
|
|
39
44
|
chats. None without an Open WebUI hub.
|
|
40
45
|
direct an endpoint of its own in config/endpoints.json whose name or
|
|
@@ -50,9 +55,28 @@ The ways this install can reach a provider, each a row's `routes`:
|
|
|
50
55
|
The same model offered by two providers (OpenRouter and NVIDIA) has a route
|
|
51
56
|
on each; the dashboard's Discover popup lists them together by `norm`.
|
|
52
57
|
|
|
53
|
-
|
|
58
|
+
THE PROVIDERS' OWN LISTS
|
|
59
|
+
------------------------
|
|
60
|
+
The catalogs trail the providers by days, so a model a provider started
|
|
61
|
+
serving this morning is in neither. Open WebUI passes each connection's own
|
|
62
|
+
/models through (owui.lists), whatever its Model IDs list allows, so
|
|
63
|
+
refresh() keeps every connection's in runs/cache/provider-lists.json. A model
|
|
64
|
+
there that neither catalog has for that provider is a row of its own, source
|
|
65
|
+
"provider": its id, and what the list says of its name, context, price and
|
|
66
|
+
abilities (each provider says different things; many say nothing of price,
|
|
67
|
+
which is then unknown, not paid). Only through Open WebUI: the providers'
|
|
68
|
+
keys stay where they are, and nothing is sent anywhere new.
|
|
69
|
+
|
|
70
|
+
LINKS
|
|
71
|
+
-----
|
|
72
|
+
A row's `page` is its freellmapi.co page (where the sitemap has one), and
|
|
73
|
+
`md_page` its models.dev page: /models/<lab>/<id> by models.dev's canonical
|
|
74
|
+
id, else the provider's own page there.
|
|
75
|
+
|
|
76
|
+
NEW is the date a model was first seen in any list, kept in
|
|
54
77
|
runs/discover-seen.json (gitignored). The first build records everything as a
|
|
55
|
-
baseline, so only models that appear afterwards are badged new
|
|
78
|
+
baseline, so only models that appear afterwards are badged new; so does the
|
|
79
|
+
first list of each connection.
|
|
56
80
|
"""
|
|
57
81
|
|
|
58
82
|
from __future__ import annotations
|
|
@@ -61,6 +85,7 @@ import datetime as dt
|
|
|
61
85
|
import json
|
|
62
86
|
import pathlib
|
|
63
87
|
import sys
|
|
88
|
+
from urllib.parse import urlsplit
|
|
64
89
|
|
|
65
90
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
66
91
|
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
@@ -74,6 +99,84 @@ FREELLMAPI_TO_CONNECTION = {"groq": "groq", "nvidia": "nvidia-api", "mistral": "
|
|
|
74
99
|
"openrouter": "openrouter", "ollama": "ollama-cloud",
|
|
75
100
|
"cerebras": "cerebras", "llm7": "llm7-api"}
|
|
76
101
|
NEW_DAYS = 7
|
|
102
|
+
LISTS = "provider-lists" # runs/cache/provider-lists.json, written by refresh()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def refresh(force: bool = False) -> list[str]:
|
|
106
|
+
"""Fetch the catalogs (sources.FEEDS) and the providers' own lists, each at
|
|
107
|
+
most once a day unless `force`. What went wrong, as sentences: a list that
|
|
108
|
+
could not be fetched keeps its cached copy."""
|
|
109
|
+
import endpoints
|
|
110
|
+
said = []
|
|
111
|
+
for name in sources.FEEDS:
|
|
112
|
+
try:
|
|
113
|
+
sources.fetch(name, force)
|
|
114
|
+
except Exception as exc: # noqa: BLE001
|
|
115
|
+
said.append(f"could not fetch {name} ({exc}); kept the cached copy")
|
|
116
|
+
age = sources.age_s(LISTS)
|
|
117
|
+
if not endpoints.find_hub("openwebui") or (not force and age is not None and age < sources.MAX_AGE_S):
|
|
118
|
+
return said
|
|
119
|
+
import owui
|
|
120
|
+
try:
|
|
121
|
+
got, failed = owui.lists()
|
|
122
|
+
except Exception as exc: # noqa: BLE001
|
|
123
|
+
return said + [f"could not ask Open WebUI for its providers' lists ({exc}); kept the cached copy"]
|
|
124
|
+
old = {c["names"][0]: c for c in sources.cached(LISTS).get("connections") or []}
|
|
125
|
+
conns = [{**c, "models": [s for s in map(_slim, c["models"]) if s]} for c in got]
|
|
126
|
+
sources.store(LISTS, {"at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
127
|
+
"connections": conns + [old[n] for n in failed if n in old]})
|
|
128
|
+
return said + [f"{n} would not list its models ({why})" + ("; kept its last list" if n in old else "")
|
|
129
|
+
for n, why in failed.items()]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _slim(m: dict) -> dict | None:
|
|
133
|
+
"""What Discover keeps of one model in a provider's own list; None when the
|
|
134
|
+
list says it is not a chat model (images, audio, embeddings, another API).
|
|
135
|
+
Each provider says it its own way: these are the ways seen 2026-11-07 in
|
|
136
|
+
OpenRouter's, Requesty's, xKiro's, Mistral's and UnoRouter's lists."""
|
|
137
|
+
arch = m.get("architecture") if isinstance(m.get("architecture"), dict) else {}
|
|
138
|
+
caps = m.get("capabilities") if isinstance(m.get("capabilities"), dict) else {}
|
|
139
|
+
outs = m.get("output_modalities") or arch.get("output_modalities")
|
|
140
|
+
ins = m.get("input_modalities") or arch.get("input_modalities")
|
|
141
|
+
apis = m.get("supported_endpoint_types")
|
|
142
|
+
if ((isinstance(outs, list) and outs and "text" not in outs) or m.get("modality") not in (None, "chat", "text")
|
|
143
|
+
or m.get("api") not in (None, "chat") or caps.get("completion_chat") is False
|
|
144
|
+
or (isinstance(apis, list) and apis and "openai" not in apis)):
|
|
145
|
+
return None
|
|
146
|
+
tier, price = m.get("access_tier"), m.get("pricing") if isinstance(m.get("pricing"), dict) else {}
|
|
147
|
+
nums = []
|
|
148
|
+
for v in (price.get("prompt"), price.get("completion"), price.get("input"), price.get("output"),
|
|
149
|
+
m.get("input_price"), m.get("output_price")):
|
|
150
|
+
try:
|
|
151
|
+
nums.append(float(v))
|
|
152
|
+
except (TypeError, ValueError):
|
|
153
|
+
pass
|
|
154
|
+
vision = caps.get("vision", m.get("supports_vision"))
|
|
155
|
+
if vision is None and isinstance(ins, list):
|
|
156
|
+
vision = "image" in ins
|
|
157
|
+
ctx = m.get("context_length") or m.get("context_window") or m.get("max_context_length")
|
|
158
|
+
return {"id": str(m["id"]), "name": m.get("display_name") or m.get("name"),
|
|
159
|
+
"context": ctx if isinstance(ctx, int) and ctx > 0 else None,
|
|
160
|
+
"free": (True if str(m["id"]).endswith(":free") or tier == "free" else
|
|
161
|
+
False if tier in ("paid", "premium") else all(x == 0 for x in nums) if nums else None),
|
|
162
|
+
"tools": caps.get("tools", caps.get("function_calling", m.get("supports_tool_calling"))),
|
|
163
|
+
"vision": vision, "reasoning": caps.get("reasoning", m.get("supports_reasoning"))}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _host(base: str) -> str:
|
|
167
|
+
return urlsplit(base).hostname or "" if "//" in base else base
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _conn_for(ids: list[str], api: str | None, hosts: dict[str, str]) -> str | None:
|
|
171
|
+
"""The Open WebUI connection (`hosts`: name -> its API host, the bench's
|
|
172
|
+
names first) that serves this provider: the provider's API host is its
|
|
173
|
+
host, or one of the provider's `ids` is its name or a part of its host."""
|
|
174
|
+
want = _host(api or "")
|
|
175
|
+
for name, host in hosts.items():
|
|
176
|
+
parts = host.split(".")[:-1] if "." in host else [host]
|
|
177
|
+
if (want and host == want) or any(i and (i == name or i in parts) for i in ids):
|
|
178
|
+
return name
|
|
179
|
+
return None
|
|
77
180
|
|
|
78
181
|
|
|
79
182
|
def _on_bench() -> list[tuple[str, str, str, str, str]]:
|
|
@@ -168,14 +271,29 @@ def build() -> list[dict]:
|
|
|
168
271
|
platform_names = {p.get("id"): p.get("name") for p in fl.get("platforms") or []}
|
|
169
272
|
md_conn = {v: k for k, v in sources.connection_to_modelsdev().items()}
|
|
170
273
|
bench = _on_bench()
|
|
171
|
-
# Open WebUI's connections, as far as the bench can tell without asking it
|
|
274
|
+
# Open WebUI's connections, as far as the bench can tell without asking it:
|
|
275
|
+
# its models' connections, then the providers' lists, with their API hosts.
|
|
172
276
|
owui = endpoints.find_hub("openwebui")
|
|
173
|
-
|
|
277
|
+
lists = (sources.cached(LISTS).get("connections") or []) if owui else []
|
|
278
|
+
hosts: dict[str, str] = dict.fromkeys(c for ep, c, *_ in bench if ep == owui and c != "?")
|
|
279
|
+
for c in lists:
|
|
280
|
+
for n in c["names"]:
|
|
281
|
+
hosts[n] = _host(c["base"])
|
|
282
|
+
hosts = {n: h or n for n, h in hosts.items()}
|
|
283
|
+
owui_conns = set(hosts)
|
|
284
|
+
|
|
285
|
+
def conn_of(ids: list[str], api: str | None, known: str | None) -> str | None:
|
|
286
|
+
return known if known in hosts else _conn_for(ids, api, hosts) or known
|
|
287
|
+
|
|
174
288
|
direct = _direct(md)
|
|
175
289
|
rows: dict[tuple[str, str], dict] = {}
|
|
290
|
+
# models.dev's own page for a model: by its canonical id, which names its lab.
|
|
291
|
+
labs = {str(m["canonical_model_id"]).split("/")[0] for p in md.values()
|
|
292
|
+
for m in (p.get("models") or {}).values() if m.get("canonical_model_id")}
|
|
176
293
|
|
|
177
294
|
for prov, p in md.items():
|
|
178
295
|
openai_like = bool(p.get("api")) or p.get("npm") == "@ai-sdk/openai-compatible"
|
|
296
|
+
conn = conn_of([prov], p.get("api"), md_conn.get(prov))
|
|
179
297
|
for mid, m in (p.get("models") or {}).items():
|
|
180
298
|
out_mods = (m.get("modalities") or {}).get("output") or ["text"]
|
|
181
299
|
if "text" not in out_mods or m.get("status") == "deprecated":
|
|
@@ -193,8 +311,10 @@ def build() -> list[dict]:
|
|
|
193
311
|
"vision": "image" in ((m.get("modalities") or {}).get("input") or []),
|
|
194
312
|
"released": m.get("release_date"), "knowledge": m.get("knowledge"),
|
|
195
313
|
"open_weights": m.get("open_weights"), "api": p.get("api"),
|
|
196
|
-
"openai_like": openai_like, "connection":
|
|
197
|
-
"rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None
|
|
314
|
+
"openai_like": openai_like, "connection": conn, "limits": None,
|
|
315
|
+
"rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None,
|
|
316
|
+
"md_page": "https://models.dev/" + (f"models/{m['canonical_model_id']}" if m.get("canonical_model_id")
|
|
317
|
+
else f"models/{prov}/{mid}" if prov in labs else f"providers/{prov}")}
|
|
198
318
|
|
|
199
319
|
for m in fl.get("models") or []:
|
|
200
320
|
if (m.get("modality") or "chat") != "chat":
|
|
@@ -222,8 +342,30 @@ def build() -> list[dict]:
|
|
|
222
342
|
"cost": None, "context": m.get("contextWindow"), "output": None,
|
|
223
343
|
"tools": m.get("supportsTools"), "reasoning": None,
|
|
224
344
|
"vision": m.get("supportsVision"), "released": None, "knowledge": None,
|
|
225
|
-
"open_weights": None, "api": None,
|
|
226
|
-
"openai_like": False, "connection": FREELLMAPI_TO_CONNECTION.get(plat)}
|
|
345
|
+
"open_weights": None, "api": None, "md_page": None,
|
|
346
|
+
"openai_like": False, "connection": conn_of([prov, plat], None, FREELLMAPI_TO_CONNECTION.get(plat))}
|
|
347
|
+
|
|
348
|
+
# What each connection's provider lists that neither catalog has for it.
|
|
349
|
+
listed = set()
|
|
350
|
+
for c in lists:
|
|
351
|
+
conn = c["names"][0] # the bench's name for it, as conn_of picks
|
|
352
|
+
provs = sorted({r["provider"] for r in rows.values() if r["connection"] in c["names"]}) or [conn]
|
|
353
|
+
have = {k[1] for k in rows if k[0] in provs}
|
|
354
|
+
name = next((r["provider_name"] for r in rows.values() if r["provider"] == provs[0]), c["names"][-1])
|
|
355
|
+
listed.add(conn)
|
|
356
|
+
for m in c["models"]:
|
|
357
|
+
key = (provs[0], sources.norm_model(m["id"]))
|
|
358
|
+
if key[1] in have:
|
|
359
|
+
continue
|
|
360
|
+
have.add(key[1])
|
|
361
|
+
rows[key] = {
|
|
362
|
+
"in_mods": ["text", "image"] if m.get("vision") else ["text"], "out_mods": ["text"],
|
|
363
|
+
"provider": provs[0], "provider_name": name, "model": m["id"], "name": m.get("name") or m["id"],
|
|
364
|
+
"sources": ["provider"], "free": m.get("free"), "cost": None, "context": m.get("context"),
|
|
365
|
+
"output": None, "tools": m.get("tools"), "reasoning": m.get("reasoning"), "vision": m.get("vision"),
|
|
366
|
+
"released": None, "knowledge": None, "open_weights": None, "api": c["base"], "openai_like": True,
|
|
367
|
+
"connection": conn, "limits": None, "rank": None, "speed": None, "size": None, "budget": None,
|
|
368
|
+
"quirks": [], "page": None, "md_page": None}
|
|
227
369
|
|
|
228
370
|
# The same model through ANOTHER provider is fine to add, but worth knowing.
|
|
229
371
|
by_model: dict[str, list[str]] = {}
|
|
@@ -234,12 +376,14 @@ def build() -> list[dict]:
|
|
|
234
376
|
seen = _read_seen()
|
|
235
377
|
today = dt.date.today().isoformat()
|
|
236
378
|
baseline = not seen
|
|
379
|
+
first_list = {c for c in listed if f"list|{c}" not in seen} # a connection's first list is its baseline
|
|
237
380
|
walled = {p for p, on in sources.free_in_own_client_only().items() if on}
|
|
238
381
|
out = []
|
|
239
382
|
for key, r in rows.items():
|
|
240
383
|
k = "|".join(key)
|
|
241
384
|
if k not in seen:
|
|
242
|
-
seen[k] = "baseline" if baseline
|
|
385
|
+
seen[k] = ("baseline" if baseline or (r["sources"] == ["provider"] and r["connection"] in first_list)
|
|
386
|
+
else today)
|
|
243
387
|
r["first_seen"] = seen[k]
|
|
244
388
|
r["new"] = seen[k] not in ("baseline",) and seen[k] >= (dt.date.today() - dt.timedelta(days=NEW_DAYS)).isoformat()
|
|
245
389
|
if r["free"] and r["provider"] in walled:
|
|
@@ -257,6 +401,7 @@ def build() -> list[dict]:
|
|
|
257
401
|
"connected" if r["routes"] else
|
|
258
402
|
"easy to add" if r["openai_like"] else "other")
|
|
259
403
|
out.append(r)
|
|
404
|
+
seen.update({f"list|{c}": today for c in first_list})
|
|
260
405
|
_write_seen(seen)
|
|
261
406
|
order = {"connected": 0, "easy to add": 1, "on bench": 2, "other": 3}
|
|
262
407
|
# Within each reach, FreeLLMAPI's smartest first; the models it does not rank
|
|
@@ -343,6 +488,38 @@ def _selftest() -> None:
|
|
|
343
488
|
assert big["page"] == "https://freellmapi.co/model/big-one" and big["quirks"] == [
|
|
344
489
|
{"title": "T", "body": "B", "severity": "info"}] and big["vision"], big
|
|
345
490
|
assert [r["model"] for r in rows] == ["acme/tiny", "acme/big", "acme/small"], [r["model"] for r in rows]
|
|
491
|
+
# A connection is found by its API host or by the provider's id in its host; the bench's name wins.
|
|
492
|
+
hosts = {"xk": "api.xkiro.com", "api.xkiro.com": "api.xkiro.com", "router.requesty.ai": "router.requesty.ai",
|
|
493
|
+
"cli-proxy-api": "203.0.113.7"}
|
|
494
|
+
assert _conn_for(["xkiro"], None, hosts) == "xk"
|
|
495
|
+
assert _conn_for(["requesty"], "https://router.requesty.ai/v1", hosts) == "router.requesty.ai"
|
|
496
|
+
assert _conn_for(["com"], None, hosts) is None and _conn_for(["acme"], "https://acme.example/v1", hosts) is None
|
|
497
|
+
# A provider's own list: chat models only, price where it says one, unknown where it does not.
|
|
498
|
+
assert _slim({"id": "a", "architecture": {"output_modalities": ["image"]}}) is None
|
|
499
|
+
assert _slim({"id": "b", "supported_endpoint_types": ["aihorde"]}) is None
|
|
500
|
+
assert _slim({"id": "c", "capabilities": {"completion_chat": False}}) is None
|
|
501
|
+
assert _slim({"id": "d:free"})["free"] is True and _slim({"id": "e"})["free"] is None
|
|
502
|
+
assert _slim({"id": "f", "pricing": {"prompt": "0", "completion": "0"}})["free"] is True
|
|
503
|
+
assert _slim({"id": "g", "input_price": 2.5e-07, "output_price": 1e-06})["free"] is False
|
|
504
|
+
assert _slim({"id": "h", "access_tier": "premium", "capabilities": {"vision": True}, "context_length": 9})[
|
|
505
|
+
"free"] is False and _slim({"id": "h", "capabilities": {"vision": True}})["vision"] is True
|
|
506
|
+
# With a list cached: acme's connection is found by its host, the list's new model is a row of its
|
|
507
|
+
# own (the first list of a connection is a baseline, so not new), and models.dev links each model.
|
|
508
|
+
feeds["provider-lists"] = {"connections": [{"names": ["acme-conn"], "base": "https://acme.example/v1",
|
|
509
|
+
"models": [_slim({"id": "acme/big"}), _slim({"id": "acme/fresh"})]}]}
|
|
510
|
+
import endpoints
|
|
511
|
+
try:
|
|
512
|
+
sources.cached = lambda name: feeds.get(name, {})
|
|
513
|
+
globals().update(_on_bench=lambda: [], SEEN=pathlib.Path(tempfile.mkdtemp()) / "seen.json")
|
|
514
|
+
endpoints_hub, endpoints.find_hub = endpoints.find_hub, lambda which: "owui-ep" if which == "openwebui" else None
|
|
515
|
+
rows = {r["model"]: r for r in build() if r["provider"] == "acme"}
|
|
516
|
+
finally:
|
|
517
|
+
sources.cached, endpoints.find_hub = kept[0], endpoints_hub
|
|
518
|
+
globals().update(_on_bench=kept[1], SEEN=kept[2])
|
|
519
|
+
assert set(rows) == {"acme/big", "acme/small", "acme/tiny", "acme/fresh"}, sorted(rows)
|
|
520
|
+
assert rows["acme/big"]["connection"] == "acme-conn" and rows["acme/big"]["reach"] == "connected", rows["acme/big"]
|
|
521
|
+
assert rows["acme/fresh"]["sources"] == ["provider"] and not rows["acme/fresh"]["new"], rows["acme/fresh"]
|
|
522
|
+
assert rows["acme/big"]["md_page"] == "https://models.dev/providers/acme", rows["acme/big"]["md_page"]
|
|
346
523
|
print("discover selftest ok")
|
|
347
524
|
|
|
348
525
|
|
|
@@ -351,6 +528,9 @@ def main() -> None:
|
|
|
351
528
|
if "--selftest" in sys.argv:
|
|
352
529
|
_selftest()
|
|
353
530
|
return
|
|
531
|
+
if "--refresh" in sys.argv:
|
|
532
|
+
for line in refresh(force=True) or ["every list fetched"]:
|
|
533
|
+
print(" ", line)
|
|
354
534
|
rows = build()
|
|
355
535
|
free = [r for r in rows if r["free"]]
|
|
356
536
|
by = {}
|
package/metrics/endpoints.py
CHANGED
|
@@ -131,8 +131,9 @@ def load() -> dict[str, dict]:
|
|
|
131
131
|
else's machine is a timeout with no explanation. Whatever needs an
|
|
132
132
|
endpoint asks default() (base, key, models and yaml_block all do), and
|
|
133
133
|
that refuses, so a job run before any endpoint exists says what to do."""
|
|
134
|
+
import locked
|
|
134
135
|
try:
|
|
135
|
-
data = json.loads(
|
|
136
|
+
data = json.loads(locked.read_text(FILE)) # inside a render, from its snapshot
|
|
136
137
|
except FileNotFoundError:
|
|
137
138
|
return {}
|
|
138
139
|
except (OSError, ValueError) as exc:
|
|
@@ -147,11 +148,8 @@ def connections() -> dict:
|
|
|
147
148
|
ties, names for bare-IP hosts, models.dev ids, providers whose free tier
|
|
148
149
|
works only in their own client. {} when absent: a fresh
|
|
149
150
|
install has no such facts, and none are assumed."""
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
except (OSError, ValueError):
|
|
153
|
-
return {}
|
|
154
|
-
return data if isinstance(data, dict) else {}
|
|
151
|
+
import locked
|
|
152
|
+
return locked.read_json(CONNECTIONS) # inside a render, from its snapshot
|
|
155
153
|
|
|
156
154
|
|
|
157
155
|
_default: tuple = (None, "")
|
package/metrics/guard.py
CHANGED
|
@@ -146,9 +146,11 @@ def cors_origin(headers, port: int) -> str:
|
|
|
146
146
|
return origin if origin.lower() in origins(port) else ""
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
def check(headers, port: int, post: bool, max_body: int, cors: bool = False
|
|
149
|
+
def check(headers, port: int, post: bool, max_body: int, cors: bool = False,
|
|
150
|
+
kind: str = "application/json") -> int:
|
|
150
151
|
"""Raise Refused, or return how many body bytes to read (0 for a GET).
|
|
151
|
-
`cors`: a gateway route, where an allowed origin may be cross-site.
|
|
152
|
+
`cors`: a gateway route, where an allowed origin may be cross-site.
|
|
153
|
+
`kind`: the body's type; never one a plain form can send (an import's zip)."""
|
|
152
154
|
host = (headers.get("Host") or "").strip().lower()
|
|
153
155
|
if host not in hosts(port):
|
|
154
156
|
raise Refused(403, f"unknown Host {host or '(none)'!r}; set BENCHWARMERS_PUBLIC_URL"
|
|
@@ -161,8 +163,8 @@ def check(headers, port: int, post: bool, max_body: int, cors: bool = False) ->
|
|
|
161
163
|
raise Refused(403, "cross-site requests are refused")
|
|
162
164
|
if origin and origin not in allowed:
|
|
163
165
|
raise Refused(403, f"Origin {origin!r} is not allowed; set BENCH_ALLOWED_ORIGINS to add one")
|
|
164
|
-
if not (headers.get("Content-Type") or "").lower().startswith(
|
|
165
|
-
raise Refused(415, "send Content-Type:
|
|
166
|
+
if not (headers.get("Content-Type") or "").lower().startswith(kind):
|
|
167
|
+
raise Refused(415, f"send Content-Type: {kind}")
|
|
166
168
|
try:
|
|
167
169
|
length = int(headers.get("Content-Length") or 0)
|
|
168
170
|
except ValueError:
|
package/metrics/ledger.py
CHANGED
|
@@ -17,7 +17,9 @@ are what a provider's request limits count. `status` is ok, empty, error,
|
|
|
17
17
|
timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
|
|
18
18
|
provider's own figures from the reply's usage, absent when it gave none.
|
|
19
19
|
`caller` is the script that made the call (gateway, perf/run, suite/run, ...)
|
|
20
|
-
and `purpose` what the caller said it was for, when it said.
|
|
20
|
+
and `purpose` what the caller said it was for, when it said. A failed call
|
|
21
|
+
(status error) carries its `error`, start and end (client.clip), from
|
|
22
|
+
2026-11-08: the uptime report classifies it (suite/run.uptime_class).
|
|
21
23
|
|
|
22
24
|
A FILE A MONTH
|
|
23
25
|
--------------
|
package/metrics/locked.py
CHANGED
|
@@ -26,6 +26,7 @@ import json
|
|
|
26
26
|
import os
|
|
27
27
|
import pathlib
|
|
28
28
|
import tempfile
|
|
29
|
+
import threading
|
|
29
30
|
import time
|
|
30
31
|
|
|
31
32
|
if os.name == "nt":
|
|
@@ -71,6 +72,7 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
|
|
|
71
72
|
writers never share it (a fixed `.part` name let one writer move another's
|
|
72
73
|
half-written file into place)."""
|
|
73
74
|
path = pathlib.Path(path)
|
|
75
|
+
_forget(path)
|
|
74
76
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
75
77
|
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
|
|
76
78
|
try:
|
|
@@ -90,10 +92,85 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
|
|
|
90
92
|
os.remove(tmp)
|
|
91
93
|
|
|
92
94
|
|
|
95
|
+
_seen = threading.local()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@contextlib.contextmanager
|
|
99
|
+
def snapshot():
|
|
100
|
+
"""Inside it, this thread reads each file from disk once and reuses the
|
|
101
|
+
text: one dashboard render asked for the same dozen config files 11,000
|
|
102
|
+
times, seconds of opens on Windows. Text, not the parsed object, so a
|
|
103
|
+
caller changing what it parsed changes nothing for the next one. Other
|
|
104
|
+
threads (the gateway, a POST) read the disk as before; a write from this
|
|
105
|
+
thread drops its copy."""
|
|
106
|
+
if getattr(_seen, "texts", None) is not None: # already inside one
|
|
107
|
+
yield
|
|
108
|
+
return
|
|
109
|
+
_seen.texts = {}
|
|
110
|
+
try:
|
|
111
|
+
yield
|
|
112
|
+
finally:
|
|
113
|
+
_seen.texts = None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def once(key, compute):
|
|
117
|
+
"""compute(); inside snapshot(), its first answer for `key` (an OSError as
|
|
118
|
+
first raised). Only for answers no caller can change, such as a string."""
|
|
119
|
+
texts = getattr(_seen, "texts", None)
|
|
120
|
+
if texts is None:
|
|
121
|
+
return compute()
|
|
122
|
+
if key not in texts:
|
|
123
|
+
try:
|
|
124
|
+
texts[key] = compute()
|
|
125
|
+
except OSError as exc:
|
|
126
|
+
texts[key] = exc # missing stays missing, as the same error
|
|
127
|
+
if isinstance(texts[key], OSError):
|
|
128
|
+
raise texts[key]
|
|
129
|
+
return texts[key]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def read_text(path) -> str:
|
|
133
|
+
"""The file's text (OSError if it cannot be read); inside snapshot(), as
|
|
134
|
+
first read."""
|
|
135
|
+
return once(str(path), lambda: pathlib.Path(path).read_text(encoding="utf-8"))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def appended(path, mark: dict) -> tuple[bool, list[str]]:
|
|
139
|
+
"""(start over?, lines) for a log that is only ever appended to: the
|
|
140
|
+
whole lines added since `mark` last saw it. The first time, or when the
|
|
141
|
+
file was replaced or rewritten rather than appended to (shorter, or the
|
|
142
|
+
bytes before where it stopped differ), it is True and every line. A last
|
|
143
|
+
line still missing its newline waits for the next call. `mark` is the
|
|
144
|
+
caller's own dict, kept between calls; hold one lock around this call
|
|
145
|
+
and whatever folds the lines in."""
|
|
146
|
+
path = str(path)
|
|
147
|
+
at, before = (mark.get("at", 0), mark.get("before", b"")) if mark.get("path") == path else (0, b"")
|
|
148
|
+
try:
|
|
149
|
+
with open(path, "rb") as fh:
|
|
150
|
+
size = fh.seek(0, os.SEEK_END)
|
|
151
|
+
if at:
|
|
152
|
+
fh.seek(max(at - len(before), 0))
|
|
153
|
+
if size < at or fh.read(len(before)) != before:
|
|
154
|
+
at, before = 0, b""
|
|
155
|
+
fh.seek(at)
|
|
156
|
+
data = fh.read()
|
|
157
|
+
except OSError:
|
|
158
|
+
mark.clear()
|
|
159
|
+
return True, []
|
|
160
|
+
end = data.rfind(b"\n") + 1
|
|
161
|
+
mark.update(path=path, at=at + end, before=(before + data[:end])[-64:])
|
|
162
|
+
return at == 0, data[:end].decode("utf-8", "replace").splitlines()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _forget(path) -> None:
|
|
166
|
+
if (texts := getattr(_seen, "texts", None)) is not None:
|
|
167
|
+
texts.pop(str(path), None)
|
|
168
|
+
|
|
169
|
+
|
|
93
170
|
def read_json(path: pathlib.Path) -> dict:
|
|
94
171
|
"""The file's object, or {} when it is missing, unreadable or not an object."""
|
|
95
172
|
try:
|
|
96
|
-
data = json.loads(
|
|
173
|
+
data = json.loads(read_text(path))
|
|
97
174
|
except (OSError, ValueError):
|
|
98
175
|
return {}
|
|
99
176
|
return data if isinstance(data, dict) else {}
|
|
@@ -105,6 +182,7 @@ def update(path: pathlib.Path, change, **dump):
|
|
|
105
182
|
json.dumps (indent defaults to 2)."""
|
|
106
183
|
dump.setdefault("indent", 2)
|
|
107
184
|
with lock(path):
|
|
185
|
+
_forget(path) # under the lock, the disk's copy
|
|
108
186
|
data = read_json(path)
|
|
109
187
|
before = json.dumps(data, sort_keys=True)
|
|
110
188
|
result = change(data)
|
|
@@ -135,6 +213,29 @@ def _selftest() -> None:
|
|
|
135
213
|
assert data == {"t": 200, "p": 150}, data
|
|
136
214
|
assert update(f, lambda d: "same") == "same" # no change, no write, result passed back
|
|
137
215
|
assert not [p for p in f.parent.iterdir() if p.suffix == ".part"], "temp file left behind"
|
|
216
|
+
# A snapshot reads each file once; its own writes, and leaving it, read the disk again.
|
|
217
|
+
g = pathlib.Path(tmp) / "g.json"
|
|
218
|
+
with snapshot():
|
|
219
|
+
assert read_json(g) == {} # missing stays missing
|
|
220
|
+
g.write_text('{"a": 1}', encoding="utf-8")
|
|
221
|
+
assert read_json(g) == {}
|
|
222
|
+
write_atomic(g, '{"a": 2}')
|
|
223
|
+
assert read_json(g) == {"a": 2}
|
|
224
|
+
g.write_text('{"a": 3}', encoding="utf-8")
|
|
225
|
+
read_json(g)["a"] = 9 # changing what it parsed changes nothing
|
|
226
|
+
assert read_json(g) == {"a": 2}
|
|
227
|
+
assert read_json(g) == {"a": 3}
|
|
228
|
+
# A log read in pieces: only what was added, a half-written line waits,
|
|
229
|
+
# and a rewrite starts over.
|
|
230
|
+
log, mark = pathlib.Path(tmp) / "l.jsonl", {}
|
|
231
|
+
assert appended(log, mark) == (True, [])
|
|
232
|
+
log.write_bytes(b"a\nb\nc")
|
|
233
|
+
assert appended(log, mark) == (True, ["a", "b"])
|
|
234
|
+
with log.open("ab") as fh:
|
|
235
|
+
fh.write(b"\nd\n")
|
|
236
|
+
assert appended(log, mark) == (False, ["c", "d"]) and appended(log, mark) == (False, [])
|
|
237
|
+
log.write_bytes(b"x\ny\nz\nw\n") # longer, but not what was there
|
|
238
|
+
assert appended(log, mark) == (True, ["x", "y", "z", "w"])
|
|
138
239
|
print("ok")
|
|
139
240
|
|
|
140
241
|
|
package/metrics/offload.py
CHANGED
|
@@ -29,34 +29,51 @@ import collections
|
|
|
29
29
|
import datetime as dt
|
|
30
30
|
import json
|
|
31
31
|
import pathlib
|
|
32
|
+
import threading
|
|
32
33
|
|
|
33
34
|
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
34
35
|
from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
|
|
35
36
|
LOG = DATA / "metrics" / "offloads.jsonl"
|
|
36
37
|
|
|
37
38
|
|
|
39
|
+
_lock = threading.Lock()
|
|
40
|
+
_seen: dict = {"mark": {}, "renames": None, "out": [], "by_call": {}}
|
|
41
|
+
|
|
42
|
+
|
|
38
43
|
def entries() -> list[dict]:
|
|
39
44
|
"""Every offload, oldest first. An entry with a `call` id (written by the
|
|
40
45
|
bench gateway) is superseded by a later one with the same id: the gateway
|
|
41
|
-
records each seat unjudged as it answers, and the verdict comes after.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
46
|
+
records each seat unjudged as it answers, and the verdict comes after.
|
|
47
|
+
|
|
48
|
+
Only the lines added since the last call are read (all of them again
|
|
49
|
+
after a rename): one dashboard render asked seven times, a log of 44,000
|
|
50
|
+
lines took most of a second each, and the gateway adds to it on every
|
|
51
|
+
call. A new list each call; the entries in it are shared, so never change
|
|
52
|
+
one."""
|
|
53
|
+
import aliases
|
|
54
|
+
import locked
|
|
55
|
+
with _lock:
|
|
56
|
+
s = _seen
|
|
57
|
+
if s["renames"] is not aliases.history(): # a rename re-reads the history as a new dict
|
|
58
|
+
s.update(mark={}, renames=aliases.history())
|
|
59
|
+
fresh, lines = locked.appended(LOG, s["mark"])
|
|
60
|
+
if fresh:
|
|
61
|
+
s.update(out=[], by_call={})
|
|
62
|
+
out, by_call = s["out"], s["by_call"]
|
|
63
|
+
for line in lines:
|
|
64
|
+
if line.strip():
|
|
65
|
+
try:
|
|
66
|
+
e = json.loads(line)
|
|
67
|
+
except ValueError:
|
|
68
|
+
continue
|
|
69
|
+
e["model"] = aliases.current(e.get("model", ""), s["renames"]) # a renamed model keeps its record
|
|
70
|
+
if e.get("call") in by_call:
|
|
71
|
+
out[by_call[e["call"]]] = e
|
|
72
|
+
continue
|
|
73
|
+
if e.get("call"):
|
|
74
|
+
by_call[e["call"]] = len(out)
|
|
75
|
+
out.append(e)
|
|
76
|
+
return list(out)
|
|
60
77
|
|
|
61
78
|
|
|
62
79
|
def add(model: str, task: str, useful: bool | None, note: str, call: str | None = None,
|
package/metrics/owui.py
CHANGED
|
@@ -114,6 +114,32 @@ def connections() -> dict[str, tuple[int, str]]:
|
|
|
114
114
|
return out
|
|
115
115
|
|
|
116
116
|
|
|
117
|
+
def lists() -> tuple[list[dict], dict[str, str]]:
|
|
118
|
+
"""What each connection's provider serves now, by its own /models, which
|
|
119
|
+
Open WebUI passes through at /openai/models/<index> whatever the
|
|
120
|
+
connection's Model IDs list allows: [{"names", "base", "models"}], names
|
|
121
|
+
as connections() has them (the bench's first), and {name: why} for the
|
|
122
|
+
connections that would not say. Switched-off connections are left out."""
|
|
123
|
+
names: dict[int, list[str]] = collections.defaultdict(list)
|
|
124
|
+
for name, (idx, _tag) in connections().items():
|
|
125
|
+
names[idx].append(name)
|
|
126
|
+
cfg = _call("/openai/config")
|
|
127
|
+
bases = cfg.get("OPENAI_API_BASE_URLS") or []
|
|
128
|
+
out, failed = [], {}
|
|
129
|
+
for idx, ns in sorted(names.items()):
|
|
130
|
+
if idx >= len(bases) or (cfg.get("OPENAI_API_CONFIGS") or {}).get(str(idx), {}).get("enable") is False:
|
|
131
|
+
continue
|
|
132
|
+
try:
|
|
133
|
+
r = _call(f"/openai/models/{idx}")
|
|
134
|
+
except Exception as exc: # noqa: BLE001
|
|
135
|
+
failed[ns[0]] = str(exc)
|
|
136
|
+
continue
|
|
137
|
+
data = r.get("data") if isinstance(r, dict) else r
|
|
138
|
+
out.append({"names": ns, "base": bases[idx],
|
|
139
|
+
"models": [m for m in data or [] if isinstance(m, dict) and m.get("id")]})
|
|
140
|
+
return out, failed
|
|
141
|
+
|
|
142
|
+
|
|
117
143
|
def plan(connection: str, model_id: str) -> dict:
|
|
118
144
|
"""What adding would change. Raises ValueError when it cannot be done here."""
|
|
119
145
|
release, note = version()
|