@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
- """One list of models worth adding, from FreeLLMAPI and models.dev, deduped.
1
+ """One list of models worth adding, from FreeLLMAPI, models.dev and the providers' own lists, deduped.
2
2
 
3
3
  python metrics/discover.py # summary: free models by reach, new this week
4
+ python metrics/discover.py --refresh # fetch every list now, as Discover's Refresh does
4
5
 
5
6
  The shopping list: Discover's Add (serve.py) puts a model on the bench by one
6
7
  of the routes below. A model offered by the same provider in both sources
@@ -34,7 +35,11 @@ ROUTES
34
35
  ------
35
36
  The ways this install can reach a provider, each a row's `routes`:
36
37
  openwebui a connection in Open WebUI, known because a bench model on the
37
- Open WebUI hub (endpoints.find_hub) is served from it. Adding
38
+ Open WebUI hub (endpoints.find_hub) is served from it, or from
39
+ the providers' lists (below). A row is that connection's when
40
+ the provider's API host is the connection's, or its id
41
+ (models.dev's or FreeLLMAPI's) is the connection's name or a
42
+ part of its host: FreeLLMAPI's xkiro is api.xkiro.com. Adding
38
43
  goes through metrics/owui.py, so the model also appears in the
39
44
  chats. None without an Open WebUI hub.
40
45
  direct an endpoint of its own in config/endpoints.json whose name or
@@ -50,9 +55,28 @@ The ways this install can reach a provider, each a row's `routes`:
50
55
  The same model offered by two providers (OpenRouter and NVIDIA) has a route
51
56
  on each; the dashboard's Discover popup lists them together by `norm`.
52
57
 
53
- NEW is the date a model was first seen in either list, kept in
58
+ THE PROVIDERS' OWN LISTS
59
+ ------------------------
60
+ The catalogs trail the providers by days, so a model a provider started
61
+ serving this morning is in neither. Open WebUI passes each connection's own
62
+ /models through (owui.lists), whatever its Model IDs list allows, so
63
+ refresh() keeps every connection's in runs/cache/provider-lists.json. A model
64
+ there that neither catalog has for that provider is a row of its own, source
65
+ "provider": its id, and what the list says of its name, context, price and
66
+ abilities (each provider says different things; many say nothing of price,
67
+ which is then unknown, not paid). Only through Open WebUI: the providers'
68
+ keys stay where they are, and nothing is sent anywhere new.
69
+
70
+ LINKS
71
+ -----
72
+ A row's `page` is its freellmapi.co page (where the sitemap has one), and
73
+ `md_page` its models.dev page: /models/<lab>/<id> by models.dev's canonical
74
+ id, else the provider's own page there.
75
+
76
+ NEW is the date a model was first seen in any list, kept in
54
77
  runs/discover-seen.json (gitignored). The first build records everything as a
55
- baseline, so only models that appear afterwards are badged new.
78
+ baseline, so only models that appear afterwards are badged new; so does the
79
+ first list of each connection.
56
80
  """
57
81
 
58
82
  from __future__ import annotations
@@ -61,6 +85,7 @@ import datetime as dt
61
85
  import json
62
86
  import pathlib
63
87
  import sys
88
+ from urllib.parse import urlsplit
64
89
 
65
90
  ROOT = pathlib.Path(__file__).resolve().parent.parent
66
91
  from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
@@ -74,6 +99,84 @@ FREELLMAPI_TO_CONNECTION = {"groq": "groq", "nvidia": "nvidia-api", "mistral": "
74
99
  "openrouter": "openrouter", "ollama": "ollama-cloud",
75
100
  "cerebras": "cerebras", "llm7": "llm7-api"}
76
101
  NEW_DAYS = 7
102
+ LISTS = "provider-lists" # runs/cache/provider-lists.json, written by refresh()
103
+
104
+
105
+ def refresh(force: bool = False) -> list[str]:
106
+ """Fetch the catalogs (sources.FEEDS) and the providers' own lists, each at
107
+ most once a day unless `force`. What went wrong, as sentences: a list that
108
+ could not be fetched keeps its cached copy."""
109
+ import endpoints
110
+ said = []
111
+ for name in sources.FEEDS:
112
+ try:
113
+ sources.fetch(name, force)
114
+ except Exception as exc: # noqa: BLE001
115
+ said.append(f"could not fetch {name} ({exc}); kept the cached copy")
116
+ age = sources.age_s(LISTS)
117
+ if not endpoints.find_hub("openwebui") or (not force and age is not None and age < sources.MAX_AGE_S):
118
+ return said
119
+ import owui
120
+ try:
121
+ got, failed = owui.lists()
122
+ except Exception as exc: # noqa: BLE001
123
+ return said + [f"could not ask Open WebUI for its providers' lists ({exc}); kept the cached copy"]
124
+ old = {c["names"][0]: c for c in sources.cached(LISTS).get("connections") or []}
125
+ conns = [{**c, "models": [s for s in map(_slim, c["models"]) if s]} for c in got]
126
+ sources.store(LISTS, {"at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
127
+ "connections": conns + [old[n] for n in failed if n in old]})
128
+ return said + [f"{n} would not list its models ({why})" + ("; kept its last list" if n in old else "")
129
+ for n, why in failed.items()]
130
+
131
+
132
+ def _slim(m: dict) -> dict | None:
133
+ """What Discover keeps of one model in a provider's own list; None when the
134
+ list says it is not a chat model (images, audio, embeddings, another API).
135
+ Each provider says it its own way: these are the ways seen 2026-11-07 in
136
+ OpenRouter's, Requesty's, xKiro's, Mistral's and UnoRouter's lists."""
137
+ arch = m.get("architecture") if isinstance(m.get("architecture"), dict) else {}
138
+ caps = m.get("capabilities") if isinstance(m.get("capabilities"), dict) else {}
139
+ outs = m.get("output_modalities") or arch.get("output_modalities")
140
+ ins = m.get("input_modalities") or arch.get("input_modalities")
141
+ apis = m.get("supported_endpoint_types")
142
+ if ((isinstance(outs, list) and outs and "text" not in outs) or m.get("modality") not in (None, "chat", "text")
143
+ or m.get("api") not in (None, "chat") or caps.get("completion_chat") is False
144
+ or (isinstance(apis, list) and apis and "openai" not in apis)):
145
+ return None
146
+ tier, price = m.get("access_tier"), m.get("pricing") if isinstance(m.get("pricing"), dict) else {}
147
+ nums = []
148
+ for v in (price.get("prompt"), price.get("completion"), price.get("input"), price.get("output"),
149
+ m.get("input_price"), m.get("output_price")):
150
+ try:
151
+ nums.append(float(v))
152
+ except (TypeError, ValueError):
153
+ pass
154
+ vision = caps.get("vision", m.get("supports_vision"))
155
+ if vision is None and isinstance(ins, list):
156
+ vision = "image" in ins
157
+ ctx = m.get("context_length") or m.get("context_window") or m.get("max_context_length")
158
+ return {"id": str(m["id"]), "name": m.get("display_name") or m.get("name"),
159
+ "context": ctx if isinstance(ctx, int) and ctx > 0 else None,
160
+ "free": (True if str(m["id"]).endswith(":free") or tier == "free" else
161
+ False if tier in ("paid", "premium") else all(x == 0 for x in nums) if nums else None),
162
+ "tools": caps.get("tools", caps.get("function_calling", m.get("supports_tool_calling"))),
163
+ "vision": vision, "reasoning": caps.get("reasoning", m.get("supports_reasoning"))}
164
+
165
+
166
+ def _host(base: str) -> str:
167
+ return urlsplit(base).hostname or "" if "//" in base else base
168
+
169
+
170
+ def _conn_for(ids: list[str], api: str | None, hosts: dict[str, str]) -> str | None:
171
+ """The Open WebUI connection (`hosts`: name -> its API host, the bench's
172
+ names first) that serves this provider: the provider's API host is its
173
+ host, or one of the provider's `ids` is its name or a part of its host."""
174
+ want = _host(api or "")
175
+ for name, host in hosts.items():
176
+ parts = host.split(".")[:-1] if "." in host else [host]
177
+ if (want and host == want) or any(i and (i == name or i in parts) for i in ids):
178
+ return name
179
+ return None
77
180
 
78
181
 
79
182
  def _on_bench() -> list[tuple[str, str, str, str, str]]:
@@ -168,14 +271,29 @@ def build() -> list[dict]:
168
271
  platform_names = {p.get("id"): p.get("name") for p in fl.get("platforms") or []}
169
272
  md_conn = {v: k for k, v in sources.connection_to_modelsdev().items()}
170
273
  bench = _on_bench()
171
- # Open WebUI's connections, as far as the bench can tell without asking it.
274
+ # Open WebUI's connections, as far as the bench can tell without asking it:
275
+ # its models' connections, then the providers' lists, with their API hosts.
172
276
  owui = endpoints.find_hub("openwebui")
173
- owui_conns = {c for ep, c, *_ in bench if ep == owui} - {"?"}
277
+ lists = (sources.cached(LISTS).get("connections") or []) if owui else []
278
+ hosts: dict[str, str] = dict.fromkeys(c for ep, c, *_ in bench if ep == owui and c != "?")
279
+ for c in lists:
280
+ for n in c["names"]:
281
+ hosts[n] = _host(c["base"])
282
+ hosts = {n: h or n for n, h in hosts.items()}
283
+ owui_conns = set(hosts)
284
+
285
+ def conn_of(ids: list[str], api: str | None, known: str | None) -> str | None:
286
+ return known if known in hosts else _conn_for(ids, api, hosts) or known
287
+
174
288
  direct = _direct(md)
175
289
  rows: dict[tuple[str, str], dict] = {}
290
+ # models.dev's own page for a model: by its canonical id, which names its lab.
291
+ labs = {str(m["canonical_model_id"]).split("/")[0] for p in md.values()
292
+ for m in (p.get("models") or {}).values() if m.get("canonical_model_id")}
176
293
 
177
294
  for prov, p in md.items():
178
295
  openai_like = bool(p.get("api")) or p.get("npm") == "@ai-sdk/openai-compatible"
296
+ conn = conn_of([prov], p.get("api"), md_conn.get(prov))
179
297
  for mid, m in (p.get("models") or {}).items():
180
298
  out_mods = (m.get("modalities") or {}).get("output") or ["text"]
181
299
  if "text" not in out_mods or m.get("status") == "deprecated":
@@ -193,8 +311,10 @@ def build() -> list[dict]:
193
311
  "vision": "image" in ((m.get("modalities") or {}).get("input") or []),
194
312
  "released": m.get("release_date"), "knowledge": m.get("knowledge"),
195
313
  "open_weights": m.get("open_weights"), "api": p.get("api"),
196
- "openai_like": openai_like, "connection": md_conn.get(prov), "limits": None,
197
- "rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None}
314
+ "openai_like": openai_like, "connection": conn, "limits": None,
315
+ "rank": None, "speed": None, "size": None, "budget": None, "quirks": [], "page": None,
316
+ "md_page": "https://models.dev/" + (f"models/{m['canonical_model_id']}" if m.get("canonical_model_id")
317
+ else f"models/{prov}/{mid}" if prov in labs else f"providers/{prov}")}
198
318
 
199
319
  for m in fl.get("models") or []:
200
320
  if (m.get("modality") or "chat") != "chat":
@@ -222,8 +342,30 @@ def build() -> list[dict]:
222
342
  "cost": None, "context": m.get("contextWindow"), "output": None,
223
343
  "tools": m.get("supportsTools"), "reasoning": None,
224
344
  "vision": m.get("supportsVision"), "released": None, "knowledge": None,
225
- "open_weights": None, "api": None,
226
- "openai_like": False, "connection": FREELLMAPI_TO_CONNECTION.get(plat)}
345
+ "open_weights": None, "api": None, "md_page": None,
346
+ "openai_like": False, "connection": conn_of([prov, plat], None, FREELLMAPI_TO_CONNECTION.get(plat))}
347
+
348
+ # What each connection's provider lists that neither catalog has for it.
349
+ listed = set()
350
+ for c in lists:
351
+ conn = c["names"][0] # the bench's name for it, as conn_of picks
352
+ provs = sorted({r["provider"] for r in rows.values() if r["connection"] in c["names"]}) or [conn]
353
+ have = {k[1] for k in rows if k[0] in provs}
354
+ name = next((r["provider_name"] for r in rows.values() if r["provider"] == provs[0]), c["names"][-1])
355
+ listed.add(conn)
356
+ for m in c["models"]:
357
+ key = (provs[0], sources.norm_model(m["id"]))
358
+ if key[1] in have:
359
+ continue
360
+ have.add(key[1])
361
+ rows[key] = {
362
+ "in_mods": ["text", "image"] if m.get("vision") else ["text"], "out_mods": ["text"],
363
+ "provider": provs[0], "provider_name": name, "model": m["id"], "name": m.get("name") or m["id"],
364
+ "sources": ["provider"], "free": m.get("free"), "cost": None, "context": m.get("context"),
365
+ "output": None, "tools": m.get("tools"), "reasoning": m.get("reasoning"), "vision": m.get("vision"),
366
+ "released": None, "knowledge": None, "open_weights": None, "api": c["base"], "openai_like": True,
367
+ "connection": conn, "limits": None, "rank": None, "speed": None, "size": None, "budget": None,
368
+ "quirks": [], "page": None, "md_page": None}
227
369
 
228
370
  # The same model through ANOTHER provider is fine to add, but worth knowing.
229
371
  by_model: dict[str, list[str]] = {}
@@ -234,12 +376,14 @@ def build() -> list[dict]:
234
376
  seen = _read_seen()
235
377
  today = dt.date.today().isoformat()
236
378
  baseline = not seen
379
+ first_list = {c for c in listed if f"list|{c}" not in seen} # a connection's first list is its baseline
237
380
  walled = {p for p, on in sources.free_in_own_client_only().items() if on}
238
381
  out = []
239
382
  for key, r in rows.items():
240
383
  k = "|".join(key)
241
384
  if k not in seen:
242
- seen[k] = "baseline" if baseline else today
385
+ seen[k] = ("baseline" if baseline or (r["sources"] == ["provider"] and r["connection"] in first_list)
386
+ else today)
243
387
  r["first_seen"] = seen[k]
244
388
  r["new"] = seen[k] not in ("baseline",) and seen[k] >= (dt.date.today() - dt.timedelta(days=NEW_DAYS)).isoformat()
245
389
  if r["free"] and r["provider"] in walled:
@@ -257,6 +401,7 @@ def build() -> list[dict]:
257
401
  "connected" if r["routes"] else
258
402
  "easy to add" if r["openai_like"] else "other")
259
403
  out.append(r)
404
+ seen.update({f"list|{c}": today for c in first_list})
260
405
  _write_seen(seen)
261
406
  order = {"connected": 0, "easy to add": 1, "on bench": 2, "other": 3}
262
407
  # Within each reach, FreeLLMAPI's smartest first; the models it does not rank
@@ -343,6 +488,38 @@ def _selftest() -> None:
343
488
  assert big["page"] == "https://freellmapi.co/model/big-one" and big["quirks"] == [
344
489
  {"title": "T", "body": "B", "severity": "info"}] and big["vision"], big
345
490
  assert [r["model"] for r in rows] == ["acme/tiny", "acme/big", "acme/small"], [r["model"] for r in rows]
491
+ # A connection is found by its API host or by the provider's id in its host; the bench's name wins.
492
+ hosts = {"xk": "api.xkiro.com", "api.xkiro.com": "api.xkiro.com", "router.requesty.ai": "router.requesty.ai",
493
+ "cli-proxy-api": "203.0.113.7"}
494
+ assert _conn_for(["xkiro"], None, hosts) == "xk"
495
+ assert _conn_for(["requesty"], "https://router.requesty.ai/v1", hosts) == "router.requesty.ai"
496
+ assert _conn_for(["com"], None, hosts) is None and _conn_for(["acme"], "https://acme.example/v1", hosts) is None
497
+ # A provider's own list: chat models only, price where it says one, unknown where it does not.
498
+ assert _slim({"id": "a", "architecture": {"output_modalities": ["image"]}}) is None
499
+ assert _slim({"id": "b", "supported_endpoint_types": ["aihorde"]}) is None
500
+ assert _slim({"id": "c", "capabilities": {"completion_chat": False}}) is None
501
+ assert _slim({"id": "d:free"})["free"] is True and _slim({"id": "e"})["free"] is None
502
+ assert _slim({"id": "f", "pricing": {"prompt": "0", "completion": "0"}})["free"] is True
503
+ assert _slim({"id": "g", "input_price": 2.5e-07, "output_price": 1e-06})["free"] is False
504
+ assert _slim({"id": "h", "access_tier": "premium", "capabilities": {"vision": True}, "context_length": 9})[
505
+ "free"] is False and _slim({"id": "h", "capabilities": {"vision": True}})["vision"] is True
506
+ # With a list cached: acme's connection is found by its host, the list's new model is a row of its
507
+ # own (the first list of a connection is a baseline, so not new), and models.dev links each model.
508
+ feeds["provider-lists"] = {"connections": [{"names": ["acme-conn"], "base": "https://acme.example/v1",
509
+ "models": [_slim({"id": "acme/big"}), _slim({"id": "acme/fresh"})]}]}
510
+ import endpoints
511
+ try:
512
+ sources.cached = lambda name: feeds.get(name, {})
513
+ globals().update(_on_bench=lambda: [], SEEN=pathlib.Path(tempfile.mkdtemp()) / "seen.json")
514
+ endpoints_hub, endpoints.find_hub = endpoints.find_hub, lambda which: "owui-ep" if which == "openwebui" else None
515
+ rows = {r["model"]: r for r in build() if r["provider"] == "acme"}
516
+ finally:
517
+ sources.cached, endpoints.find_hub = kept[0], endpoints_hub
518
+ globals().update(_on_bench=kept[1], SEEN=kept[2])
519
+ assert set(rows) == {"acme/big", "acme/small", "acme/tiny", "acme/fresh"}, sorted(rows)
520
+ assert rows["acme/big"]["connection"] == "acme-conn" and rows["acme/big"]["reach"] == "connected", rows["acme/big"]
521
+ assert rows["acme/fresh"]["sources"] == ["provider"] and not rows["acme/fresh"]["new"], rows["acme/fresh"]
522
+ assert rows["acme/big"]["md_page"] == "https://models.dev/providers/acme", rows["acme/big"]["md_page"]
346
523
  print("discover selftest ok")
347
524
 
348
525
 
@@ -351,6 +528,9 @@ def main() -> None:
351
528
  if "--selftest" in sys.argv:
352
529
  _selftest()
353
530
  return
531
+ if "--refresh" in sys.argv:
532
+ for line in refresh(force=True) or ["every list fetched"]:
533
+ print(" ", line)
354
534
  rows = build()
355
535
  free = [r for r in rows if r["free"]]
356
536
  by = {}
@@ -131,8 +131,9 @@ def load() -> dict[str, dict]:
131
131
  else's machine is a timeout with no explanation. Whatever needs an
132
132
  endpoint asks default() (base, key, models and yaml_block all do), and
133
133
  that refuses, so a job run before any endpoint exists says what to do."""
134
+ import locked
134
135
  try:
135
- data = json.loads(FILE.read_text(encoding="utf-8"))
136
+ data = json.loads(locked.read_text(FILE)) # inside a render, from its snapshot
136
137
  except FileNotFoundError:
137
138
  return {}
138
139
  except (OSError, ValueError) as exc:
@@ -147,11 +148,8 @@ def connections() -> dict:
147
148
  ties, names for bare-IP hosts, models.dev ids, providers whose free tier
148
149
  works only in their own client. {} when absent: a fresh
149
150
  install has no such facts, and none are assumed."""
150
- try:
151
- data = json.loads(CONNECTIONS.read_text(encoding="utf-8"))
152
- except (OSError, ValueError):
153
- return {}
154
- return data if isinstance(data, dict) else {}
151
+ import locked
152
+ return locked.read_json(CONNECTIONS) # inside a render, from its snapshot
155
153
 
156
154
 
157
155
  _default: tuple = (None, "")
package/metrics/guard.py CHANGED
@@ -146,9 +146,11 @@ def cors_origin(headers, port: int) -> str:
146
146
  return origin if origin.lower() in origins(port) else ""
147
147
 
148
148
 
149
- def check(headers, port: int, post: bool, max_body: int, cors: bool = False) -> int:
149
+ def check(headers, port: int, post: bool, max_body: int, cors: bool = False,
150
+ kind: str = "application/json") -> int:
150
151
  """Raise Refused, or return how many body bytes to read (0 for a GET).
151
- `cors`: a gateway route, where an allowed origin may be cross-site."""
152
+ `cors`: a gateway route, where an allowed origin may be cross-site.
153
+ `kind`: the body's type; never one a plain form can send (an import's zip)."""
152
154
  host = (headers.get("Host") or "").strip().lower()
153
155
  if host not in hosts(port):
154
156
  raise Refused(403, f"unknown Host {host or '(none)'!r}; set BENCHWARMERS_PUBLIC_URL"
@@ -161,8 +163,8 @@ def check(headers, port: int, post: bool, max_body: int, cors: bool = False) ->
161
163
  raise Refused(403, "cross-site requests are refused")
162
164
  if origin and origin not in allowed:
163
165
  raise Refused(403, f"Origin {origin!r} is not allowed; set BENCH_ALLOWED_ORIGINS to add one")
164
- if not (headers.get("Content-Type") or "").lower().startswith("application/json"):
165
- raise Refused(415, "send Content-Type: application/json")
166
+ if not (headers.get("Content-Type") or "").lower().startswith(kind):
167
+ raise Refused(415, f"send Content-Type: {kind}")
166
168
  try:
167
169
  length = int(headers.get("Content-Length") or 0)
168
170
  except ValueError:
package/metrics/ledger.py CHANGED
@@ -17,7 +17,9 @@ are what a provider's request limits count. `status` is ok, empty, error,
17
17
  timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
18
18
  provider's own figures from the reply's usage, absent when it gave none.
19
19
  `caller` is the script that made the call (gateway, perf/run, suite/run, ...)
20
- and `purpose` what the caller said it was for, when it said.
20
+ and `purpose` what the caller said it was for, when it said. A failed call
21
+ (status error) carries its `error`, start and end (client.clip), from
22
+ 2026-11-08: the uptime report classifies it (suite/run.uptime_class).
21
23
 
22
24
  A FILE A MONTH
23
25
  --------------
package/metrics/locked.py CHANGED
@@ -26,6 +26,7 @@ import json
26
26
  import os
27
27
  import pathlib
28
28
  import tempfile
29
+ import threading
29
30
  import time
30
31
 
31
32
  if os.name == "nt":
@@ -71,6 +72,7 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
71
72
  writers never share it (a fixed `.part` name let one writer move another's
72
73
  half-written file into place)."""
73
74
  path = pathlib.Path(path)
75
+ _forget(path)
74
76
  path.parent.mkdir(parents=True, exist_ok=True)
75
77
  fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
76
78
  try:
@@ -90,10 +92,85 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
90
92
  os.remove(tmp)
91
93
 
92
94
 
95
+ _seen = threading.local()
96
+
97
+
98
+ @contextlib.contextmanager
99
+ def snapshot():
100
+ """Inside it, this thread reads each file from disk once and reuses the
101
+ text: one dashboard render asked for the same dozen config files 11,000
102
+ times, seconds of opens on Windows. Text, not the parsed object, so a
103
+ caller changing what it parsed changes nothing for the next one. Other
104
+ threads (the gateway, a POST) read the disk as before; a write from this
105
+ thread drops its copy."""
106
+ if getattr(_seen, "texts", None) is not None: # already inside one
107
+ yield
108
+ return
109
+ _seen.texts = {}
110
+ try:
111
+ yield
112
+ finally:
113
+ _seen.texts = None
114
+
115
+
116
+ def once(key, compute):
117
+ """compute(); inside snapshot(), its first answer for `key` (an OSError as
118
+ first raised). Only for answers no caller can change, such as a string."""
119
+ texts = getattr(_seen, "texts", None)
120
+ if texts is None:
121
+ return compute()
122
+ if key not in texts:
123
+ try:
124
+ texts[key] = compute()
125
+ except OSError as exc:
126
+ texts[key] = exc # missing stays missing, as the same error
127
+ if isinstance(texts[key], OSError):
128
+ raise texts[key]
129
+ return texts[key]
130
+
131
+
132
+ def read_text(path) -> str:
133
+ """The file's text (OSError if it cannot be read); inside snapshot(), as
134
+ first read."""
135
+ return once(str(path), lambda: pathlib.Path(path).read_text(encoding="utf-8"))
136
+
137
+
138
+ def appended(path, mark: dict) -> tuple[bool, list[str]]:
139
+ """(start over?, lines) for a log that is only ever appended to: the
140
+ whole lines added since `mark` last saw it. The first time, or when the
141
+ file was replaced or rewritten rather than appended to (shorter, or the
142
+ bytes before where it stopped differ), it is True and every line. A last
143
+ line still missing its newline waits for the next call. `mark` is the
144
+ caller's own dict, kept between calls; hold one lock around this call
145
+ and whatever folds the lines in."""
146
+ path = str(path)
147
+ at, before = (mark.get("at", 0), mark.get("before", b"")) if mark.get("path") == path else (0, b"")
148
+ try:
149
+ with open(path, "rb") as fh:
150
+ size = fh.seek(0, os.SEEK_END)
151
+ if at:
152
+ fh.seek(max(at - len(before), 0))
153
+ if size < at or fh.read(len(before)) != before:
154
+ at, before = 0, b""
155
+ fh.seek(at)
156
+ data = fh.read()
157
+ except OSError:
158
+ mark.clear()
159
+ return True, []
160
+ end = data.rfind(b"\n") + 1
161
+ mark.update(path=path, at=at + end, before=(before + data[:end])[-64:])
162
+ return at == 0, data[:end].decode("utf-8", "replace").splitlines()
163
+
164
+
165
+ def _forget(path) -> None:
166
+ if (texts := getattr(_seen, "texts", None)) is not None:
167
+ texts.pop(str(path), None)
168
+
169
+
93
170
  def read_json(path: pathlib.Path) -> dict:
94
171
  """The file's object, or {} when it is missing, unreadable or not an object."""
95
172
  try:
96
- data = json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
173
+ data = json.loads(read_text(path))
97
174
  except (OSError, ValueError):
98
175
  return {}
99
176
  return data if isinstance(data, dict) else {}
@@ -105,6 +182,7 @@ def update(path: pathlib.Path, change, **dump):
105
182
  json.dumps (indent defaults to 2)."""
106
183
  dump.setdefault("indent", 2)
107
184
  with lock(path):
185
+ _forget(path) # under the lock, the disk's copy
108
186
  data = read_json(path)
109
187
  before = json.dumps(data, sort_keys=True)
110
188
  result = change(data)
@@ -135,6 +213,29 @@ def _selftest() -> None:
135
213
  assert data == {"t": 200, "p": 150}, data
136
214
  assert update(f, lambda d: "same") == "same" # no change, no write, result passed back
137
215
  assert not [p for p in f.parent.iterdir() if p.suffix == ".part"], "temp file left behind"
216
+ # A snapshot reads each file once; its own writes, and leaving it, read the disk again.
217
+ g = pathlib.Path(tmp) / "g.json"
218
+ with snapshot():
219
+ assert read_json(g) == {} # missing stays missing
220
+ g.write_text('{"a": 1}', encoding="utf-8")
221
+ assert read_json(g) == {}
222
+ write_atomic(g, '{"a": 2}')
223
+ assert read_json(g) == {"a": 2}
224
+ g.write_text('{"a": 3}', encoding="utf-8")
225
+ read_json(g)["a"] = 9 # changing what it parsed changes nothing
226
+ assert read_json(g) == {"a": 2}
227
+ assert read_json(g) == {"a": 3}
228
+ # A log read in pieces: only what was added, a half-written line waits,
229
+ # and a rewrite starts over.
230
+ log, mark = pathlib.Path(tmp) / "l.jsonl", {}
231
+ assert appended(log, mark) == (True, [])
232
+ log.write_bytes(b"a\nb\nc")
233
+ assert appended(log, mark) == (True, ["a", "b"])
234
+ with log.open("ab") as fh:
235
+ fh.write(b"\nd\n")
236
+ assert appended(log, mark) == (False, ["c", "d"]) and appended(log, mark) == (False, [])
237
+ log.write_bytes(b"x\ny\nz\nw\n") # longer, but not what was there
238
+ assert appended(log, mark) == (True, ["x", "y", "z", "w"])
138
239
  print("ok")
139
240
 
140
241
 
@@ -29,34 +29,51 @@ import collections
29
29
  import datetime as dt
30
30
  import json
31
31
  import pathlib
32
+ import threading
32
33
 
33
34
  ROOT = pathlib.Path(__file__).resolve().parent.parent
34
35
  from paths import DATA # noqa: E402 -- the install's data folder (metrics/paths.py)
35
36
  LOG = DATA / "metrics" / "offloads.jsonl"
36
37
 
37
38
 
39
+ _lock = threading.Lock()
40
+ _seen: dict = {"mark": {}, "renames": None, "out": [], "by_call": {}}
41
+
42
+
38
43
  def entries() -> list[dict]:
39
44
  """Every offload, oldest first. An entry with a `call` id (written by the
40
45
  bench gateway) is superseded by a later one with the same id: the gateway
41
- records each seat unjudged as it answers, and the verdict comes after."""
42
- if not LOG.exists():
43
- return []
44
- out, by_call = [], {}
45
- from aliases import current
46
- for line in LOG.read_text(encoding="utf-8").splitlines():
47
- if line.strip():
48
- try:
49
- e = json.loads(line)
50
- except ValueError:
51
- continue
52
- e["model"] = current(e.get("model", "")) # a renamed model keeps its record
53
- if e.get("call") in by_call:
54
- out[by_call[e["call"]]] = e
55
- continue
56
- if e.get("call"):
57
- by_call[e["call"]] = len(out)
58
- out.append(e)
59
- return out
46
+ records each seat unjudged as it answers, and the verdict comes after.
47
+
48
+ Only the lines added since the last call are read (all of them again
49
+ after a rename): one dashboard render asked seven times, a log of 44,000
50
+ lines took most of a second each, and the gateway adds to it on every
51
+ call. A new list each call; the entries in it are shared, so never change
52
+ one."""
53
+ import aliases
54
+ import locked
55
+ with _lock:
56
+ s = _seen
57
+ if s["renames"] is not aliases.history(): # a rename re-reads the history as a new dict
58
+ s.update(mark={}, renames=aliases.history())
59
+ fresh, lines = locked.appended(LOG, s["mark"])
60
+ if fresh:
61
+ s.update(out=[], by_call={})
62
+ out, by_call = s["out"], s["by_call"]
63
+ for line in lines:
64
+ if line.strip():
65
+ try:
66
+ e = json.loads(line)
67
+ except ValueError:
68
+ continue
69
+ e["model"] = aliases.current(e.get("model", ""), s["renames"]) # a renamed model keeps its record
70
+ if e.get("call") in by_call:
71
+ out[by_call[e["call"]]] = e
72
+ continue
73
+ if e.get("call"):
74
+ by_call[e["call"]] = len(out)
75
+ out.append(e)
76
+ return list(out)
60
77
 
61
78
 
62
79
  def add(model: str, task: str, useful: bool | None, note: str, call: str | None = None,
package/metrics/owui.py CHANGED
@@ -114,6 +114,32 @@ def connections() -> dict[str, tuple[int, str]]:
114
114
  return out
115
115
 
116
116
 
117
+ def lists() -> tuple[list[dict], dict[str, str]]:
118
+ """What each connection's provider serves now, by its own /models, which
119
+ Open WebUI passes through at /openai/models/<index> whatever the
120
+ connection's Model IDs list allows: [{"names", "base", "models"}], names
121
+ as connections() has them (the bench's first), and {name: why} for the
122
+ connections that would not say. Switched-off connections are left out."""
123
+ names: dict[int, list[str]] = collections.defaultdict(list)
124
+ for name, (idx, _tag) in connections().items():
125
+ names[idx].append(name)
126
+ cfg = _call("/openai/config")
127
+ bases = cfg.get("OPENAI_API_BASE_URLS") or []
128
+ out, failed = [], {}
129
+ for idx, ns in sorted(names.items()):
130
+ if idx >= len(bases) or (cfg.get("OPENAI_API_CONFIGS") or {}).get(str(idx), {}).get("enable") is False:
131
+ continue
132
+ try:
133
+ r = _call(f"/openai/models/{idx}")
134
+ except Exception as exc: # noqa: BLE001
135
+ failed[ns[0]] = str(exc)
136
+ continue
137
+ data = r.get("data") if isinstance(r, dict) else r
138
+ out.append({"names": ns, "base": bases[idx],
139
+ "models": [m for m in data or [] if isinstance(m, dict) and m.get("id")]})
140
+ return out, failed
141
+
142
+
117
143
  def plan(connection: str, model_id: str) -> dict:
118
144
  """What adding would change. Raises ValueError when it cannot be done here."""
119
145
  release, note = version()