@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/logo-dark.svg +49 -0
- package/.github/logo.svg +49 -0
- package/AGENTS.md +4 -1
- package/BENCHMARKS.md +160 -0
- package/GATEWAY.md +59 -12
- package/PRIVACY.md +25 -6
- package/README.md +62 -15
- package/SECURITY.md +20 -9
- package/USAGE.md +87 -14
- package/VERSION +1 -1
- package/cli.py +12 -1
- package/gateway.py +122 -91
- package/mcp_server.py +2 -1
- package/metrics/alerts.py +25 -0
- package/metrics/bench_collections.py +58 -9
- package/metrics/catalog.py +54 -9
- package/metrics/client.py +51 -1
- package/metrics/connect.py +225 -0
- package/metrics/decide.py +690 -27
- package/metrics/discover.py +191 -11
- package/metrics/endpoints.py +4 -6
- package/metrics/guard.py +6 -4
- package/metrics/ledger.py +3 -1
- package/metrics/locked.py +102 -1
- package/metrics/offload.py +36 -19
- package/metrics/owui.py +26 -0
- package/metrics/route.py +19 -4
- package/metrics/scheduler.py +11 -1
- package/metrics/sources.py +15 -2
- package/metrics/test_decide.py +275 -7
- package/metrics/wrong.py +21 -9
- package/package.json +3 -1
- package/scripts/auto_retire.py +96 -21
- package/scripts/onboard.py +44 -3
- package/scripts/test_connect.py +152 -3
- package/scripts/test_first_run.py +4 -2
- package/scripts/test_formats.py +100 -1
- package/scripts/test_serve_edit.py +327 -4
- package/serve.py +1746 -170
- package/suite/README.md +5 -2
- package/suite/run.py +226 -6
- package/suite/test_pick.py +15 -0
- package/suite/test_score.py +82 -1
- package/ui/bench.css +44 -0
- package/ui/bench.js +940 -207
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
Each rule the popup enforces is tried once and must refuse; then a role change
|
|
6
6
|
and a rename are applied to the copy and checked. The llm sync is not run:
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
with the built-in client it is skipped, and with the llm CLI serve.ROOT points
|
|
8
|
+
at the copy, which has no scripts/sync_config.py, so the sync reports that it
|
|
9
|
+
could not run. Alerts and endpoints point at copies too,
|
|
9
10
|
because removing a model clears its alerts and a regressed endpoint check
|
|
10
11
|
would write the endpoint file.
|
|
11
12
|
"""
|
|
@@ -170,6 +171,50 @@ def main() -> int:
|
|
|
170
171
|
ok = (set(view["served"]) == set(view["models"]) and victim not in view["served"]
|
|
171
172
|
and view["served"].get(other) == d3[other]["served"])
|
|
172
173
|
print(("ok " if ok else "FAIL ") + "collection view: served name for every member choice"); failures += 0 if ok else 1
|
|
174
|
+
# Clients: every snippet parses in its format, names each collection, sets the
|
|
175
|
+
# timeouts a panel needs, and a made key goes only into a shell line or a field.
|
|
176
|
+
import tomllib
|
|
177
|
+
try:
|
|
178
|
+
import yaml # comes with the llm CLI; the built-in client's venv may lack it
|
|
179
|
+
load_yaml = yaml.safe_load
|
|
180
|
+
except ImportError:
|
|
181
|
+
load_yaml = None
|
|
182
|
+
bad = []
|
|
183
|
+
for keys in (False, True):
|
|
184
|
+
setups = {c["id"]: c for c in serve.client_setups("http://127.0.0.1:8122", [("core", 200_000, False),
|
|
185
|
+
("fast", None, True)], keys)}
|
|
186
|
+
parsed = {}
|
|
187
|
+
for c in setups.values():
|
|
188
|
+
for b in c["blocks"]:
|
|
189
|
+
parse = {"json": json.loads, "toml": tomllib.loads, "yaml": load_yaml}.get(b["lang"])
|
|
190
|
+
try:
|
|
191
|
+
parsed[c["id"], b["label"]] = parse(b["text"]) if parse else b["text"]
|
|
192
|
+
except Exception as exc: # noqa: BLE001
|
|
193
|
+
bad.append(f"{c['id']} {b['label']} does not parse: {exc}")
|
|
194
|
+
if b["key"] and b["lang"] not in ("sh", "bat", "text"):
|
|
195
|
+
bad.append(f"{c['id']} {b['label']}: a key would go into a file")
|
|
196
|
+
alltext = "\n".join(b["text"] for b in c["blocks"])
|
|
197
|
+
want = ("bench/core:merged", "bench/fast:merged") if c["id"] == "chat" else \
|
|
198
|
+
("bench/core:route", "bench/fast:route") if c["id"] != "codex" else ("bench/core:route",)
|
|
199
|
+
bad += [f"{c['id']} lacks {w}" for w in want if w not in alltext]
|
|
200
|
+
oc, cx = parsed.get(("opencode", "opencode.json")), parsed.get(("codex", "config.toml"))
|
|
201
|
+
if oc and cx:
|
|
202
|
+
p = oc["provider"]["benchwarmers"]
|
|
203
|
+
if (oc["mcp"]["benchwarmers"]["timeout"] != 900_000 or cx["mcp_servers"]["benchwarmers"]["tool_timeout_sec"] != 900
|
|
204
|
+
or cx["model_providers"]["benchwarmers"]["wire_api"] != "responses" or cx["web_search"] != "disabled"
|
|
205
|
+
or p["models"]["bench/fast:route"]["limit"]["context"] != serve.UNKNOWN_CONTEXT
|
|
206
|
+
or ("apiKey" in p["options"]) != keys or ("env_key" in cx["model_providers"]["benchwarmers"]) != keys):
|
|
207
|
+
bad.append(f"opencode or Codex settings wrong (keys={keys})")
|
|
208
|
+
if parsed.get(("cline", "Cline: cline_mcp_settings.json"), {}).get("mcpServers", {}).get("benchwarmers", {}).get("timeout") != 900:
|
|
209
|
+
bad.append("Cline's MCP timeout is not 900 s")
|
|
210
|
+
if not all(m["toolCalling"] for m in parsed.get(("vscode", "chatLanguageModels.json"), [{}])[0].get("models", [{}])):
|
|
211
|
+
bad.append("a VS Code model without toolCalling is hidden from the agent picker")
|
|
212
|
+
if load_yaml and parsed.get(("goose", "config.yaml"), {}).get("extensions", {}).get("benchwarmers", {}).get("timeout") != 900:
|
|
213
|
+
bad.append("goose's extension timeout is not 900 s")
|
|
214
|
+
ok = not bad
|
|
215
|
+
print(("ok " if ok else "FAIL ") + "client set-ups: JSON, TOML" + (", YAML" if load_yaml else " (YAML unread: no PyYAML)")
|
|
216
|
+
+ " parse, collections named, timeouts set, keys only in shell lines and fields" + ("" if ok else f" -> {bad}"))
|
|
217
|
+
failures += 0 if ok else 1
|
|
173
218
|
# restore: same alias back, out of retired.json, role and ad hoc kept, and
|
|
174
219
|
# back into the collections that still exist (one deleted, one changed since)
|
|
175
220
|
serve.delete_collection("doomed")
|
|
@@ -255,7 +300,7 @@ def main() -> int:
|
|
|
255
300
|
|
|
256
301
|
# Call options: max_tokens set, refused above the model's max output or as
|
|
257
302
|
# a fraction or for no model, then cleared; the llm CLI's copy is stubbed.
|
|
258
|
-
told, real_info = [], serve.catalog.info
|
|
303
|
+
told, real_info, llm_option = [], serve.catalog.info, serve._llm_option
|
|
259
304
|
serve._llm_option = lambda alias, name, value: told.append((alias, name, value)) or "stub"
|
|
260
305
|
serve.catalog.info = lambda a: {**real_info(a), "max_output": 50_000}
|
|
261
306
|
chat = next(a for a, m in sorted(serve.model_directory().items()) if m.get("kind") != "decision")
|
|
@@ -290,6 +335,7 @@ def main() -> int:
|
|
|
290
335
|
import client as model_client
|
|
291
336
|
model_client.CHOICE = cfg / "client.json"
|
|
292
337
|
env, real_which = os.environ.pop("BENCH_CLIENT", None), shutil.which
|
|
338
|
+
ran, real_run = [], serve.subprocess.run
|
|
293
339
|
try:
|
|
294
340
|
shutil.which = lambda name: "/usr/bin/" + name # llm installed: still built-in
|
|
295
341
|
first = model_client.chosen()
|
|
@@ -305,8 +351,14 @@ def main() -> int:
|
|
|
305
351
|
no_llm = serve.save_client("llm")["ok"]
|
|
306
352
|
shutil.which = real_which
|
|
307
353
|
unknown = serve.save_client("curl")["ok"]
|
|
354
|
+
# With the built-in client nothing is mirrored into the llm CLI (5-6 s
|
|
355
|
+
# a save); choosing the llm CLI syncs it once.
|
|
356
|
+
serve.subprocess.run = lambda args, **kw: ran.append(args) or serve.subprocess.CompletedProcess(args, 0, "", "")
|
|
357
|
+
quiet = (serve._sync_cli(), llm_option(chat, "max_tokens", 5), list(ran))
|
|
358
|
+
shutil.which = lambda name: "/usr/bin/" + name
|
|
359
|
+
switched, again = serve.save_client("llm"), serve.save_client("llm")
|
|
308
360
|
finally:
|
|
309
|
-
shutil.which = real_which
|
|
361
|
+
serve.subprocess.run, shutil.which = real_run, real_which
|
|
310
362
|
os.environ.pop("BENCH_CLIENT", None)
|
|
311
363
|
if env is not None:
|
|
312
364
|
os.environ["BENCH_CLIENT"] = env
|
|
@@ -314,6 +366,12 @@ def main() -> int:
|
|
|
314
366
|
and pinned == (("llm", "environment"), False) and not no_llm and not unknown)
|
|
315
367
|
print(("ok " if ok else "FAIL ") + "model client choice" + ("" if ok else f" -> {first} {made} {picked} {pinned}"))
|
|
316
368
|
failures += 0 if ok else 1
|
|
369
|
+
# Saving it again syncs again: the way to retry a failed sync.
|
|
370
|
+
ok = (quiet == (serve.NOT_SYNCED, serve.NOT_SYNCED, []) and switched["sync"] == "synced to the llm CLI"
|
|
371
|
+
and again["sync"] == "synced to the llm CLI" and len(ran) == 2
|
|
372
|
+
and all(str(r[-1]).endswith("sync_config.py") for r in ran))
|
|
373
|
+
print(("ok " if ok else "FAIL ") + "llm CLI synced only when it is the client" + ("" if ok else f" -> {quiet} {switched} {ran}"))
|
|
374
|
+
failures += 0 if ok else 1
|
|
317
375
|
|
|
318
376
|
# Free only in their own app: only what differs from the built-in is written; bad input refused.
|
|
319
377
|
import sources
|
|
@@ -514,6 +572,25 @@ def main() -> int:
|
|
|
514
572
|
print(("ok " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
|
|
515
573
|
failures += 0 if ok else 1
|
|
516
574
|
|
|
575
|
+
# An answer clears every alert it disproves (2026-11-02: big-pickle answered a
|
|
576
|
+
# Retry and stayed listed, its unrecognised-failure alert open), never a price;
|
|
577
|
+
# a held connection loses the model, and goes with its last one.
|
|
578
|
+
alerts.raise_alert("model:unclassified:t/served", "registration", "t fails oddly")
|
|
579
|
+
alerts.raise_alert("model:pending:t/served", "retiring", "t retires", alias="t")
|
|
580
|
+
alerts.raise_alert("model:paid:t/served", "went paid", "t has a price", alias="t")
|
|
581
|
+
alerts.raise_alert("model:pending:t/served-too", "retiring", "another retires", alias="t2")
|
|
582
|
+
alerts.raise_alert("connection:held:x", "connection", "all fail", aliases=["t", "t2"])
|
|
583
|
+
alerts.raise_alert("connection:held:y", "connection", "all fail", aliases=["t"])
|
|
584
|
+
cleared = alerts.answered("t", "t/served")
|
|
585
|
+
left = alerts.active()
|
|
586
|
+
ok = (set(cleared) == {"model:unclassified:t/served", "model:pending:t/served", "connection:held:y"}
|
|
587
|
+
and "model:paid:t/served" in left and "model:pending:t/served-too" in left
|
|
588
|
+
and left["connection:held:x"]["aliases"] == ["t2"])
|
|
589
|
+
for k in ("model:paid:t/served", "model:pending:t/served-too", "connection:held:x"):
|
|
590
|
+
alerts.clear(k)
|
|
591
|
+
print(("ok " if ok else "FAIL ") + "an answer clears what it disproves, not a price" + ("" if ok else f" -> {cleared}"))
|
|
592
|
+
failures += 0 if ok else 1
|
|
593
|
+
|
|
517
594
|
# Got it clears a notice (NVIDIA changed its figures), never a state still wrong.
|
|
518
595
|
alerts.raise_alert("test:notice", "limits", "a notice", dismiss=True)
|
|
519
596
|
alerts.raise_alert("test:state", "limits", "a state")
|
|
@@ -534,6 +611,252 @@ def main() -> int:
|
|
|
534
611
|
if (DATA / "config" / "collections.json").exists() else None))
|
|
535
612
|
print(("ok " if ok else "FAIL ") + "the real alerts, endpoints and collections were not touched")
|
|
536
613
|
failures += 0 if ok else 1
|
|
614
|
+
|
|
615
|
+
# The Logs page reads a log from its end, and still gets whole lines.
|
|
616
|
+
log = cfg / "big.log"
|
|
617
|
+
lines = [f"{i} " + "x" * 2000 for i in range(400)]
|
|
618
|
+
log.write_text("\n".join(lines) + "\n\n", encoding="utf-8")
|
|
619
|
+
ok = (serve._log_rows(log) == lines[::-1][:200] and serve._log_rows(log, 100) == lines[::-1][:100]
|
|
620
|
+
and serve._log_rows(log.with_name("none.log")) == [] and list(serve._from_end(log)) == lines[::-1])
|
|
621
|
+
print(("ok " if ok else "FAIL ") + "a log's last lines, read from its end")
|
|
622
|
+
failures += 0 if ok else 1
|
|
623
|
+
|
|
624
|
+
# Confidently wrong is scored again when a task or a saved answer changes.
|
|
625
|
+
answer = cfg / "tasks" / "t" / "output" / "m.md"
|
|
626
|
+
answer.parent.mkdir(parents=True)
|
|
627
|
+
(cfg / "tasks" / "t" / "task.json").write_text("{}", encoding="utf-8")
|
|
628
|
+
answer.write_text("a", encoding="utf-8")
|
|
629
|
+
before = serve._tasks_key(cfg / "tasks")
|
|
630
|
+
answer.write_text("ab", encoding="utf-8")
|
|
631
|
+
ok = len(before) == 2 and serve._tasks_key(cfg / "tasks") != before and serve._tasks_key(cfg / "none") == ()
|
|
632
|
+
print(("ok " if ok else "FAIL ") + "a changed answer changes the scoring's key")
|
|
633
|
+
failures += 0 if ok else 1
|
|
634
|
+
|
|
635
|
+
# Suite scores and statuses are worked out once a render; what one pane
|
|
636
|
+
# changes in its copy, the next does not see.
|
|
637
|
+
(serve.DATA / "runs" / "suite-2026-01-01.md").write_text("| t1 | `m1` | 1/2 | — | 3.0 |\n", encoding="utf-8")
|
|
638
|
+
with serve.locked.snapshot():
|
|
639
|
+
serve.model_status().clear()
|
|
640
|
+
for v in serve.suite_scores().values():
|
|
641
|
+
v.clear()
|
|
642
|
+
ok = (serve.model_status() == serve._model_status() and serve._suite_scores()
|
|
643
|
+
and serve.suite_scores() == serve._suite_scores())
|
|
644
|
+
print(("ok " if ok else "FAIL ") + "suite scores and statuses once a render, a copy each")
|
|
645
|
+
failures += 0 if ok else 1
|
|
646
|
+
|
|
647
|
+
# Trend: a model against itself, task by task. A drop in two runs is a trend;
|
|
648
|
+
# the same drop in one run, or against one earlier run, is not. A retried row
|
|
649
|
+
# ("2.0 (x2)") is a score, and a renamed alias goes by its current name.
|
|
650
|
+
import datetime as dt
|
|
651
|
+
hist = json.loads(serve.HISTORY_PATH.read_text(encoding="utf-8"))
|
|
652
|
+
serve.HISTORY_PATH.write_text(json.dumps({**hist, "tr-old": "tr-down"}), encoding="utf-8")
|
|
653
|
+
aliases.forget()
|
|
654
|
+
today = dt.date.today()
|
|
655
|
+
|
|
656
|
+
def note(ago, rows):
|
|
657
|
+
day = (today - dt.timedelta(days=ago)).isoformat()
|
|
658
|
+
with (serve.DATA / "runs" / f"suite-{day}.md").open("a", encoding="utf-8") as fh:
|
|
659
|
+
fh.write(f"\n## Suite run {day}\n\n| Task | Model | Found | Invented | Seconds |\n| --- | --- | --- | --- | --- |\n"
|
|
660
|
+
+ "".join(f"| {t} | `{m}` | {f} | — | 2.0 (x2) |\n" for t, m, f in rows))
|
|
661
|
+
three = ("ta", "tb", "tc")
|
|
662
|
+
note(50, [(t, m, "5/5") for m in ("tr-old", "tr-once", "tr-thin") for t in three] + [("ta", "tr-twice", "0/1")])
|
|
663
|
+
note(50, [("ta", "tr-twice", "1/1")]) # the same task again that day: one score, the mean
|
|
664
|
+
note(40, [(t, m, "5/5") for m in ("tr-old", "tr-once") for t in three] + [("ta", "tr-down", "timeout")])
|
|
665
|
+
note(10, [("ta", "tr-down", "3/5"), ("tb", "tr-down", "2/5"), ("ta", "tr-thin", "2/5"), ("tb", "tr-thin", "2/5")]
|
|
666
|
+
+ [(t, "tr-once", "5/5") for t in three])
|
|
667
|
+
note(3, [("tc", "tr-down", "3/5"), ("tc", "tr-thin", "2/5")] + [(t, "tr-once", "1/5") for t in three])
|
|
668
|
+
note(10, [("ta", "tr-twice", "1/1")])
|
|
669
|
+
# A note appended in the middle of a render is read from disk, not from that render's copy.
|
|
670
|
+
with serve.locked.snapshot():
|
|
671
|
+
serve.note_rows()
|
|
672
|
+
note(0, [("ta", "tr-late", "1/1")])
|
|
673
|
+
late = any(r[3] == "tr-late" for r in serve.note_rows())
|
|
674
|
+
t = serve.trend_rows()["models"]
|
|
675
|
+
w = {m: t[m]["windows"][serve.TREND_DAYS[0]] for m in ("tr-down", "tr-once", "tr-thin")}
|
|
676
|
+
twice = t["tr-twice"]
|
|
677
|
+
late = late and "tr-late" in t and not t["tr-late"]["compared"]
|
|
678
|
+
# The chart's and the table's order: by flag, the biggest move first; too little by name.
|
|
679
|
+
flags = {"x": ("higher", 0.1), "y": ("higher", 0.3), "z": ("lower", -0.1), "b": ("too little", 0.1), "a": ("too little", 0.9)}
|
|
680
|
+
order = sorted(flags, key=lambda m: serve._trend_order(m, dict(zip(("state", "change"), flags[m]))))
|
|
681
|
+
ok = ({m: s["state"] for m, s in w.items()} == {"tr-down": "lower", "tr-once": "steady", "tr-thin": "too little"}
|
|
682
|
+
and [[k, round(v, 9)] for k, v in w["tr-down"]["per"]] == [["tb", -0.6], ["ta", -0.4], ["tc", -0.4]]
|
|
683
|
+
and order == ["z", "y", "x", "a", "b"]
|
|
684
|
+
and (w["tr-down"]["tasks"], w["tr-down"]["down"], w["tr-down"]["runs"], w["tr-down"]["before"]) == (3, 3, 2, 2)
|
|
685
|
+
and abs(w["tr-down"]["change"] + 1.4 / 3) < 1e-9 and w["tr-once"]["runsDown"] == 1
|
|
686
|
+
and "tr-old" not in t and t["tr-down"]["last"] == (today - dt.timedelta(days=3)).isoformat()
|
|
687
|
+
and ("ta", "tr-down") in serve._suite_scores() and serve.trend_rows() is serve.trend_rows()
|
|
688
|
+
and late and twice["compared"] and twice["last"] == (today - dt.timedelta(days=10)).isoformat()
|
|
689
|
+
and abs(twice["windows"][serve.TREND_DAYS[0]]["change"] - 0.5) < 1e-9) # day 50's two scores: one mean
|
|
690
|
+
print(("ok " if ok else "FAIL ") + "trend: a model against itself, two runs to be a trend"
|
|
691
|
+
+ ("" if ok else f" -> {w} {twice} late {late}"))
|
|
692
|
+
failures += 0 if ok else 1
|
|
693
|
+
|
|
694
|
+
# gzip for a caller off this machine that takes it; never over loopback.
|
|
695
|
+
import gzip
|
|
696
|
+
|
|
697
|
+
class Reply:
|
|
698
|
+
def __init__(self, address, accept):
|
|
699
|
+
self.headers, self.client_address, self.sent = {"Accept-Encoding": accept}, (address, 1), {}
|
|
700
|
+
|
|
701
|
+
def send_header(self, k, v):
|
|
702
|
+
self.sent[k] = v
|
|
703
|
+
body = b"<p>a page</p>" * 500
|
|
704
|
+
far, near, plain = Reply("203.0.113.20", "gzip, deflate, br"), Reply("::ffff:127.0.0.1", "gzip"), Reply("10.0.0.2", "")
|
|
705
|
+
refused, other = Reply("10.0.0.3", "gzip;q=0, identity"), Reply("10.0.0.4", "x-gzip-not")
|
|
706
|
+
ok = (gzip.decompress(serve.Handler._length(far, body)) == body and far.sent["Content-Encoding"] == "gzip"
|
|
707
|
+
and int(far.sent["Content-Length"]) < len(body) and serve.Handler._length(near, body) == body
|
|
708
|
+
and serve.Handler._length(plain, body) == body and serve.Handler._length(refused, body) == body
|
|
709
|
+
and serve.Handler._length(other, body) == body
|
|
710
|
+
and "Content-Encoding" not in {**near.sent, **plain.sent, **refused.sent, **other.sent}
|
|
711
|
+
and plain.sent["Content-Length"] == str(len(body)))
|
|
712
|
+
print(("ok " if ok else "FAIL ") + "gzip off this machine only" + ("" if ok else f" -> {far.sent} {near.sent}"))
|
|
713
|
+
failures += 0 if ok else 1
|
|
714
|
+
|
|
715
|
+
# Uptime counts each call once: a gateway seat from its log, a probe from its
|
|
716
|
+
# row, the suite's calls from the ledger only after its last failure kept
|
|
717
|
+
# without text; the ledger's own lines for the gateway and the probe are skipped.
|
|
718
|
+
import datetime as dt
|
|
719
|
+
import ledger
|
|
720
|
+
import time
|
|
721
|
+
home = serve.DATA # `tmp` names another folder by now
|
|
722
|
+
ledger.DIR = home / "usage"
|
|
723
|
+
ledger.DIR.mkdir()
|
|
724
|
+
now = time.time()
|
|
725
|
+
at = dt.datetime.fromtimestamp(now).astimezone().isoformat(timespec="seconds")
|
|
726
|
+
(home / "runs" / "gateway").mkdir(parents=True, exist_ok=True)
|
|
727
|
+
seat = {"call": "p.member1", "seat": "member", "alias": "m1", "status": "ok", "answer": "yes", "error": "", "seconds": 2}
|
|
728
|
+
panel = {"at": at, "shape": "design", "seats": [
|
|
729
|
+
seat, {**seat, "status": "error", "answer": "", "error": "Error: Error code: 503 - high demand"},
|
|
730
|
+
{**seat, "status": "error", "answer": "", "error": "Error: Error code: 429 - Daily token quota exceeded."}]}
|
|
731
|
+
calls = home / "runs" / "gateway" / "calls.jsonl"
|
|
732
|
+
calls.write_text(json.dumps(panel) + "\n", encoding="utf-8")
|
|
733
|
+
(home / "perf").mkdir(exist_ok=True)
|
|
734
|
+
(home / "perf" / "history.csv").write_text(
|
|
735
|
+
"timestamp,model,prompt,n,p50_s,p95_s,min_s,max_s,chars_per_s,answered,refused,error,timeout,failure,detail\n"
|
|
736
|
+
f"{at[:19]},m1,tiny,1,1.5,1.5,1.5,1.5,0,1,0,0,0,,\n", encoding="utf-8")
|
|
737
|
+
|
|
738
|
+
def line(dt_s, caller, status, **more):
|
|
739
|
+
return json.dumps({"t": now + dt_s, "at": at, "alias": "m1", "connection": "c1", "endpoint": "e",
|
|
740
|
+
"served": "m1", "status": status, "attempts": 1, "seconds": 3.0, "caller": caller,
|
|
741
|
+
"purpose": "", **more}, separators=(",", ":")) + "\n"
|
|
742
|
+
ledger._file(now).write_text(
|
|
743
|
+
line(-30, "serve", "ok") + line(-20, "suite/run", "ok") + line(-10, "suite/run", "error")
|
|
744
|
+
+ line(10, "suite/run", "ok") + line(20, "suite/run", "error", error="Error: Error code: 503 - overloaded"),
|
|
745
|
+
encoding="utf-8")
|
|
746
|
+
first = serve._uptime_rows()["models"]["m1"][-1]
|
|
747
|
+
with calls.open("a", encoding="utf-8") as fh:
|
|
748
|
+
fh.write(json.dumps({**panel, "seats": [seat]}) + "\n")
|
|
749
|
+
again = serve._uptime_rows()
|
|
750
|
+
ok = (first[:3] == [3, 2, 1] and again["models"]["m1"][-1][:3] == [4, 2, 1]
|
|
751
|
+
and again["providers"]["c1"][-1][:3] == [4, 2, 1] and first[3] == {"busy": 2, "our quota": 1})
|
|
752
|
+
print(("ok " if ok else "FAIL ") + "uptime counts each call once, the new ones only"
|
|
753
|
+
+ ("" if ok else f" -> {first} {again['models']['m1'][-1]}"))
|
|
754
|
+
failures += 0 if ok else 1
|
|
755
|
+
|
|
756
|
+
# Decision models in Uptime, from their own log; and an error no rule knows, counted as a
|
|
757
|
+
# decision model said, marked so in the hover and the list (never asked here).
|
|
758
|
+
real_decide = decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS
|
|
759
|
+
decide.FILE, decide.LOG = home / "config" / "decision-models.json", home / "metrics" / "decisions.jsonl"
|
|
760
|
+
decide.DECIDED, decide.RESULTS = home / "metrics" / "decided.json", home / "metrics" / "decide-bench.json"
|
|
761
|
+
decide.FILE.write_text(json.dumps({"dmA": {"served": "dmA", "endpoint": "e", "via": "e",
|
|
762
|
+
"question_types": ["choice", "noul"]},
|
|
763
|
+
"dmB": {"served": "v/dmB:free", "endpoint": "e", "via": "e",
|
|
764
|
+
"question_types": ["choice"]}}),
|
|
765
|
+
encoding="utf-8")
|
|
766
|
+
decide.LOG.write_text(
|
|
767
|
+
json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": {"q": {}}, "error": "",
|
|
768
|
+
"seconds": 0.4}) + "\n"
|
|
769
|
+
+ json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": None,
|
|
770
|
+
"error": "404 No endpoints found for v/dmB:free.", "seconds": 0.1}) + "\n", encoding="utf-8")
|
|
771
|
+
odd_error = "Error: the gremlins ate it"
|
|
772
|
+
decide.DECIDED.write_text(json.dumps({"errors": {decide.shape(odd_error): {"class": "busy", "confidence": 0.81,
|
|
773
|
+
"by": "dmA"}},
|
|
774
|
+
"roles": {"m1": {"role": "code", "confidence": 0.7, "by": "dmA"}}}),
|
|
775
|
+
encoding="utf-8")
|
|
776
|
+
with calls.open("a", encoding="utf-8") as fh:
|
|
777
|
+
fh.write(json.dumps({**panel, "seats": [{**seat, "status": "error", "answer": "", "error": odd_error}]}) + "\n")
|
|
778
|
+
u = serve._uptime_rows()
|
|
779
|
+
marked = "busy (decided by dmA at 0.81)"
|
|
780
|
+
ok = (u["models"]["dmB"][-1][:4] == [1, 0, 1, {"setup: not found": 1}]
|
|
781
|
+
and u["models"]["m1"][-1][3].get(marked) == 1 and serve._down(marked)
|
|
782
|
+
and [o[4:] for o in u["odd"]] == [[serve._shape(odd_error), "busy, decided by dmA at 0.81"]]
|
|
783
|
+
and any(r["name"] == "dmB" for r in serve.uptime_data()["models"])
|
|
784
|
+
and "decided by dmA at 0.81" in serve.uptime_card())
|
|
785
|
+
print(("ok " if ok else "FAIL ") + "uptime: decision models' calls, and decided shapes marked"
|
|
786
|
+
+ ("" if ok else f" -> {u['models'].get('dmB')} {u['models']['m1'][-1]} {u['odd']}"))
|
|
787
|
+
failures += 0 if ok else 1
|
|
788
|
+
|
|
789
|
+
# The Decision models card: ranked from MIN_ITEMS items, said so below that; the role
|
|
790
|
+
# suggestion reaches the popup's data, and the collection pick is saved beside the limits.
|
|
791
|
+
cal = [{"from": 0.8, "n": 8, "said": 0.9, "right": 0.875}]
|
|
792
|
+
decide.RESULTS.write_text(json.dumps({"updated": at, "models": {
|
|
793
|
+
"dmA": {"errors": {"asked": 8, "answered": 8, "correct": 7, "accuracy": 0.875, "macro_f1": 0.8, "brier": 0.1,
|
|
794
|
+
"ece": 0.03, "calibration": cal, "confident": {"cutoff": 0.6, "coverage": 1.0, "accuracy": 0.875},
|
|
795
|
+
"seconds": 0.3, "failures": {}, "last": at}},
|
|
796
|
+
"dmB": {"errors": {"asked": 1, "answered": 0, "correct": 0, "failures": {"setup": 1}, "seconds": 0.1,
|
|
797
|
+
"last": at}}}}), encoding="utf-8")
|
|
798
|
+
html = serve.decision_card()
|
|
799
|
+
data = serve.decision_data()
|
|
800
|
+
ok = ("1 of 1" in html and "below 8 items" in html and "7/8" in html and "88% <span class='muted'>of 8"
|
|
801
|
+
in html and data["points"] == [{"model": "dmA", "set": "errors", "said": 0.9, "right": 0.875, "n": 8}]
|
|
802
|
+
and "best of 1 ranked on the errors set: 7 right of 8 asked" in html
|
|
803
|
+
and json.loads(serve.directory_json())["suggested"]["m1"]["role"] == "code")
|
|
804
|
+
print(("ok " if ok else "FAIL ") + "decision card: ranks, below the floor, uses, role suggestion"
|
|
805
|
+
+ ("" if ok else f" -> {data}"))
|
|
806
|
+
failures += 0 if ok else 1
|
|
807
|
+
gw = serve.bench_collections.SETTINGS_FILE
|
|
808
|
+
saved = [serve.save_gateway_settings(5, 2, pick_collection=False, pick_confidence=0.8),
|
|
809
|
+
serve.save_gateway_settings(5, 2)] # not sent: left as they are
|
|
810
|
+
kept = serve.bench_collections.pick_settings()
|
|
811
|
+
saved.append(serve.save_gateway_settings(5, 2, pick_collection=True, pick_confidence=None))
|
|
812
|
+
bad = [serve.save_gateway_settings(5, 2, pick_collection="yes"), serve.save_gateway_settings(5, 2, pick_confidence=0),
|
|
813
|
+
serve.save_gateway_settings(5, 2, pick_confidence=True)]
|
|
814
|
+
ok = (all(r["ok"] for r in saved) and not any(r["ok"] for r in bad)
|
|
815
|
+
and kept == {"pick_collection": False, "pick_confidence": 0.8}
|
|
816
|
+
and json.loads(gw.read_text(encoding="utf-8")).get("pick_collection") is True
|
|
817
|
+
and "pick_confidence" not in json.loads(gw.read_text(encoding="utf-8")))
|
|
818
|
+
print(("ok " if ok else "FAIL ") + "gateway settings: the collection pick, on, off and its confidence"
|
|
819
|
+
+ ("" if ok else f" -> {saved} {bad} {kept}"))
|
|
820
|
+
failures += 0 if ok else 1
|
|
821
|
+
# Export and import: the settings and the endpoints' keys out of this copy, into
|
|
822
|
+
# an empty install; a preview writes nothing; a name outside the scope is refused whole.
|
|
823
|
+
import io
|
|
824
|
+
import zipfile
|
|
825
|
+
real_llm, there = serve.endpoints.LLM_DIR, serve.DATA
|
|
826
|
+
serve.endpoints.LLM_DIR = there
|
|
827
|
+
ep = next(e for e in serve.endpoints.load().values() if e.get("key"))
|
|
828
|
+
(there / "keys.json").write_text(json.dumps({ep["key"]: "k-1", "unrelated": "k-2"}), encoding="utf-8")
|
|
829
|
+
(cfg / "x.json.lock").write_text("", encoding="utf-8")
|
|
830
|
+
plain, with_keys, evil = io.BytesIO(), io.BytesIO(), io.BytesIO()
|
|
831
|
+
serve.export_zip("settings", False, plain)
|
|
832
|
+
made = serve.export_zip("settings", True, with_keys)
|
|
833
|
+
names = zipfile.ZipFile(plain).namelist()
|
|
834
|
+
carried = json.loads(zipfile.ZipFile(with_keys).read("keys.json"))
|
|
835
|
+
other = there / "other"
|
|
836
|
+
other.mkdir()
|
|
837
|
+
serve.DATA = serve.endpoints.LLM_DIR = other
|
|
838
|
+
with_keys.seek(0)
|
|
839
|
+
look = serve.import_zip(with_keys, apply=False)
|
|
840
|
+
wrote_nothing = not (other / "config").exists()
|
|
841
|
+
with_keys.seek(0)
|
|
842
|
+
done = serve.import_zip(with_keys, apply=True)
|
|
843
|
+
with zipfile.ZipFile(evil, "w") as z:
|
|
844
|
+
z.writestr("manifest.json", json.dumps({"benchwarmers_export": 1, "scope": "settings"}))
|
|
845
|
+
z.writestr("config/../../outside.txt", "x")
|
|
846
|
+
evil.seek(0)
|
|
847
|
+
refused = serve.import_zip(evil, apply=True)
|
|
848
|
+
ok = ("manifest.json" in names and "keys.json" not in names and "config/endpoints.json" in names
|
|
849
|
+
and not any(n.endswith(".lock") for n in names) and carried == {ep["key"]: "k-1"} and made["keys"] == 1
|
|
850
|
+
and look["ok"] and look["replaces"] == 0 and look["keys"] == 1 and wrote_nothing
|
|
851
|
+
and done["ok"] and (other / done["backup"]).is_file()
|
|
852
|
+
and (other / "config" / "endpoints.json").read_bytes() == (cfg / "endpoints.json").read_bytes()
|
|
853
|
+
and json.loads((other / "keys.json").read_text(encoding="utf-8")) == {ep["key"]: "k-1"}
|
|
854
|
+
and not refused["ok"] and not (there / "outside.txt").exists() and not (there.parent / "outside.txt").exists())
|
|
855
|
+
serve.DATA, serve.endpoints.LLM_DIR = there, real_llm
|
|
856
|
+
print(("ok " if ok else "FAIL ") + "export and import: settings and keys out, into an empty install, nothing outside"
|
|
857
|
+
+ ("" if ok else f" -> {names[:5]} {carried} {look} {done} {refused}"))
|
|
858
|
+
failures += 0 if ok else 1
|
|
859
|
+
decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS = real_decide
|
|
537
860
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
538
861
|
print("\nall rules hold" if not failures else f"\n{failures} FAILED")
|
|
539
862
|
return 1 if failures else 0
|