@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,8 +4,9 @@
4
4
 
5
5
  Each rule the popup enforces is tried once and must refuse; then a role change
6
6
  and a rename are applied to the copy and checked. The llm sync is not run:
7
- serve.ROOT points at the copy, which has no scripts/sync_config.py, so the
8
- sync reports that it could not run. Alerts and endpoints point at copies too,
7
+ with the built-in client it is skipped, and with the llm CLI serve.ROOT points
8
+ at the copy, which has no scripts/sync_config.py, so the sync reports that it
9
+ could not run. Alerts and endpoints point at copies too,
9
10
  because removing a model clears its alerts and a regressed endpoint check
10
11
  would write the endpoint file.
11
12
  """
@@ -170,6 +171,50 @@ def main() -> int:
170
171
  ok = (set(view["served"]) == set(view["models"]) and victim not in view["served"]
171
172
  and view["served"].get(other) == d3[other]["served"])
172
173
  print(("ok " if ok else "FAIL ") + "collection view: served name for every member choice"); failures += 0 if ok else 1
174
+ # Clients: every snippet parses in its format, names each collection, sets the
175
+ # timeouts a panel needs, and a made key goes only into a shell line or a field.
176
+ import tomllib
177
+ try:
178
+ import yaml # comes with the llm CLI; the built-in client's venv may lack it
179
+ load_yaml = yaml.safe_load
180
+ except ImportError:
181
+ load_yaml = None
182
+ bad = []
183
+ for keys in (False, True):
184
+ setups = {c["id"]: c for c in serve.client_setups("http://127.0.0.1:8122", [("core", 200_000, False),
185
+ ("fast", None, True)], keys)}
186
+ parsed = {}
187
+ for c in setups.values():
188
+ for b in c["blocks"]:
189
+ parse = {"json": json.loads, "toml": tomllib.loads, "yaml": load_yaml}.get(b["lang"])
190
+ try:
191
+ parsed[c["id"], b["label"]] = parse(b["text"]) if parse else b["text"]
192
+ except Exception as exc: # noqa: BLE001
193
+ bad.append(f"{c['id']} {b['label']} does not parse: {exc}")
194
+ if b["key"] and b["lang"] not in ("sh", "bat", "text"):
195
+ bad.append(f"{c['id']} {b['label']}: a key would go into a file")
196
+ alltext = "\n".join(b["text"] for b in c["blocks"])
197
+ want = ("bench/core:merged", "bench/fast:merged") if c["id"] == "chat" else \
198
+ ("bench/core:route", "bench/fast:route") if c["id"] != "codex" else ("bench/core:route",)
199
+ bad += [f"{c['id']} lacks {w}" for w in want if w not in alltext]
200
+ oc, cx = parsed.get(("opencode", "opencode.json")), parsed.get(("codex", "config.toml"))
201
+ if oc and cx:
202
+ p = oc["provider"]["benchwarmers"]
203
+ if (oc["mcp"]["benchwarmers"]["timeout"] != 900_000 or cx["mcp_servers"]["benchwarmers"]["tool_timeout_sec"] != 900
204
+ or cx["model_providers"]["benchwarmers"]["wire_api"] != "responses" or cx["web_search"] != "disabled"
205
+ or p["models"]["bench/fast:route"]["limit"]["context"] != serve.UNKNOWN_CONTEXT
206
+ or ("apiKey" in p["options"]) != keys or ("env_key" in cx["model_providers"]["benchwarmers"]) != keys):
207
+ bad.append(f"opencode or Codex settings wrong (keys={keys})")
208
+ if parsed.get(("cline", "Cline: cline_mcp_settings.json"), {}).get("mcpServers", {}).get("benchwarmers", {}).get("timeout") != 900:
209
+ bad.append("Cline's MCP timeout is not 900 s")
210
+ if not all(m["toolCalling"] for m in parsed.get(("vscode", "chatLanguageModels.json"), [{}])[0].get("models", [{}])):
211
+ bad.append("a VS Code model without toolCalling is hidden from the agent picker")
212
+ if load_yaml and parsed.get(("goose", "config.yaml"), {}).get("extensions", {}).get("benchwarmers", {}).get("timeout") != 900:
213
+ bad.append("goose's extension timeout is not 900 s")
214
+ ok = not bad
215
+ print(("ok " if ok else "FAIL ") + "client set-ups: JSON, TOML" + (", YAML" if load_yaml else " (YAML unread: no PyYAML)")
216
+ + " parse, collections named, timeouts set, keys only in shell lines and fields" + ("" if ok else f" -> {bad}"))
217
+ failures += 0 if ok else 1
173
218
  # restore: same alias back, out of retired.json, role and ad hoc kept, and
174
219
  # back into the collections that still exist (one deleted, one changed since)
175
220
  serve.delete_collection("doomed")
@@ -255,7 +300,7 @@ def main() -> int:
255
300
 
256
301
  # Call options: max_tokens set, refused above the model's max output or as
257
302
  # a fraction or for no model, then cleared; the llm CLI's copy is stubbed.
258
- told, real_info = [], serve.catalog.info
303
+ told, real_info, llm_option = [], serve.catalog.info, serve._llm_option
259
304
  serve._llm_option = lambda alias, name, value: told.append((alias, name, value)) or "stub"
260
305
  serve.catalog.info = lambda a: {**real_info(a), "max_output": 50_000}
261
306
  chat = next(a for a, m in sorted(serve.model_directory().items()) if m.get("kind") != "decision")
@@ -290,6 +335,7 @@ def main() -> int:
290
335
  import client as model_client
291
336
  model_client.CHOICE = cfg / "client.json"
292
337
  env, real_which = os.environ.pop("BENCH_CLIENT", None), shutil.which
338
+ ran, real_run = [], serve.subprocess.run
293
339
  try:
294
340
  shutil.which = lambda name: "/usr/bin/" + name # llm installed: still built-in
295
341
  first = model_client.chosen()
@@ -305,8 +351,14 @@ def main() -> int:
305
351
  no_llm = serve.save_client("llm")["ok"]
306
352
  shutil.which = real_which
307
353
  unknown = serve.save_client("curl")["ok"]
354
+ # With the built-in client nothing is mirrored into the llm CLI (5-6 s
355
+ # a save); choosing the llm CLI syncs it once.
356
+ serve.subprocess.run = lambda args, **kw: ran.append(args) or serve.subprocess.CompletedProcess(args, 0, "", "")
357
+ quiet = (serve._sync_cli(), llm_option(chat, "max_tokens", 5), list(ran))
358
+ shutil.which = lambda name: "/usr/bin/" + name
359
+ switched, again = serve.save_client("llm"), serve.save_client("llm")
308
360
  finally:
309
- shutil.which = real_which
361
+ serve.subprocess.run, shutil.which = real_run, real_which
310
362
  os.environ.pop("BENCH_CLIENT", None)
311
363
  if env is not None:
312
364
  os.environ["BENCH_CLIENT"] = env
@@ -314,6 +366,12 @@ def main() -> int:
314
366
  and pinned == (("llm", "environment"), False) and not no_llm and not unknown)
315
367
  print(("ok " if ok else "FAIL ") + "model client choice" + ("" if ok else f" -> {first} {made} {picked} {pinned}"))
316
368
  failures += 0 if ok else 1
369
+ # Saving it again syncs again: the way to retry a failed sync.
370
+ ok = (quiet == (serve.NOT_SYNCED, serve.NOT_SYNCED, []) and switched["sync"] == "synced to the llm CLI"
371
+ and again["sync"] == "synced to the llm CLI" and len(ran) == 2
372
+ and all(str(r[-1]).endswith("sync_config.py") for r in ran))
373
+ print(("ok " if ok else "FAIL ") + "llm CLI synced only when it is the client" + ("" if ok else f" -> {quiet} {switched} {ran}"))
374
+ failures += 0 if ok else 1
317
375
 
318
376
  # Free only in their own app: only what differs from the built-in is written; bad input refused.
319
377
  import sources
@@ -514,6 +572,25 @@ def main() -> int:
514
572
  print(("ok " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
515
573
  failures += 0 if ok else 1
516
574
 
575
+ # An answer clears every alert it disproves (2026-11-02: big-pickle answered a
576
+ # Retry and stayed listed, its unrecognised-failure alert open), never a price;
577
+ # a held connection loses the model, and goes with its last one.
578
+ alerts.raise_alert("model:unclassified:t/served", "registration", "t fails oddly")
579
+ alerts.raise_alert("model:pending:t/served", "retiring", "t retires", alias="t")
580
+ alerts.raise_alert("model:paid:t/served", "went paid", "t has a price", alias="t")
581
+ alerts.raise_alert("model:pending:t/served-too", "retiring", "another retires", alias="t2")
582
+ alerts.raise_alert("connection:held:x", "connection", "all fail", aliases=["t", "t2"])
583
+ alerts.raise_alert("connection:held:y", "connection", "all fail", aliases=["t"])
584
+ cleared = alerts.answered("t", "t/served")
585
+ left = alerts.active()
586
+ ok = (set(cleared) == {"model:unclassified:t/served", "model:pending:t/served", "connection:held:y"}
587
+ and "model:paid:t/served" in left and "model:pending:t/served-too" in left
588
+ and left["connection:held:x"]["aliases"] == ["t2"])
589
+ for k in ("model:paid:t/served", "model:pending:t/served-too", "connection:held:x"):
590
+ alerts.clear(k)
591
+ print(("ok " if ok else "FAIL ") + "an answer clears what it disproves, not a price" + ("" if ok else f" -> {cleared}"))
592
+ failures += 0 if ok else 1
593
+
517
594
  # Got it clears a notice (NVIDIA changed its figures), never a state still wrong.
518
595
  alerts.raise_alert("test:notice", "limits", "a notice", dismiss=True)
519
596
  alerts.raise_alert("test:state", "limits", "a state")
@@ -534,6 +611,252 @@ def main() -> int:
534
611
  if (DATA / "config" / "collections.json").exists() else None))
535
612
  print(("ok " if ok else "FAIL ") + "the real alerts, endpoints and collections were not touched")
536
613
  failures += 0 if ok else 1
614
+
615
+ # The Logs page reads a log from its end, and still gets whole lines.
616
+ log = cfg / "big.log"
617
+ lines = [f"{i} " + "x" * 2000 for i in range(400)]
618
+ log.write_text("\n".join(lines) + "\n\n", encoding="utf-8")
619
+ ok = (serve._log_rows(log) == lines[::-1][:200] and serve._log_rows(log, 100) == lines[::-1][:100]
620
+ and serve._log_rows(log.with_name("none.log")) == [] and list(serve._from_end(log)) == lines[::-1])
621
+ print(("ok " if ok else "FAIL ") + "a log's last lines, read from its end")
622
+ failures += 0 if ok else 1
623
+
624
+ # Confidently wrong is scored again when a task or a saved answer changes.
625
+ answer = cfg / "tasks" / "t" / "output" / "m.md"
626
+ answer.parent.mkdir(parents=True)
627
+ (cfg / "tasks" / "t" / "task.json").write_text("{}", encoding="utf-8")
628
+ answer.write_text("a", encoding="utf-8")
629
+ before = serve._tasks_key(cfg / "tasks")
630
+ answer.write_text("ab", encoding="utf-8")
631
+ ok = len(before) == 2 and serve._tasks_key(cfg / "tasks") != before and serve._tasks_key(cfg / "none") == ()
632
+ print(("ok " if ok else "FAIL ") + "a changed answer changes the scoring's key")
633
+ failures += 0 if ok else 1
634
+
635
+ # Suite scores and statuses are worked out once a render; what one pane
636
+ # changes in its copy, the next does not see.
637
+ (serve.DATA / "runs" / "suite-2026-01-01.md").write_text("| t1 | `m1` | 1/2 | — | 3.0 |\n", encoding="utf-8")
638
+ with serve.locked.snapshot():
639
+ serve.model_status().clear()
640
+ for v in serve.suite_scores().values():
641
+ v.clear()
642
+ ok = (serve.model_status() == serve._model_status() and serve._suite_scores()
643
+ and serve.suite_scores() == serve._suite_scores())
644
+ print(("ok " if ok else "FAIL ") + "suite scores and statuses once a render, a copy each")
645
+ failures += 0 if ok else 1
646
+
647
+ # Trend: a model against itself, task by task. A drop in two runs is a trend;
648
+ # the same drop in one run, or against one earlier run, is not. A retried row
649
+ # ("2.0 (x2)") is a score, and a renamed alias goes by its current name.
650
+ import datetime as dt
651
+ hist = json.loads(serve.HISTORY_PATH.read_text(encoding="utf-8"))
652
+ serve.HISTORY_PATH.write_text(json.dumps({**hist, "tr-old": "tr-down"}), encoding="utf-8")
653
+ aliases.forget()
654
+ today = dt.date.today()
655
+
656
+ def note(ago, rows):
657
+ day = (today - dt.timedelta(days=ago)).isoformat()
658
+ with (serve.DATA / "runs" / f"suite-{day}.md").open("a", encoding="utf-8") as fh:
659
+ fh.write(f"\n## Suite run {day}\n\n| Task | Model | Found | Invented | Seconds |\n| --- | --- | --- | --- | --- |\n"
660
+ + "".join(f"| {t} | `{m}` | {f} | — | 2.0 (x2) |\n" for t, m, f in rows))
661
+ three = ("ta", "tb", "tc")
662
+ note(50, [(t, m, "5/5") for m in ("tr-old", "tr-once", "tr-thin") for t in three] + [("ta", "tr-twice", "0/1")])
663
+ note(50, [("ta", "tr-twice", "1/1")]) # the same task again that day: one score, the mean
664
+ note(40, [(t, m, "5/5") for m in ("tr-old", "tr-once") for t in three] + [("ta", "tr-down", "timeout")])
665
+ note(10, [("ta", "tr-down", "3/5"), ("tb", "tr-down", "2/5"), ("ta", "tr-thin", "2/5"), ("tb", "tr-thin", "2/5")]
666
+ + [(t, "tr-once", "5/5") for t in three])
667
+ note(3, [("tc", "tr-down", "3/5"), ("tc", "tr-thin", "2/5")] + [(t, "tr-once", "1/5") for t in three])
668
+ note(10, [("ta", "tr-twice", "1/1")])
669
+ # A note appended in the middle of a render is read from disk, not from that render's copy.
670
+ with serve.locked.snapshot():
671
+ serve.note_rows()
672
+ note(0, [("ta", "tr-late", "1/1")])
673
+ late = any(r[3] == "tr-late" for r in serve.note_rows())
674
+ t = serve.trend_rows()["models"]
675
+ w = {m: t[m]["windows"][serve.TREND_DAYS[0]] for m in ("tr-down", "tr-once", "tr-thin")}
676
+ twice = t["tr-twice"]
677
+ late = late and "tr-late" in t and not t["tr-late"]["compared"]
678
+ # The chart's and the table's order: by flag, the biggest move first; too little by name.
679
+ flags = {"x": ("higher", 0.1), "y": ("higher", 0.3), "z": ("lower", -0.1), "b": ("too little", 0.1), "a": ("too little", 0.9)}
680
+ order = sorted(flags, key=lambda m: serve._trend_order(m, dict(zip(("state", "change"), flags[m]))))
681
+ ok = ({m: s["state"] for m, s in w.items()} == {"tr-down": "lower", "tr-once": "steady", "tr-thin": "too little"}
682
+ and [[k, round(v, 9)] for k, v in w["tr-down"]["per"]] == [["tb", -0.6], ["ta", -0.4], ["tc", -0.4]]
683
+ and order == ["z", "y", "x", "a", "b"]
684
+ and (w["tr-down"]["tasks"], w["tr-down"]["down"], w["tr-down"]["runs"], w["tr-down"]["before"]) == (3, 3, 2, 2)
685
+ and abs(w["tr-down"]["change"] + 1.4 / 3) < 1e-9 and w["tr-once"]["runsDown"] == 1
686
+ and "tr-old" not in t and t["tr-down"]["last"] == (today - dt.timedelta(days=3)).isoformat()
687
+ and ("ta", "tr-down") in serve._suite_scores() and serve.trend_rows() is serve.trend_rows()
688
+ and late and twice["compared"] and twice["last"] == (today - dt.timedelta(days=10)).isoformat()
689
+ and abs(twice["windows"][serve.TREND_DAYS[0]]["change"] - 0.5) < 1e-9) # day 50's two scores: one mean
690
+ print(("ok " if ok else "FAIL ") + "trend: a model against itself, two runs to be a trend"
691
+ + ("" if ok else f" -> {w} {twice} late {late}"))
692
+ failures += 0 if ok else 1
693
+
694
+ # gzip for a caller off this machine that takes it; never over loopback.
695
+ import gzip
696
+
697
+ class Reply:
698
+ def __init__(self, address, accept):
699
+ self.headers, self.client_address, self.sent = {"Accept-Encoding": accept}, (address, 1), {}
700
+
701
+ def send_header(self, k, v):
702
+ self.sent[k] = v
703
+ body = b"<p>a page</p>" * 500
704
+ far, near, plain = Reply("203.0.113.20", "gzip, deflate, br"), Reply("::ffff:127.0.0.1", "gzip"), Reply("10.0.0.2", "")
705
+ refused, other = Reply("10.0.0.3", "gzip;q=0, identity"), Reply("10.0.0.4", "x-gzip-not")
706
+ ok = (gzip.decompress(serve.Handler._length(far, body)) == body and far.sent["Content-Encoding"] == "gzip"
707
+ and int(far.sent["Content-Length"]) < len(body) and serve.Handler._length(near, body) == body
708
+ and serve.Handler._length(plain, body) == body and serve.Handler._length(refused, body) == body
709
+ and serve.Handler._length(other, body) == body
710
+ and "Content-Encoding" not in {**near.sent, **plain.sent, **refused.sent, **other.sent}
711
+ and plain.sent["Content-Length"] == str(len(body)))
712
+ print(("ok " if ok else "FAIL ") + "gzip off this machine only" + ("" if ok else f" -> {far.sent} {near.sent}"))
713
+ failures += 0 if ok else 1
714
+
715
+ # Uptime counts each call once: a gateway seat from its log, a probe from its
716
+ # row, the suite's calls from the ledger only after its last failure kept
717
+ # without text; the ledger's own lines for the gateway and the probe are skipped.
718
+ import datetime as dt
719
+ import ledger
720
+ import time
721
+ home = serve.DATA # `tmp` names another folder by now
722
+ ledger.DIR = home / "usage"
723
+ ledger.DIR.mkdir()
724
+ now = time.time()
725
+ at = dt.datetime.fromtimestamp(now).astimezone().isoformat(timespec="seconds")
726
+ (home / "runs" / "gateway").mkdir(parents=True, exist_ok=True)
727
+ seat = {"call": "p.member1", "seat": "member", "alias": "m1", "status": "ok", "answer": "yes", "error": "", "seconds": 2}
728
+ panel = {"at": at, "shape": "design", "seats": [
729
+ seat, {**seat, "status": "error", "answer": "", "error": "Error: Error code: 503 - high demand"},
730
+ {**seat, "status": "error", "answer": "", "error": "Error: Error code: 429 - Daily token quota exceeded."}]}
731
+ calls = home / "runs" / "gateway" / "calls.jsonl"
732
+ calls.write_text(json.dumps(panel) + "\n", encoding="utf-8")
733
+ (home / "perf").mkdir(exist_ok=True)
734
+ (home / "perf" / "history.csv").write_text(
735
+ "timestamp,model,prompt,n,p50_s,p95_s,min_s,max_s,chars_per_s,answered,refused,error,timeout,failure,detail\n"
736
+ f"{at[:19]},m1,tiny,1,1.5,1.5,1.5,1.5,0,1,0,0,0,,\n", encoding="utf-8")
737
+
738
+ def line(dt_s, caller, status, **more):
739
+ return json.dumps({"t": now + dt_s, "at": at, "alias": "m1", "connection": "c1", "endpoint": "e",
740
+ "served": "m1", "status": status, "attempts": 1, "seconds": 3.0, "caller": caller,
741
+ "purpose": "", **more}, separators=(",", ":")) + "\n"
742
+ ledger._file(now).write_text(
743
+ line(-30, "serve", "ok") + line(-20, "suite/run", "ok") + line(-10, "suite/run", "error")
744
+ + line(10, "suite/run", "ok") + line(20, "suite/run", "error", error="Error: Error code: 503 - overloaded"),
745
+ encoding="utf-8")
746
+ first = serve._uptime_rows()["models"]["m1"][-1]
747
+ with calls.open("a", encoding="utf-8") as fh:
748
+ fh.write(json.dumps({**panel, "seats": [seat]}) + "\n")
749
+ again = serve._uptime_rows()
750
+ ok = (first[:3] == [3, 2, 1] and again["models"]["m1"][-1][:3] == [4, 2, 1]
751
+ and again["providers"]["c1"][-1][:3] == [4, 2, 1] and first[3] == {"busy": 2, "our quota": 1})
752
+ print(("ok " if ok else "FAIL ") + "uptime counts each call once, the new ones only"
753
+ + ("" if ok else f" -> {first} {again['models']['m1'][-1]}"))
754
+ failures += 0 if ok else 1
755
+
756
+ # Decision models in Uptime, from their own log; and an error no rule knows, counted as a
757
+ # decision model said, marked so in the hover and the list (never asked here).
758
+ real_decide = decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS
759
+ decide.FILE, decide.LOG = home / "config" / "decision-models.json", home / "metrics" / "decisions.jsonl"
760
+ decide.DECIDED, decide.RESULTS = home / "metrics" / "decided.json", home / "metrics" / "decide-bench.json"
761
+ decide.FILE.write_text(json.dumps({"dmA": {"served": "dmA", "endpoint": "e", "via": "e",
762
+ "question_types": ["choice", "noul"]},
763
+ "dmB": {"served": "v/dmB:free", "endpoint": "e", "via": "e",
764
+ "question_types": ["choice"]}}),
765
+ encoding="utf-8")
766
+ decide.LOG.write_text(
767
+ json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": {"q": {}}, "error": "",
768
+ "seconds": 0.4}) + "\n"
769
+ + json.dumps({"at": at, "alias": "dmB", "served": "v/dmB:free", "via": "e", "answers": None,
770
+ "error": "404 No endpoints found for v/dmB:free.", "seconds": 0.1}) + "\n", encoding="utf-8")
771
+ odd_error = "Error: the gremlins ate it"
772
+ decide.DECIDED.write_text(json.dumps({"errors": {decide.shape(odd_error): {"class": "busy", "confidence": 0.81,
773
+ "by": "dmA"}},
774
+ "roles": {"m1": {"role": "code", "confidence": 0.7, "by": "dmA"}}}),
775
+ encoding="utf-8")
776
+ with calls.open("a", encoding="utf-8") as fh:
777
+ fh.write(json.dumps({**panel, "seats": [{**seat, "status": "error", "answer": "", "error": odd_error}]}) + "\n")
778
+ u = serve._uptime_rows()
779
+ marked = "busy (decided by dmA at 0.81)"
780
+ ok = (u["models"]["dmB"][-1][:4] == [1, 0, 1, {"setup: not found": 1}]
781
+ and u["models"]["m1"][-1][3].get(marked) == 1 and serve._down(marked)
782
+ and [o[4:] for o in u["odd"]] == [[serve._shape(odd_error), "busy, decided by dmA at 0.81"]]
783
+ and any(r["name"] == "dmB" for r in serve.uptime_data()["models"])
784
+ and "decided by dmA at 0.81" in serve.uptime_card())
785
+ print(("ok " if ok else "FAIL ") + "uptime: decision models' calls, and decided shapes marked"
786
+ + ("" if ok else f" -> {u['models'].get('dmB')} {u['models']['m1'][-1]} {u['odd']}"))
787
+ failures += 0 if ok else 1
788
+
789
+ # The Decision models card: ranked from MIN_ITEMS items, said so below that; the role
790
+ # suggestion reaches the popup's data, and the collection pick is saved beside the limits.
791
+ cal = [{"from": 0.8, "n": 8, "said": 0.9, "right": 0.875}]
792
+ decide.RESULTS.write_text(json.dumps({"updated": at, "models": {
793
+ "dmA": {"errors": {"asked": 8, "answered": 8, "correct": 7, "accuracy": 0.875, "macro_f1": 0.8, "brier": 0.1,
794
+ "ece": 0.03, "calibration": cal, "confident": {"cutoff": 0.6, "coverage": 1.0, "accuracy": 0.875},
795
+ "seconds": 0.3, "failures": {}, "last": at}},
796
+ "dmB": {"errors": {"asked": 1, "answered": 0, "correct": 0, "failures": {"setup": 1}, "seconds": 0.1,
797
+ "last": at}}}}), encoding="utf-8")
798
+ html = serve.decision_card()
799
+ data = serve.decision_data()
800
+ ok = ("1 of 1" in html and "below 8 items" in html and "7/8" in html and "88% <span class='muted'>of 8"
801
+ in html and data["points"] == [{"model": "dmA", "set": "errors", "said": 0.9, "right": 0.875, "n": 8}]
802
+ and "best of 1 ranked on the errors set: 7 right of 8 asked" in html
803
+ and json.loads(serve.directory_json())["suggested"]["m1"]["role"] == "code")
804
+ print(("ok " if ok else "FAIL ") + "decision card: ranks, below the floor, uses, role suggestion"
805
+ + ("" if ok else f" -> {data}"))
806
+ failures += 0 if ok else 1
807
+ gw = serve.bench_collections.SETTINGS_FILE
808
+ saved = [serve.save_gateway_settings(5, 2, pick_collection=False, pick_confidence=0.8),
809
+ serve.save_gateway_settings(5, 2)] # not sent: left as they are
810
+ kept = serve.bench_collections.pick_settings()
811
+ saved.append(serve.save_gateway_settings(5, 2, pick_collection=True, pick_confidence=None))
812
+ bad = [serve.save_gateway_settings(5, 2, pick_collection="yes"), serve.save_gateway_settings(5, 2, pick_confidence=0),
813
+ serve.save_gateway_settings(5, 2, pick_confidence=True)]
814
+ ok = (all(r["ok"] for r in saved) and not any(r["ok"] for r in bad)
815
+ and kept == {"pick_collection": False, "pick_confidence": 0.8}
816
+ and json.loads(gw.read_text(encoding="utf-8")).get("pick_collection") is True
817
+ and "pick_confidence" not in json.loads(gw.read_text(encoding="utf-8")))
818
+ print(("ok " if ok else "FAIL ") + "gateway settings: the collection pick, on, off and its confidence"
819
+ + ("" if ok else f" -> {saved} {bad} {kept}"))
820
+ failures += 0 if ok else 1
821
+ # Export and import: the settings and the endpoints' keys out of this copy, into
822
+ # an empty install; a preview writes nothing; a name outside the scope is refused whole.
823
+ import io
824
+ import zipfile
825
+ real_llm, there = serve.endpoints.LLM_DIR, serve.DATA
826
+ serve.endpoints.LLM_DIR = there
827
+ ep = next(e for e in serve.endpoints.load().values() if e.get("key"))
828
+ (there / "keys.json").write_text(json.dumps({ep["key"]: "k-1", "unrelated": "k-2"}), encoding="utf-8")
829
+ (cfg / "x.json.lock").write_text("", encoding="utf-8")
830
+ plain, with_keys, evil = io.BytesIO(), io.BytesIO(), io.BytesIO()
831
+ serve.export_zip("settings", False, plain)
832
+ made = serve.export_zip("settings", True, with_keys)
833
+ names = zipfile.ZipFile(plain).namelist()
834
+ carried = json.loads(zipfile.ZipFile(with_keys).read("keys.json"))
835
+ other = there / "other"
836
+ other.mkdir()
837
+ serve.DATA = serve.endpoints.LLM_DIR = other
838
+ with_keys.seek(0)
839
+ look = serve.import_zip(with_keys, apply=False)
840
+ wrote_nothing = not (other / "config").exists()
841
+ with_keys.seek(0)
842
+ done = serve.import_zip(with_keys, apply=True)
843
+ with zipfile.ZipFile(evil, "w") as z:
844
+ z.writestr("manifest.json", json.dumps({"benchwarmers_export": 1, "scope": "settings"}))
845
+ z.writestr("config/../../outside.txt", "x")
846
+ evil.seek(0)
847
+ refused = serve.import_zip(evil, apply=True)
848
+ ok = ("manifest.json" in names and "keys.json" not in names and "config/endpoints.json" in names
849
+ and not any(n.endswith(".lock") for n in names) and carried == {ep["key"]: "k-1"} and made["keys"] == 1
850
+ and look["ok"] and look["replaces"] == 0 and look["keys"] == 1 and wrote_nothing
851
+ and done["ok"] and (other / done["backup"]).is_file()
852
+ and (other / "config" / "endpoints.json").read_bytes() == (cfg / "endpoints.json").read_bytes()
853
+ and json.loads((other / "keys.json").read_text(encoding="utf-8")) == {ep["key"]: "k-1"}
854
+ and not refused["ok"] and not (there / "outside.txt").exists() and not (there.parent / "outside.txt").exists())
855
+ serve.DATA, serve.endpoints.LLM_DIR = there, real_llm
856
+ print(("ok " if ok else "FAIL ") + "export and import: settings and keys out, into an empty install, nothing outside"
857
+ + ("" if ok else f" -> {names[:5]} {carried} {look} {done} {refused}"))
858
+ failures += 0 if ok else 1
859
+ decide.FILE, decide.LOG, decide.DECIDED, decide.RESULTS = real_decide
537
860
  shutil.rmtree(tmp, ignore_errors=True)
538
861
  print("\nall rules hold" if not failures else f"\n{failures} FAILED")
539
862
  return 1 if failures else 0