@wendellresearch-ai/benchwarmers 0.4.0 → 0.5.0

package/scripts/test_serve_edit.py

@@ -398,7 +398,7 @@ def main() -> int:
         stopped = {i["served"]: i["state"] for i in serve.onboard_queue()["items"]}
         order.remove("quiet")
         serve.queue_onboard([{"served": s, "endpoint": "e", "mode": m, "vision": s == "quiet"} for s, m in
-                             (("ok", "full"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
+                             (("ok", "full-adhoc"), ("quiet", "quick"), ("bad", "quick-adhoc"))])
         settle()
         batch = {i["served"]: (i["state"], i["alias"], bool(i["note"])) for i in serve.onboard_queue()["items"]}
         cleared = serve.clear_onboard_queue()["items"]
@@ -412,10 +412,55 @@ def main() -> int:
           and batch == {"ok": ("done", "ok-x", False), "quiet": ("no answer", "quiet-x", True),
                         "bad": ("failed", "bad-x", True)}
           and cleared == [] and bad_input == (False, False, False)
-          and modes == {"a": "full", "ok": "full", "quiet": "quick vision", "bad": "quick-adhoc"})
+          and modes == {"a": "full", "ok": "full-adhoc", "quiet": "quick vision", "bad": "quick-adhoc"}
+          and serve.ONBOARD_MODES["full-adhoc"] == ["--ad-hoc"])            # the suite now, then ad hoc
     print(("ok   " if ok else "FAIL ") + "onboarding queue" + ("" if ok else f" -> {stopped} {order} {batch} {modes}"))
     failures += 0 if ok else 1
 
+    # Restart: refused unless serve.py's parent started it (nothing would start it
+    # again); refused while a job runs, unless forced; then the server shuts down.
+    import scheduler
+    kept = os.environ.get("BENCH_SERVED"), serve.SERVER
+    shut = []
+    try:
+        os.environ.pop("BENCH_SERVED", None)
+        alone = serve.restart()
+        os.environ["BENCH_SERVED"] = "1"
+        serve.SERVER = type("Server", (), {"shutdown": lambda self: shut.append(1)})()
+        scheduler.RUNNING["suite"] = True
+        busy, forced = serve.restart(), serve.restart(force=True)
+        time.sleep(0.8)                                  # the shutdown waits half a second
+    finally:
+        scheduler.RUNNING.pop("suite", None)
+        serve.SERVER = kept[1]
+        os.environ.pop("BENCH_SERVED", None) if kept[0] is None else os.environ.update(BENCH_SERVED=kept[0])
+    ok = (not alone["ok"] and busy.get("busy") and not busy["ok"] and "suite" in busy["errors"][0]
+          and forced["ok"] and forced["cut"] and shut == [1])
+    print(("ok   " if ok else "FAIL ") + "restart" + ("" if ok else f" -> {alone} {busy} {forced} {shut}"))
+    failures += 0 if ok else 1
+
+    # Action needed: an alert whose models have all left the bench is not shown,
+    # before the daily check rewrites the file (2026-10-28: a kilo connection
+    # alert stayed after Retire all 4).
+    kept_data, kept_reg = serve.DATA, serve.registered
+    with tempfile.TemporaryDirectory() as tmp:
+        (pathlib.Path(tmp) / "metrics").mkdir()
+        (pathlib.Path(tmp) / "metrics" / "alerts.json").write_text(json.dumps({
+            "connection:held:gone.example": {"kind": "connection", "message": "every model on gone.example is failing",
+                                             "aliases": ["g1", "g2"], "first_seen": "2026-10-28"},
+            "connection:held:here.example": {"kind": "connection", "message": "every model on here.example is failing",
+                                             "aliases": ["h1", "g1"], "first_seen": "2026-10-28"},
+            "retiring:x:old": {"kind": "retiring", "alias": "old", "message": "old retires", "first_seen": "2026-10-28"}}),
+            encoding="utf-8")
+        serve.DATA, serve.registered = pathlib.Path(tmp), lambda: {"h1-served": "h1"}
+        try:
+            shown = serve.problems()
+        finally:
+            serve.DATA, serve.registered = kept_data, kept_reg
+    ok = "here.example" in shown and "gone.example" not in shown and "old retires" not in shown
+    print(("ok   " if ok else "FAIL ") + "action needed drops alerts on retired models")
+    failures += 0 if ok else 1
+
     # Dismiss: one picker key or many in one write, a change-log line each;
     # undo brings back only the ones named; nothing named is refused.
     serve.DISMISSED_PATH = cfg / "dismissed.json"
@@ -433,6 +478,14 @@ def main() -> int:
     print(("ok   " if ok else "FAIL ") + "dismiss in bulk, and undo" + ("" if ok else f" -> {gone} {left} {log} {refused}"))
     failures += 0 if ok else 1
 
+    # The member picker's provider chips: a proxy's vendor, else the bare host.
+    names = {"nvidia-api": "nvidia", "api.unorouter.com": "unorouter", "router.requesty.ai": "requesty",
+             "cli-proxy-api/codex": "codex", "openrouter": "openrouter", "apihub.agnes-ai.com": "agnes-ai"}
+    got = {t: serve.provider_label(t) for t in names}
+    ok = got == names
+    print(("ok   " if ok else "FAIL ") + "provider chip names" + ("" if ok else f" -> {got}"))
+    failures += 0 if ok else 1
+
     # The owner's files are as they were. alerts.json can change under a
     # running suite, so only what this test could do counts: the victim's
     # alerts cleared from the real file.