@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.0

package/serve.py

@@ -13,8 +13,9 @@ It writes three kinds of thing, each behind a popup and each logged to
 the bench itself — a role the config defines; an alias in the standing form,
 unused, never a retired name — written atomically, the rename recorded in
 `config/alias-history.json`, the `llm` registry re-synced. A model's REMOVAL
-(same popup, typed back to confirm, reason required): deregistered and
-recorded in `config/retired.json`; no evidence is deleted. And ONBOARDING
+(same popup, typed back to confirm, reason required): deregistered, taken
+out of every collection, and recorded in `config/retired.json`; no evidence
+is deleted. And ONBOARDING
 (header button): `scripts/onboard.py` started in the background with its log
 tailed here. The upstream model id is never editable. Nothing else can write.
 
@@ -60,6 +61,16 @@ import guard  # noqa: E402
 import locked  # noqa: E402
 
 
+def gateway_url() -> str:
+    """Where a caller reaches the gateway: this dashboard's own address (it
+    answers the gateway's routes), or BENCHWARMERS_PUBLIC_URL behind a proxy."""
+    if public := guard.public_url():
+        return public + "/v1"
+    address = guard.bind()
+    host = "127.0.0.1" if address in ("127.0.0.1", "0.0.0.0", "::") else address
+    return f"http://{'[' + host + ']' if ':' in host else host}:{PORT}/v1"
+
+
 UI = ROOT / "ui"
 
 # THE THEME IS A FILE IN THIS REPO. Until 2026-10-19 the colours and faces were
@@ -480,6 +491,65 @@ def _log_change(stamp: str, who: str, alias: str, field: str, old, new) -> None:
 RETIRED_PATH = DATA / "config" / "retired.json"
 
 
+def _drop_from_collections(alias: str, stamp: str, who: str) -> dict[str, list[str]]:
+    """Take a retired alias out of every collection: off `members`, and off
+    `merger` (empty then, which means chosen by verdicts). Otherwise each call
+    to that collection seats a model that is no longer registered. Returns
+    {collection: [the fields it was in]} for retired.json, so Restore can put
+    it back. An unreadable collections.json is left alone (a save would wipe
+    it); the gateway skips a retired member anyway."""
+    if bench_collections.unreadable(bench_collections.FILE):
+        return {}
+    data = bench_collections.load()
+    was: dict[str, list[str]] = {}
+    for name, c in data.items():
+        if not isinstance(c, dict):
+            continue
+        fields = []
+        if alias in (c.get("members") or []):
+            fields.append("members")
+        if c.get("merger") == alias:
+            fields.append("merger")
+        if not fields:
+            continue
+        old = json.loads(json.dumps(c))
+        if "members" in fields:
+            c["members"] = [m for m in c["members"] if m != alias]
+        if "merger" in fields:
+            c["merger"] = ""
+        was[name] = fields
+        _log_change(stamp, who, alias, f"collection {name}", old, c)
+    if was:
+        _write_atomic(bench_collections.FILE, _dump(data))
+    return was
+
+
+def _return_to_collections(alias: str, was: dict, stamp: str, who: str) -> list[str]:
+    """Undo _drop_from_collections for the collections that still exist; a
+    collection deleted since, or one now full (max_calls), is left as it is.
+    Returns the names it went back into."""
+    if not was or bench_collections.unreadable(bench_collections.FILE):
+        return []
+    data, cap = bench_collections.load(), bench_collections.settings()["max_calls"]
+    back = []
+    for name, fields in was.items():
+        c = data.get(name)
+        if not isinstance(c, dict):
+            continue
+        old = json.loads(json.dumps(c))
+        members = c.get("members") or []
+        if "members" in fields and alias not in members and len(members) < cap:
+            c["members"] = members + [alias]
+        if "merger" in fields and not c.get("merger"):
+            c["merger"] = alias
+        if c != old:
+            back.append(name)
+            _log_change(stamp, who, alias, f"collection {name}", old, c)
+    if back:
+        _write_atomic(bench_collections.FILE, _dump(data))
+    return back
+
+
 def _sync_cli() -> str:
     try:
         result = subprocess.run(
@@ -500,7 +570,8 @@ def remove_model(alias: str, typed: str, reason: str, who: str = "ui") -> dict:
     The daily check (`scripts/check_deprecations.py`, via `auto_retire.py`)
     calls this too, with `who="auto"`, when a model is clearly gone; this is
     also the button for a person who knows sooner. It deregisters the alias from the llm registry (and re-syncs),
-    drops it from the role and direct-endpoint configs, records it in
+    drops it from the role and direct-endpoint configs and from every
+    collection (members and merger), records it in
     `config/retired.json` — so the alias is never reused, and onboarding the
     same served model again is a deliberate act — clears that model's alerts,
     and logs the change. It deletes NO evidence: suite outputs, offloads,
@@ -527,11 +598,14 @@ def remove_model(alias: str, typed: str, reason: str, who: str = "ui") -> dict:
         data = decide.load()
         entry = data.pop(alias)
         _write_atomic(decide.FILE, _dump(data))
+        # A decision model is never saved into a collection; a hand edit may have.
+        was = _drop_from_collections(alias, stamp, who)
         retired = load_json(RETIRED_PATH)
         retired[alias] = {"served": served, "tag": directory[alias]["tag"], "role": "decision",
                           "at": stamp, "by": who, "why": reason.strip(),
                           "endpoint": entry["endpoint"], "decision": entry,
-                          **({"ad_hoc": True} if directory[alias]["ad_hoc"] else {})}
+                          **({"ad_hoc": True} if directory[alias]["ad_hoc"] else {}),
+                          **({"collections": was} if was else {})}
         _write_atomic(RETIRED_PATH, _dump(retired))
         roles = roles_config()
         roles.get("models", {}).pop(alias, None)
@@ -540,7 +614,8 @@ def remove_model(alias: str, typed: str, reason: str, who: str = "ui") -> dict:
         _log_change(stamp, who, alias, "removed", served, reason.strip())
         return {"ok": True, "errors": [], "sync": "",
                 "changed": ["config/decision-models.json", "config/retired.json",
-                            "config/model-roles.json"]}
+                            "config/model-roles.json"]
+                           + (["config/collections.json"] if was else [])}
 
     # 1. the registry block — model_id line plus its indented lines
     text = read(YAML_PATH)
@@ -551,17 +626,23 @@ def remove_model(alias: str, typed: str, reason: str, who: str = "ui") -> dict:
                 "errors": [f"expected one registry block for {alias!r}, found {n}; nothing removed"]}
     _write_atomic(YAML_PATH, text.rstrip("\n") + "\n")
     changed.append("config/extra-openai-models.yaml")
-    # 2. the memory of it -- enough to restore it exactly: endpoint, options too
+    # 2. the collections it sat in, so no call seats a model that is gone
+    was = _drop_from_collections(alias, stamp, who)
+    if was:
+        changed.append("config/collections.json")
+    # 3. the memory of it -- enough to restore it exactly: endpoint, options,
+    # collections too
     retired = load_json(RETIRED_PATH)
     options = load_json(OPTIONS_PATH)
     retired[alias] = {"served": served, "tag": directory[alias]["tag"],
                       "role": directory[alias]["role"], "at": stamp, "by": who,
                       "why": reason.strip(), "endpoint": directory[alias]["endpoint"],
                       **({"ad_hoc": True} if directory[alias]["ad_hoc"] else {}),
-                      **({"options": options[alias]} if alias in options else {})}
+                      **({"options": options[alias]} if alias in options else {}),
+                      **({"collections": was} if was else {})}
     _write_atomic(RETIRED_PATH, _dump(retired))
     changed.append("config/retired.json")
-    # 3. configs keyed by alias
+    # 4. configs keyed by alias
     roles = roles_config()
     if alias in roles.get("models", {}) or alias in roles.get("ad_hoc", []):
         roles.get("models", {}).pop(alias, None)
@@ -573,7 +654,7 @@ def remove_model(alias: str, typed: str, reason: str, who: str = "ui") -> dict:
         options.pop(alias)
         _write_atomic(OPTIONS_PATH, _dump(options))
         changed.append("config/model-options.json")
-    # 4. its alerts — the action they asked for has been taken
+    # 5. its alerts — the action they asked for has been taken
     try:
         import alerts as _alerts
         for key in list(_alerts.active()):
@@ -652,13 +733,15 @@ def restore_model(alias: str, who: str = "ui") -> dict:
         if entry.get("ad_hoc"):
             roles["ad_hoc"] = sorted(set(roles.get("ad_hoc", [])) | {alias})
         _write_atomic(ROLES_PATH, _dump(roles))
+        stamp = datetime.datetime.now().astimezone().isoformat(timespec="seconds")
+        back = _return_to_collections(alias, entry.get("collections") or {}, stamp, who)
         del retired[alias]
         _write_atomic(RETIRED_PATH, _dump(retired))
-        _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
-                    who, alias, "restored", None, entry["served"])
+        _log_change(stamp, who, alias, "restored", None, entry["served"])
         return {"ok": True, "errors": [], "sync": "",
                 "changed": ["config/decision-models.json", "config/model-roles.json",
-                            "config/retired.json"]}
+                            "config/retired.json"]
+                           + (["config/collections.json"] if back else [])}
     if entry["served"] in registered():
         return {"ok": False, "changed": [], "sync": "",
                 "errors": [f"{entry['served']} is registered again as"
@@ -682,11 +765,13 @@ def restore_model(alias: str, who: str = "ui") -> dict:
         options[alias] = entry["options"]
         _write_atomic(OPTIONS_PATH, _dump(options))
         changed.append("config/model-options.json")
+    stamp = datetime.datetime.now().astimezone().isoformat(timespec="seconds")
+    if _return_to_collections(alias, entry.get("collections") or {}, stamp, who):
+        changed.append("config/collections.json")
     del retired[alias]
     _write_atomic(RETIRED_PATH, _dump(retired))
     changed.append("config/retired.json")
-    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
-                who, alias, "restored", None, entry["served"])
+    _log_change(stamp, who, alias, "restored", None, entry["served"])
     return {"ok": True, "errors": [], "changed": changed, "sync": _sync_cli()}
 
 
@@ -1247,8 +1332,11 @@ def collections_view() -> dict:
     # collection already uses, so the same idea is spelled the same way.
     used = [t for r in rows for t in r.get("tags") or []]
     suggestions = list(dict.fromkeys(shapes + list(TAG_PURPOSES) + used))
+    # The served name beside each alias: the member search matches either.
+    directory = model_directory()
+    served = {a: directory[a].get("served", "") for a in _chat_aliases()}
     return {"ok": True, "errors": [], "collections": rows, "settings": s,
-            "shapes": shapes, "models": _chat_aliases(), "tag_suggestions": suggestions}
+            "shapes": shapes, "models": _chat_aliases(), "served": served, "tag_suggestions": suggestions}
 
 
 TAG_PURPOSES = ("second-opinion", "security", "long-context", "fast", "free")
@@ -1683,7 +1771,7 @@ def first_run() -> str:
     elif not bench_collections.load():
         step, head, label = "collection", "Make your first collection", "New collection"
         text = ("The models a panel always asks, and what it is for. Callers then ask for"
-                " bench/&lt;name&gt; through the gateway (python gateway.py).")
+                f" bench/&lt;name&gt; through the gateway at {esc(gateway_url())}.")
     else:
         return ""
     return (f"<section class='problems' id='first-run' data-step='{step}' aria-labelledby='first-head'>"
@@ -2333,8 +2421,8 @@ def gateway_card(limit: int = 30) -> str:
             f"<td>{' '.join(chips)}</td>"
             f"<td class='num{' muted' if not row_owed else ''}'>{row_owed}</td></tr>")
     if not records:
-        body.append("<tr><td colspan='6' class='muted'>No gateway requests yet. Run"
-                    " <code>python gateway.py</code> and call a collection.</td></tr>")
+        body.append("<tr><td colspan='6' class='muted'>No gateway requests yet. Call a collection"
+                    f" at <code>{esc(gateway_url())}</code>.</td></tr>")
     body.append("</tbody></table>")
     key = """<dl>
       <dt>Seats</dt><dd>Every model that answered the request, members first, then
@@ -2525,7 +2613,7 @@ def collections_tab() -> str:
     return (
         "<div class='panel form-panel'><h2 id='c-title'>Collections</h2>"
         "<p class='served'>Named groups of models the bench gateway runs together. A caller"
-        " asks for <code>bench/&lt;name&gt;</code> at <code>http://127.0.0.1:8123/v1</code>;"
+        f" asks for <code>bench/&lt;name&gt;</code> at <code>{esc(gateway_url())}</code>;"
         " every member answers, plus explorer seats the bench fills by staleness. Two to four"
         " members is the sweet spot. Open one to edit or delete it.</p>"
         "<div class='list-actions'><button class='btn primary' id='c-new'>New collection</button>"
@@ -2543,7 +2631,12 @@ def collections_tab() -> str:
         " aria-describedby='c-name-rules'></div>"
         "<p class='rules' id='c-name-rules'>Callers ask for <code>bench/&lt;name&gt;</code>. Say what"
         " it is for, not who is in it: members change. Lower-case letters, digits and '-'.</p>"
-        "<label class='field'>Members</label><div id='c-members' class='members'></div>"
+        # Not data-filter: that binds to the whole panel. The same match (any
+        # text, case-insensitive), on the member list only; ticked ones stay shown.
+        "<label class='field' for='c-member-filter'>Members</label>"
+        "<input type='search' id='c-member-filter' class='member-filter' spellcheck='false' autocomplete='off'"
+        " placeholder='Filter by alias or served name' aria-controls='c-members' aria-describedby='c-count'>"
+        "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
         "<label class='field' for='c-shape'>Explorer shape</label><select id='c-shape'></select>"
         "<p class='rules'>Which models may fill the explorer seats. Automatic asks a decision"
@@ -3009,16 +3102,24 @@ def models_page() -> str:
     # the connection (provider) it comes FROM: an OpenRouter model may be on
     # OpenRouter's own endpoint or arrive through Open WebUI's.
     through = {r["alias"]: r["endpoint"] for r in endpoints.registry()}
-    through.update({a: d["endpoint"] for a, d in decide.load().items()})
-    for c in connections_view()["connections"]:
+    decisions = decide.load()
+    through.update({a: d["endpoint"] for a, d in decisions.items()})
+    # Every model, decision models too (connections_view is chat models only,
+    # for what a collection may hold): they are on the bench like any other.
+    by: dict[str, list[str]] = {}
+    for a in model_directory():
+        by.setdefault(catalog.connection_of(a), []).append(a)
+    for c in ({"name": n, "models": sorted(m)} for n, m in sorted(by.items())):
         rows = []
         for a in c["models"]:
             f = catalog.info(a)
             lim = f.get("limits") or {}
             num = lambda v: f"{v:,}" if isinstance(v, int) else "—"          # noqa: E731
             ep = through.get(a, "?")
+            kind = (" " + chip("decision", "", "a decision model: answers typed questions, not chat")
+                    if a in decisions else "")
             rows.append(f"<tr><td><a href='#' class='model' data-alias='{esc(a)}'>"
-                        f"<span class='alias'>{esc(a)}</span></a></td>"
+                        f"<span class='alias'>{esc(a)}</span></a>{kind}</td>"
                         f"<td data-sort='{esc(ep)}'>{chip(ep, 'via', f'reached through the {ep} endpoint')}</td>"
                         f"<td class='num' data-sort='{f.get('context_window') or 0}'>{num(f.get('context_window'))}</td>"
                         f"<td class='num' data-sort='{f.get('max_output') or 0}'>{num(f.get('max_output'))}</td>"
@@ -3254,8 +3355,9 @@ def editor() -> str:
         "<button class='btn primary' id='m-save'>Save</button></div>"
         "<details class='remove' id='m-remove-box'><summary>Retire this model…</summary>"
         "<p class='rules'>Deregisters the alias, records it in <code>retired.json</code> so the"
-        " name is never reused, and clears its problems. Deletes no evidence, and Restore"
-        " brings it back. The daily check also retires models on its own when they are"
+        " name is never reused, takes it out of every collection it is a member or merger of,"
+        " and clears its problems. Deletes no evidence, and Restore brings it back, into the"
+        " collections that still exist. The daily check also retires models on its own when they are"
         " clearly gone.</p>"
         "<label class='field' for='m-reason'>Why</label>"
         "<input id='m-reason' placeholder='e.g. free tier withdrawn 2026-10-15'>"

package/suite/run.py

@@ -826,8 +826,10 @@ def ask_once(binary: str, model: str, prompt: str,
         # mid-stream crash `out.stdout` holds real but TRUNCATED text, so
         # `stdout or stderr` prefers it and the failure becomes invisible --
         # it reads as a model that simply stopped early. The exit code is the
-        # only signal that separates those, so it decides, not the text.
-        if out.returncode != 0:
+        # only signal that separates those, so it decides, not the text --
+        # except when it is only llm failing to log an answer it printed in
+        # full, its logs.db locked by another llm (metrics/client.py).
+        if out.returncode != 0 and not client.llm_log_failed_only(out.stdout, out.stderr):
             detail = (out.stderr or body or "").strip().replace("\n", " ")
             body = f"Error: llm exited {out.returncode}: {detail[:300]}"
     except subprocess.TimeoutExpired:

package/ui/bench.css

@@ -286,6 +286,7 @@ dialog h2 { margin: 0 0 .2rem; font-size: 1.375rem; font-weight: 700; }
 :is(dialog, .form-panel) .members { display: grid; grid-template-columns: repeat(auto-fill, minmax(14rem, 1fr)); gap: 0 1rem;
                   max-height: 14rem; overflow: auto; }
 :is(dialog, .form-panel) .members label.check { margin-top: .35rem; font-size: .875rem; }
+:is(dialog, .form-panel) input.member-filter { font-size: .9375rem; padding: .4rem .6rem; margin-bottom: .2rem; }
 :is(dialog, .form-panel) .members input:disabled + span { color: var(--muted); }
 :is(dialog, .form-panel) .limits { display: flex; flex-wrap: wrap; gap: .3rem .8rem; margin-top: .3rem; font-size: .875rem; }
 :is(dialog, .form-panel) .limits input { width: 6.5rem !important; padding: .2rem .4rem !important; }

package/ui/bench.js

@@ -1138,7 +1138,7 @@
            shape: $('c-shape'), desc: $('c-desc'), tags: $('c-tags'), errors: $('c-errors'), done: $('c-done'),
            head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
            gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
-           del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger')};
+           del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter')};
   var C = null, open = null;         // the collection whose details are showing
   function explorersFor(n) {
     return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
@@ -1149,10 +1149,28 @@
   }
   function recount() {
     var n = picked().length, ex = explorersFor(n);
+    var rows = c.members.querySelectorAll('label.check'), shown = c.members.querySelectorAll('label.check:not([hidden])');
     c.count.textContent = n + ' member' + (n === 1 ? '' : 's') + ' + ' + ex + ' explorer' + (ex === 1 ? '' : 's') +
-      ' = ' + (n + ex) + ' calls per request' + (n && (n < 2 || n > 4) ? ' · two to four members is recommended' : '');
+      ' = ' + (n + ex) + ' calls per request' + (n && (n < 2 || n > 4) ? ' · two to four members is recommended' : '') +
+      (shown.length < rows.length ? ' · showing ' + shown.length + ' of ' + rows.length + ' models' : '');
     c.members.querySelectorAll('input').forEach(function (x) { x.disabled = !x.checked && n >= C.settings.max_calls; });
   }
+  // The member search: the filter boxes' match (any text, case-insensitive),
+  // over alias and served name. It only hides rows, and never a ticked one, so
+  // what is picked is always in view and every count still includes it.
+  function filterMembers() {
+    var q = c.find.value.trim().toLowerCase();
+    c.members.querySelectorAll('label.check').forEach(function (lab) {
+      var box = lab.querySelector('input');
+      lab.hidden = !!q && !box.checked && (box.value + ' ' + (lab.dataset.served || '')).toLowerCase().indexOf(q) === -1;
+    });
+    recount();
+  }
+  c.find.addEventListener('input', filterMembers);
+  // Escape in a box with text clears it; only an empty box lets Escape close the popup.
+  c.find.addEventListener('keydown', function (e) {
+    if (e.key === 'Escape' && c.find.value) { e.preventDefault(); e.stopPropagation(); c.find.value = ''; filterMembers(); }
+  });
   // Suggested tags under the field: a click adds or removes one, and the
   // field stays free for any other.
   function tagList() {
@@ -1188,12 +1206,15 @@
     c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
     c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
     drawTagPicks();
-    c.members.textContent = '';
+    c.members.textContent = ''; c.find.value = '';
     C.models.forEach(function (alias) {
       var box = el('input', {type: 'checkbox', value: alias});
       box.checked = !!row && row.members.indexOf(alias) !== -1;
       box.addEventListener('change', recount);
-      var lab = el('label', {class: 'check'}); lab.appendChild(box); lab.appendChild(el('span', {class: 'mono'}, alias));
+      var served = (C.served || {})[alias] || '';
+      var lab = el('label', {class: 'check', 'data-served': served});
+      if (served && served !== alias) lab.title = served;
+      lab.appendChild(box); lab.appendChild(el('span', {class: 'mono'}, alias));
       c.members.appendChild(lab);
     });
     c.shape.textContent = '';