@wendellresearch-ai/benchwarmers 0.5.4 → 0.5.6

package/serve.py

@@ -554,7 +554,7 @@ def _return_to_collections(alias: str, was: dict, stamp: str, who: str) -> list[
     Returns the names it went back into."""
     if not was or bench_collections.unreadable(bench_collections.FILE):
         return []
-    data, cap = bench_collections.load(), bench_collections.settings()["max_calls"]
+    data, s = bench_collections.load(), bench_collections.settings()
     back = []
     for name, fields in was.items():
         c = data.get(name)
@@ -562,7 +562,7 @@ def _return_to_collections(alias: str, was: dict, stamp: str, who: str) -> list[
             continue
         old = json.loads(json.dumps(c))
         members = c.get("members") or []
-        if "members" in fields and alias not in members and len(members) < cap:
+        if "members" in fields and alias not in members and len(members) < bench_collections.limits(c, s)["max_calls"]:
             c["members"] = members + [alias]
         if "merger" in fields and not c.get("merger"):
             c["merger"] = alias
@@ -698,15 +698,21 @@ def dismiss_alert(key: str) -> dict:
 
 
 def retry_model(alias: str, who: str = "ui") -> dict:
-    """One fresh call to a model on a retiring alert. An answer is written to
+    """One fresh call to a model on Action needed. An answer is written to
     perf/history.csv as evidence (scripts/auto_retire.py reads it, so the
-    clock resets) and the alert is cleared now rather than at the next 5am
-    check. A failure writes nothing: the daily probe keeps the record."""
+    clock resets) and its alerts are cleared now rather than at the next 5am
+    check (alerts.answered). A failure writes nothing: the daily probe keeps
+    the record.
+
+    One request, never the client's retries: a 429's stated wait (UnoRouter's
+    60 s) then a second request spent a minute and another call of a 1-a-minute
+    free tier, and Retry all did that for each model in turn. A person can
+    click again."""
     names = registered()
     if alias not in names.values():
         return {"ok": False, "errors": [f"{alias!r} is not registered"]}
     from run import ask_once, failure_class, llm_binary
-    body, seconds = ask_once(llm_binary(), alias, "Reply with exactly one word: ready", 90)
+    body, seconds = ask_once(llm_binary(), alias, "Reply with exactly one word: ready", 90, retries=0)
     cls = failure_class(body, min_length=1)
     if cls is not None:
         try:                                           # a stated quota or wait still teaches a cooldown
@@ -720,12 +726,10 @@ def retry_model(alias: str, who: str = "ui") -> dict:
                                  f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}",
                                  0.0, 1, 0, 0, 0, "", "retry from the dashboard", ""])
     import alerts as _alerts
-    served = {a: s for s, a in names.items()}.get(alias, "")
-    # Exactly this model's: auto_retire.py raises model:pending:<served> with its alias.
-    cleared = [k for k, a in _alerts.active().items()
-               if a.get("kind") == "retiring" and (a.get("alias") == alias or k == f"model:pending:{served}")]
-    for key in cleared:
-        _alerts.clear(key)
+    # Every alert the answer disproves, not only a retiring one: 2026-11-02,
+    # big-pickle answered and stayed on Action needed as "healthy", its
+    # unrecognised-failure alert still open.
+    cleared = _alerts.answered(alias, {a: s for s, a in names.items()}.get(alias, ""))
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, alias,
                 "retried", None, f"answered in {seconds:.1f}s")
     return {"ok": True, "errors": [], "sync": "",
@@ -1506,19 +1510,25 @@ LONG_CONTEXT = 128_000          # the long-context tag: members known to hold at
 
 
 def save_collection(name: str, members, shape: str, description: str, tags=None,
-                    reply: str = "all", merger: str = "", who: str = "ui") -> dict:
+                    reply: str = "all", merger: str = "", ad_hoc: bool = False, who: str = "ui",
+                    max_calls=None, max_explorers=None) -> dict:
     """Create or replace a collection. Every member must be a chat model on the
     bench (a decision model cannot answer a prompt), no more than max_calls.
     The description and tags are what an orchestrator reads on /v1/models to
-    choose a collection, so they say what it is for."""
+    choose a collection, so they say what it is for. `ad_hoc` takes it off
+    /v1/models: it runs only when a request names it. `max_calls` and
+    `max_explorers`: its own limits; None follows the gateway settings."""
     # The name is what follows "bench/"; one typed with the prefix is taken as meant.
     name, shape = (name or "").strip().removeprefix(bench_collections.PREFIX), (shape or "").strip()
     description = " ".join((description or "").split())
     if isinstance(tags, str):
         tags = tags.split(",")
     tags = list(dict.fromkeys(t.strip().lower() for t in (tags or []) if isinstance(t, str) and t.strip()))
-    s = bench_collections.settings()
-    errors = []
+    own = {k: v for k, v in (("max_calls", max_calls), ("max_explorers", max_explorers)) if v is not None}
+    s = bench_collections.limits(own)
+    errors = [f"{k.replace('_', ' ')} must be a whole number from {bench_collections.BOUNDS[k][0]} to"
+              f" {bench_collections.BOUNDS[k][1]}, or empty for the gateway setting"
+              for k, v in own.items() if not bench_collections.limit_ok(k, v)]
     if not bench_collections.NAME.match(name):
         errors.append("name: 2-32 lower-case letters, digits or '-', saying what it is for,"
                       " e.g. reviewers; callers ask for bench/<name>")
@@ -1529,7 +1539,7 @@ def save_collection(name: str, members, shape: str, description: str, tags=None,
     if not members:
         errors.append("a collection needs at least one member")
     if len(members) > s["max_calls"]:
-        errors.append(f"at most {s['max_calls']} members (max calls per request, in settings)")
+        errors.append(f"at most {s['max_calls']} members (max calls per request)")
     unknown = [m for m in members if m not in known]
     if unknown:
         errors.append(f"not chat models on the bench: {', '.join(unknown)}")
@@ -1553,7 +1563,7 @@ def save_collection(name: str, members, shape: str, description: str, tags=None,
     data = bench_collections.load()
     old = data.get(name)
     data[name] = {"members": members, "shape": shape, "description": description, "tags": tags,
-                  "reply": reply, "merger": merger}
+                  "reply": reply, "merger": merger, **({"ad_hoc": True} if ad_hoc else {}), **own}
     _write_atomic(bench_collections.FILE, _dump(data))
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
                 who, "", f"collection {name}", old, data[name])
@@ -1580,7 +1590,8 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
     if not isinstance(max_explorers, int) or isinstance(max_explorers, bool) or not 0 <= max_explorers <= 2:
         errors.append("max explorers must be 0, 1 or 2 (route.py seats at most two)")
     too_big = [n for n, c in bench_collections.load().items()
-               if isinstance(max_calls, int) and len(c.get("members", [])) > max_calls]
+               if isinstance(max_calls, int) and len(c.get("members", [])) > max_calls
+               and "max_calls" not in bench_collections.own_limits(c)]
     if too_big:
         errors.append(f"these collections have more members than {max_calls}: {', '.join(too_big)}")
     if bench_collections.unreadable(bench_collections.SETTINGS_FILE):
@@ -2129,21 +2140,31 @@ def actions_page(groups: dict, auto: list) -> str:
         shown = re.sub(r"^https?://", "", url).split("?")[0]
         return f" <a href='{esc(url)}' target='_blank' rel='noopener'>{esc(shown)} ↗</a>"
 
+    def why(text: str, url: str, title: str) -> str:
+        return (f"<td class='why' title='{esc(title)}'><div class='said-line'><span class='said'>{esc(text)}</span>"
+                f"{link(url)}</div></td>")
+
     def model_row(r: dict) -> str:
         if not r["alias"]:                             # a notice: what it says, and Got it if it goes once seen
             got = (f"<button class='linkish' data-act='dismiss' data-key='{esc(r['key'])}'>Got it</button>"
                    if r.get("dismiss") else "")
-            return (f"<div class='a-row' data-row><span class='what'>{esc(r['said'])}</span>"
-                    f"<span>{chip(r['label'], r['css'])}</span><span class='why'>{esc(r['title'])}</span>"
-                    f"<span class='acts'>{got}</span></div>")
+            return (f"<tr data-row><td class='what'>{esc(r['said'])}</td><td>{chip(r['label'], r['css'])}</td>"
+                    f"<td class='why'>{esc(r['title'])}</td><td class='acts'>{got}</td></tr>")
         alias = esc(r["alias"])
-        return (f"<div class='a-row' data-row><a href='#' class='model' data-alias='{alias}'>"
-                f"<span class='alias'>{breakable(alias)}</span></a><span>{chip(r['label'], r['css'])}</span>"
-                f"<span class='why' title='{esc(r['title'])}'><span class='said'>{esc(r['said'])}</span>{link(r['url'])}</span>"
-                f"<span class='acts'><button class='linkish' data-act='retry' data-alias='{alias}'"
+        return (f"<tr data-row><td><a href='#' class='model' data-alias='{alias}'><span class='alias'>{breakable(alias)}"
+                f"</span></a></td><td data-sort='{r['days']:03d}{esc(r['label'])}'>{chip(r['label'], r['css'])}</td>"
+                + why(r["said"], r["url"], r["title"])
+                + f"<td class='acts'><button class='linkish' data-act='retry' data-alias='{alias}'"
                 " title='One call now; an answer clears the alert'>Retry</button>"
                 f"<button class='linkish danger' data-act='retire' data-alias='{alias}'"
-                f" data-why='{esc(r['why'] or r['label'])}'>Retire</button></span></div>")
+                f" data-why='{esc(r['why'] or r['label'])}'>Retire</button></td></tr>")
+
+    # One column plan for every table, as on Models, so they line up down the page.
+    cols = "<colgroup><col class='c-model'><col class='c-state'><col><col class='c-acts'></colgroup>"
+
+    def table(first: str, second: str, rows: str) -> str:
+        return (f"<table class='a-table'>{cols}<thead><tr>{th(first)}{th(second)}{th('Reason')}<th></th></tr></thead>"
+                f"<tbody>{rows}</tbody></table>")
 
     def section(conn: str, g: dict) -> str:
         aliases = [r["alias"] for r in g["rows"] if r["alias"]]
@@ -2156,29 +2177,30 @@ def actions_page(groups: dict, auto: list) -> str:
             f"<div class='a-held' title='{esc(h['title'])}'><span class='chip'>held</span><span>Every model"
             f" here is failing, likely the connection (key, credit, prefix), so nothing was retired."
             f" Latest: {esc(h['said'])}{link(h['url'])}</span></div>" for h in g["held"])
-        n = len(g["rows"])
-        return (f"<div class='a-group'><div class='a-head' data-row><span class='conn'>{esc(conn)}</span>"
-                f"<span class='count'>{n} {'notice' if conn == 'Notices' else 'model'}{'s' * (n != 1)}</span>"
-                f"<span class='why'></span><span class='acts'>{both}</span></div>{held}"
-                f"<div class='a-rows'>{''.join(model_row(r) for r in g['rows'])}</div></div>")
+        n, notices = len(g["rows"]), conn == "Notices"
+        return (f"<div class='panel'><div class='a-head' data-row><h3 class='chart-head'>{esc(conn)} · {n}"
+                f" {'notice' if notices else 'model'}{'s' * (n != 1)}</h3><span class='why'></span>"
+                f"<span class='acts'>{both}</span></div>{held}"
+                + table("Notice" if notices else "Model", "Kind" if notices else "State",
+                        "".join(model_row(r) for r in g["rows"])) + "</div>")
 
     tags = load_json(DATA / "config" / "provider-tags.json")      # a retired model's connection, by served name
     gone = "".join(
-        f"<div class='a-row' data-row><a href='#' class='model' data-alias='{esc(alias)}'>"
-        f"<span class='alias'>{breakable(esc(alias))}</span></a><span class='when'>{esc(r['at'][:10])}</span>"
-        f"<span class='why' title='{esc(r.get('why', ''))}'><span class='said'>"
-        f"{esc(tags.get(r.get('served', ''), '').strip('?') or r.get('endpoint') or endpoints.default())}"
-        f" · {esc(said(r.get('why', '').removeprefix('auto: '))[0])}</span></span>"
-        f"<span class='acts'><button class='linkish' data-act='restore' data-alias='{esc(alias)}'>Restore</button></span></div>"
+        f"<tr data-row><td><a href='#' class='model' data-alias='{esc(alias)}'>"
+        f"<span class='alias'>{breakable(esc(alias))}</span></a></td><td class='when'>{esc(r['at'][:10])}</td>"
+        + why(f"{tags.get(r.get('served', ''), '').strip('?') or r.get('endpoint') or endpoints.default()}"
+              f" · {said(r.get('why', '').removeprefix('auto: '))[0]}", "", r.get("why", ""))
+        + f"<td class='acts'><button class='linkish' data-act='restore' data-alias='{esc(alias)}'>Restore</button></td></tr>"
         for alias, r in auto)
     # Folded: a week of automatic retirements is a record to check, not a list of things to do.
-    folded = (f"<details class='a-retired'><summary>Retired automatically this week · {len(auto)}"
-              f" <span class='muted'>— each can be restored</span></summary><div class='a-rows'>{gone}</div></details>"
-              if auto else "")
-    body = "".join(section(conn, g) for conn, g in groups.items()) or "<p class='a-none'>Nothing open.</p>"
+    folded = (f"<div class='panel'><details class='a-retired'><summary><h3 class='chart-head'>Retired automatically"
+              f" this week · {len(auto)}</h3> <span class='muted'>each can be restored</span></summary>"
+              f"{table('Model', 'Retired', gone)}</details></div>" if auto else "")
+    body = "".join(section(conn, g) for conn, g in groups.items())
     return ("<div class='panel'><h2>Action needed</h2><p class='sub'>States that will not fix themselves, by the"
             " connection they come from. A reason's full error is on hover, and a name opens the model."
-            f" Retry spends one call; an answer clears the alert.</p>{body}{folded}</div>")
+            " Retry spends one call; an answer clears the alert.</p>"
+            + ("" if body else "<p class='a-none'>Nothing open.</p>") + f"</div>{body}{folded}")
 
 
 CAP_LABELS = {"tools": "tools", "vision": "vision", "structured_output": "json", "reasoning": "reasons"}
@@ -3116,8 +3138,8 @@ def _discover_rows() -> list[dict]:
     (Settings' free-only-in-their-own-app switches)."""
     import discover
     import sources
-    paths = [sources.CACHE / f"{n}.json" for n in sources.FEEDS] + [endpoints.YAML, endpoints.FILE,
-                                                                     endpoints.CONNECTIONS]
+    paths = [sources.CACHE / f"{n}.json" for n in [*sources.FEEDS, discover.LISTS]] + [
+        endpoints.YAML, endpoints.FILE, endpoints.CONNECTIONS]
     stamp = tuple(p.stat().st_mtime if p.exists() else 0 for p in paths)
     if _DISCOVER.get("stamp") != stamp:
         _DISCOVER.update(stamp=stamp, rows=discover.build())
@@ -3142,6 +3164,17 @@ def discover_model(provider: str, model: str) -> dict:
     return {"ok": True, "errors": errors, "model": row, "routes": routes}
 
 
+def discover_refresh(who: str = "ui") -> dict:
+    """Discover's Refresh: every list fetched now (discover.refresh), for a
+    model too new to have waited for the daily fetch. A list that would not
+    come keeps its last copy and is said; the rest are still used."""
+    import discover
+    said = discover.refresh(force=True)
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "", "discover-refresh",
+                None, "; ".join(said) or "every list fetched")
+    return {"ok": True, "errors": said}
+
+
 def discover_card() -> str:
     """Models worth adding, from FreeLLMAPI and models.dev (metrics/discover.py),
     deduped. Read from the daily cache, so the page never waits on the network;
@@ -3158,7 +3191,7 @@ def discover_card() -> str:
         caps = [c for c, on in (("tools", r["tools"]), ("vision", r["vision"]),
                                 ("reasons", r["reasoning"])) if on]
         cost = r.get("cost") or {}
-        price = ("free" if r["free"] else
+        price = ("free" if r["free"] else "unknown" if r["free"] is None else
                  f"{cost.get('input', '?')} / {cost.get('output', '?')}" if cost else "paid")
         lim = r.get("limits") or {}
         lim_text = " · ".join([f"{v:,} {k}" for k, v in lim.items()]
@@ -3192,9 +3225,17 @@ def discover_card() -> str:
             f"<td>{add}</td></tr>")
     body.append("</tbody></table>")
     free = sum(1 for r in rows if r["free"])
+    import discover
+    import sources
+    ages = [a for a in map(sources.age_s, [*sources.FEEDS, discover.LISTS]) if a is not None]
     toolbar = ("<div class='toolbar'><input type='search' data-filter placeholder='Filter by model,"
                " provider or reach' aria-label='Filter rows'>"
-               "<label><input type='checkbox' data-paid> Show paid (connected providers)</label></div>")
+               "<label><input type='checkbox' data-paid> Show paid and unpriced (connected providers)</label>"
+               "<button class='btn small' type='button' id='d-refresh' title='Fetch FreeLLMAPI, models.dev and"
+               " every Open WebUI connection&#39;s own model list now, instead of waiting for the daily fetch'>"
+               "Refresh lists</button><span class='muted' id='d-refreshed'>"
+               + ("never fetched" if not ages else f"oldest list {max(ages) / 60:.0f} minutes old" if max(ages) < 5400
+                  else f"oldest list {max(ages) / 3600:.0f} hours old") + "</span></div>")
     key = """<dl>
       <dt>Rank</dt><dd>FreeLLMAPI's smartest-first order (1 is the smartest it lists); within
         each reach the list follows it. Hover for its fastest-first rank and size. A dash where
@@ -3207,7 +3248,8 @@ def discover_card() -> str:
         for its base URL); <em>other</em> a native API or an unvetted reseller.</dd>
       <dt>Price</dt><dd>Free if FreeLLMAPI lists it (a catalog of free tiers) or models.dev
         prices it at 0: models.dev shows the PAID price even where a free tier exists. Otherwise
-        input / output per million tokens.</dd>
+        input / output per million tokens. <em>Unknown</em> where only the provider's own list has
+        the model and it says nothing of price; those are behind <em>Show paid and unpriced</em>.</dd>
       <dt>Limits</dt><dd>The free tier's, from FreeLLMAPI: requests and tokens per minute (rpm,
         tpm) and per day (rpd, tpd), and roughly how many tokens a month that allows.</dd>
       <dt>New</dt><dd>First seen in either list in the last 7 days.</dd>
@@ -3225,11 +3267,13 @@ def discover_card() -> str:
       <dt>Sources</dt><dd>FreeLLMAPI's free catalog first: where it lists a model, its name,
         provider, context, tools, limits, ranks and quirks lead, and the popup links its
         freellmapi.co page. models.dev fills the rest and alone has the paid models, prices,
-        base URLs and modalities. A model one provider offers in both is shown once. Both are
-        refreshed at most daily.</dd>
+        base URLs and modalities; the popup links its page for the model. A model one provider
+        offers in both is shown once. Then each Open WebUI connection's own model list, for
+        what the provider serves that neither catalog has yet. All are fetched daily;
+        <em>Refresh lists</em> fetches them now.</dd>
     </dl>"""
-    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI"
-            f" and models.dev, deduped; {hidden:,} paid models on providers you have no route"
+    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI,"
+            f" models.dev and your providers' own lists, deduped; {hidden:,} paid models on providers you have no route"
             f" to are left out. Click a model for everything known about it and the routes it can be"
             f" added by.</p>{toolbar}"
             f"<div class='scroll'>{''.join(body)}</div>"
@@ -3271,7 +3315,7 @@ def collections_tab() -> str:
         " as <code>code</code> or <code>prose</code> the roles it does not admit; <code>free</code> hides"
         " the paid ones. Click a suggestion or type your own: comma-separated lower-case words, up to 8.</p>"
         # Not data-filter: that binds to the whole panel. The same match (any
-        # text, case-insensitive), on the member list only; ticked ones stay shown.
+        # text, case-insensitive), on the member list only; the seats show who is ticked.
         "<label class='field' for='c-member-filter'>Members</label>"
         # The panel as it will sit, drawn as the Collections list draws it
         # (bench.js recount): a seat per ticked member, explorer seats dashed.
@@ -3282,6 +3326,15 @@ def collections_tab() -> str:
         "<option value=''>Every provider</option></select></div>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
+        "<div class='settings'>"
+        "<div><label class='field' for='c-max-calls'>Max calls per request</label>"
+        "<input id='c-max-calls' type='number' min='1' max='10'></div>"
+        "<div><label class='field' for='c-max-explorers'>Max explorers</label>"
+        "<input id='c-max-explorers' type='number' min='0' max='2'></div>"
+        "</div>"
+        "<p class='rules'>Empty follows the Gateway settings, shown greyed. A member out of quota stays"
+        " on the collection and an explorer sits in for it until the quota is back. Max explorers 0"
+        " means only the members ever run, stand-ins included.</p>"
         "<label class='field' for='c-shape'>Explorer shape</label><select id='c-shape'></select>"
         "<p class='rules'>Which models may fill the explorer seats. Automatic asks a decision"
         " model to label each prompt when one is onboarded, else general.</p>"
@@ -3300,6 +3353,10 @@ def collections_tab() -> str:
         "<label class='field' for='c-desc'>Description</label>"
         "<textarea id='c-desc' rows='2' maxlength='300'"
         " placeholder='e.g. Adversarial code review: finding bugs and broken guarantees in a diff'></textarea>"
+        "<label class='check'><input type='checkbox' id='c-ad-hoc'> Ad hoc: only when called by name</label>"
+        "<p class='rules'>Left off <code>GET /v1/models</code> and the list agents choose from, so an"
+        " agent picking for itself never spends it. For the expensive ones you decide to run: say"
+        " <code>bench/&lt;name&gt;</code> and it runs. Not a lock: anyone who knows the name can call it.</p>"
         "<div class='errors' id='c-errors' role='alert'></div><div class='done' id='c-done'></div>"
         "<div class='actions'><button class='btn danger' id='c-delete' hidden>Delete collection</button>"
         "<span class='spacer'></span><button class='btn' id='c-close'>Close</button>"
@@ -3313,7 +3370,8 @@ def collections_tab() -> str:
         "<input id='g-max-explorers' type='number' min='0' max='2'></div>"
         "</div>"
         "<p class='rules'>Explorers = min(max explorers, max calls − members): with 5 and 2, a"
-        " 2- or 3-model collection gets 2, a 4-model one 1, a 5-model one none. A request that"
+        " 2- or 3-model collection gets 2, a 4-model one 1, a 5-model one none. These apply to"
+        " every collection that does not set its own in its dialog. A request that"
         " names no collection is refused with each one's description and tags, so the caller"
         " can choose.</p>"
         "<div class='errors' id='g-errors' role='alert'></div><div class='done' id='g-done'></div>"
@@ -4423,6 +4481,7 @@ class Handler(gateway.Handler):
                "/api/remove": lambda p: remove_model(str(p.get("alias", "")), str(p.get("reason", ""))),
                "/api/restore": lambda p: restore_model(str(p.get("alias", ""))),
                "/api/retry": lambda p: retry_model(str(p.get("alias") or "")),
+               "/api/discover/refresh": lambda p: discover_refresh(),
                "/api/discover/add": lambda p: discover_add(str(p.get("connection") or ""),
                                                            str(p.get("model") or ""),
                                                            endpoint=str(p.get("endpoint") or ""),
@@ -4455,7 +4514,9 @@ class Handler(gateway.Handler):
                                                             str(p.get("shape") or ""),
                                                             str(p.get("description") or ""),
                                                             p.get("tags"), str(p.get("reply") or ""),
-                                                            str(p.get("merger") or "")),
+                                                            str(p.get("merger") or ""), p.get("ad_hoc") is True,
+                                                            max_calls=p.get("max_calls"),
+                                                            max_explorers=p.get("max_explorers")),
                "/api/collection/remove": lambda p: delete_collection(str(p.get("name") or "")),
                "/api/model-facts": lambda p: save_model_facts(str(p.get("alias") or ""), p.get("facts")),
                "/api/model-options": lambda p: save_model_options(str(p.get("alias") or ""), p.get("max_tokens")),

package/suite/run.py

@@ -920,9 +920,11 @@ def _child_env() -> dict:
 
 
 def ask_once(binary: str, model: str, prompt: str,
-             timeout: int, images: list[str] | None = None) -> tuple[str, float]:
+             timeout: int, images: list[str] | None = None, retries: int | None = None) -> tuple[str, float]:
     # BENCH_CLIENT=builtin: the same call without the `llm` subprocess, giving
-    # the same text for the same failure (metrics/client.py).
+    # the same text for the same failure (metrics/client.py). `retries`: the
+    # built-in client's tries after a 429, 5xx or connection error (its two
+    # by default); the llm backend keeps its SDK's own.
     if str(ROOT / "metrics") not in sys.path:
         sys.path.insert(0, str(ROOT / "metrics"))
     import client
@@ -935,7 +937,7 @@ def ask_once(binary: str, model: str, prompt: str,
                           purpose="vision task")
         return client.output(res, timeout), res["seconds"]
     if client.backend(model) == "builtin":
-        res = client.chat(model, prompt, timeout=timeout)
+        res = client.chat(model, prompt, timeout=timeout, retries=retries)
         return client.output(res, timeout), res["seconds"]
     started = time.monotonic()
     try:

package/ui/bench.css

@@ -115,16 +115,11 @@ main { max-width: var(--page, 104rem); margin: 0 auto; padding: 0 var(--gutter)
 .problems .head { display: flex; align-items: baseline; gap: .5rem; }
 .problems .head h2 { margin-bottom: .25rem; }
 
-/* Action needed: its own page, by connection. Each model is one line of a
-   grid shared by its group (name, state, the provider's words, actions), so
-   the columns line up down the group. */
+/* Action needed: its own page, a panel per connection as on Models, each a
+   table (name, state, the provider's words, actions). */
 .a-line { display: flex; flex-wrap: wrap; align-items: baseline; gap: .25rem .6rem; margin: 1.4rem 1.6rem 0;
           padding: .45rem .75rem; background: var(--err-ground); border-radius: var(--radius); }
-.a-group { padding: 1rem 0 .75rem; border-top: 1px solid var(--line); }
-.a-group:first-of-type { border-top: 0; padding-top: .5rem; }
-.a-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: .25rem .6rem; margin-bottom: .4rem; }
-.a-head .conn { font-size: 1.0625rem; font-weight: 700; }
-.a-head .count { color: var(--muted); }
+.a-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: .25rem .6rem; }
 .a-head .why { font-size: .875rem; }
 .a-head .why:empty { display: none; }
 .a-head .acts { margin-left: auto; display: flex; gap: .5rem; }
@@ -132,26 +127,26 @@ main { max-width: var(--page, 104rem); margin: 0 auto; padding: 0 var(--gutter)
           background: var(--warn-ground); border-radius: var(--radius); font-size: .9375rem; overflow-wrap: anywhere; }
 .a-held .chip { flex: none; background: var(--ink); color: var(--paper); }
 .a-held a { color: var(--ink); font-weight: 500; }
-/* The same fixed columns in every group and the retired list, so state,
-   words and actions line up down the whole page whatever each group's
-   longest name; a long name wraps in its column. */
+/* The same fixed columns in every table, so state, words and actions line
+   up down the whole page whatever each group's longest name; a long name
+   wraps in its column. */
 #tab-needs > .panel { container-type: inline-size; }
-.a-rows { display: grid; grid-template-columns: minmax(0, 32%) 8.25rem minmax(0, 1fr) 6.25rem;
-          gap: 0 1rem; align-items: baseline; }
-.a-row { display: contents; }
-.a-row > * { padding: .3rem 0; border-top: 1px solid color-mix(in oklab, var(--line) 55%, transparent); }
-.a-row:first-child > * { border-top: 0; }
-.a-row .alias, .a-row .what { font-weight: 500; overflow-wrap: anywhere; }   /* a name breaks at its dots first (<wbr>) */
-.a-row .why, .a-row .when { color: var(--muted); font-size: .875rem; overflow-wrap: anywhere; }
+table.a-table { table-layout: fixed; }
+table.a-table col.c-model { width: 30%; }
+table.a-table col.c-state { width: 9.5rem; }
+table.a-table col.c-acts { width: 7.5rem; }
+.a-table .alias, .a-table .what { font-weight: 500; overflow-wrap: anywhere; }   /* a name breaks at its dots first (<wbr>) */
+.a-table .why, .a-table .when { color: var(--muted); font-size: .875rem; overflow-wrap: anywhere; }
 /* One line a model: the provider's words cut to the column, whole on hover; the link gives way last. */
-.a-row .why:has(> .said) { display: flex; gap: .5rem; min-width: 0; }
-.a-row .said { flex: 0 1 auto; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
-.a-row .why > a { flex: none; max-width: 50%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
-.a-row .acts { display: flex; gap: .9rem; font-size: .875rem; }
+.a-table .said-line { display: flex; gap: .5rem; min-width: 0; }
+.a-table .said { flex: 0 1 auto; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.a-table .said-line > a { flex: none; max-width: 50%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.a-table td.acts { white-space: nowrap; font-size: .875rem; }
+.a-table td.acts .linkish + .linkish { margin-left: .9rem; }
 .linkish.danger { color: var(--err); }
-.a-retired { margin-top: .75rem; border-top: 1px solid var(--line); padding-top: .5rem; }
-.a-retired > summary { cursor: pointer; font-weight: 500; padding: .35rem 0; }
-.a-retired > summary .muted { color: var(--muted); font-weight: 400; }
+.a-retired > summary { cursor: pointer; }
+.a-retired > summary h3 { display: inline; }
+.a-retired > summary .muted { color: var(--muted); }
 .a-none { color: var(--muted); }
 
 /* pages: a grouped sidebar (Observe, Gateway, Control); the group names and
@@ -550,13 +545,14 @@ dialog .pick { display: flex; gap: 1.5rem; }
    state on one line, the words and actions under them. No gap between name
    and state, so the line above a model runs unbroken. */
 @container (max-width: 46rem) {
-  .a-rows { grid-template-columns: minmax(0, 1fr) max-content; column-gap: 0; }
-  .a-row > :nth-child(2) { padding-left: .75rem; }
-  .a-row .why, .a-row .acts { grid-column: 1 / -1; padding-top: 0; border-top: 0; }
-  .a-row .acts { padding-bottom: .5rem; }
-  .a-row .why:has(> .said) { flex-wrap: wrap; }
-  .a-row .said { white-space: normal; }             /* the height to read it whole */
-  .a-row .why > a { max-width: none; }
+  .a-table thead { display: none; }
+  .a-table, .a-table tbody { display: block; }
+  .a-table tr { display: grid; grid-template-columns: minmax(0, 1fr) max-content; border-bottom: 1px solid var(--line); }
+  .a-table td { border-bottom: 0; }
+  .a-table td.why, .a-table td.acts { grid-column: 1 / -1; padding-top: 0; }
+  .a-table .said-line { flex-wrap: wrap; }
+  .a-table .said { white-space: normal; }           /* the height to read it whole */
+  .a-table .said-line > a { max-width: none; }
 }
 @media (prefers-reduced-motion: reduce) {
   * { transition: none !important; }

package/ui/bench.js

@@ -766,7 +766,8 @@
   var dd = $('discover-dialog');
   function price(x) {
     var c = x.cost || {};
-    return x.free ? 'free' : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
+    return x.free ? 'free' : x.free == null ? 'price unknown'
+      : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
   }
   function limits(l) {
     return Object.keys(l || {}).map(function (k) { return Number(l[k]).toLocaleString() + ' ' + k; }).join(' · ');
@@ -835,13 +836,15 @@
         if (res.errors.length) listErrors($('d-errors'), res.errors);   // a hub that could not be asked
         var r = res.model, p = $('d-profile'), list = $('d-routes');
         $('d-title').textContent = r.name;
-        // Each source a link: FreeLLMAPI's page for the model where its sitemap has one, else its catalog.
+        // Each source a link to its page for the model: FreeLLMAPI's where its sitemap has one, models.dev's by its
+        // canonical id. The provider's own list has no page, only its connection.
         var sv = $('d-served');
         sv.textContent = r.provider_name + ' · ' + r.model + ' · listed by ';
         r.sources.forEach(function (s, i) {
-          var fl = s === 'freellmapi';
           if (i) sv.appendChild(document.createTextNode(' and '));
-          sv.appendChild(el('a', {href: fl ? r.page || 'https://freellmapi.co/models' : 'https://models.dev/',
+          if (s === 'provider') { sv.appendChild(document.createTextNode(r.provider_name + '\'s own list (' + r.connection + ' connection)')); return; }
+          var fl = s === 'freellmapi';
+          sv.appendChild(el('a', {href: fl ? r.page || 'https://freellmapi.co/models' : r.md_page || 'https://models.dev/',
                                   target: '_blank', rel: 'noopener'}, fl ? 'FreeLLMAPI' : 'models.dev'));
         });
         if (r.on_bench_as) sv.appendChild(document.createTextNode(' · on the bench as ' + r.on_bench_as));
@@ -905,6 +908,16 @@
     e.preventDefault(); openDiscover(row.dataset.provider, row.dataset.model);
   });
   $('d-close').addEventListener('click', function () { dd.close(); });
+  // Refresh lists: every list fetched now (serve.discover_refresh), then the page with them.
+  var dr = $('d-refresh');
+  if (dr) dr.addEventListener('click', function () {
+    var said = $('d-refreshed');
+    dr.disabled = true; dr.textContent = 'Fetching…'; said.textContent = 'about half a minute';
+    post('/api/discover/refresh', {}).then(function (res) {
+      said.textContent = (res.errors.length ? res.errors.join(' · ') + ' · ' : '') + 'Reloading…';
+      setTimeout(function () { location.reload(); }, res.errors.length ? 4000 : 500);
+    }).catch(function (err) { said.textContent = String(err); dr.disabled = false; dr.textContent = 'Refresh lists'; });
+  });
   function follow(log, title, before) {
     runView(title, before + 'Running in the background; closing this does not stop it. Log: runs/' + log);
     timer = setInterval(function () {
@@ -1331,10 +1344,14 @@
            head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
            gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
            del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
-           seats: $('c-seats'), prov: $('c-member-provider')};
+           seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
+           ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
   var C = null, open = null;         // the collection whose details are showing
+  // What the dialog's two limits come to: empty follows the gateway settings.
+  function limit(input, fallback) { return input.value === '' ? fallback : parseInt(input.value, 10); }
+  function maxCalls() { return limit(c.ownCalls, C.settings.max_calls); }
   function explorersFor(n) {
-    return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
+    return Math.max(0, Math.min(limit(c.ownExp, C.settings.max_explorers), maxCalls() - n));
   }
   function picked() {
     return Array.prototype.filter.call(c.members.querySelectorAll('input'), function (x) { return x.checked; })
@@ -1356,7 +1373,7 @@
         return ' · ' + t.name + ' is ticked on ' + t.on.join(' and ') + ': one model agrees with itself, it does not check itself';
       }).join('');
     c.members.querySelectorAll('input').forEach(function (x) {
-      x.disabled = !x.checked && (n >= C.settings.max_calls || !!x.dataset.blocked);
+      x.disabled = !x.checked && (n >= maxCalls() || !!x.dataset.blocked);
     });
     drawSeats(ex);
   }
@@ -1435,9 +1452,9 @@
   }
   // The member search: the filter boxes' match (any text, case-insensitive),
   // over the model's name and each chip's alias, served name and provider,
-  // and the provider select beside it. It only hides chips, and never a ticked
-  // one, so what is picked is always in view and every count still includes
-  // it; a row goes when all its chips do.
+  // and the provider select beside it. It only hides chips, ticked ones too:
+  // the seats above the list always show what is picked, each with its ×, and
+  // every count still includes it. A row goes when all its chips do.
   function filterMembers() {
     var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
     c.members.querySelectorAll('.m-row').forEach(function (row) {
@@ -1446,8 +1463,8 @@
         var box = lab.querySelector('input');
         var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
                     lab.textContent).toLowerCase();
-        lab.hidden = !box.checked && (!!lab.dataset.paid || !named && text.indexOf(q) === -1
-                                      || !!prov && (fact(box.value).provider || '') !== prov);
+        lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
+                     || !!prov && (fact(box.value).provider || '') !== prov;
         any = any || !lab.hidden;
       });
       row.hidden = !any;
@@ -1483,6 +1500,8 @@
   function showMerger() { $('c-merger-row').hidden = c.reply.value !== 'merged'; }
   c.reply.addEventListener('change', showMerger);
   c.shape.addEventListener('change', recount);          // the explorer seats say their shape
+  c.ownCalls.addEventListener('input', recount);
+  c.ownExp.addEventListener('input', recount);
   // A collection opens in a popup, as every other edit on the dashboard does:
   // the one clicked, or a new one from New collection.
   function closeForm() {
@@ -1497,6 +1516,10 @@
     if (!c.form.open) c.form.showModal();
     c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
     c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
+    c.adHoc.checked = !!row && row.ad_hoc;
+    c.ownCalls.value = row && row.max_calls !== undefined ? row.max_calls : '';
+    c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
+    c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
     drawTagPicks();
     c.members.textContent = ''; c.find.value = '';
     c.prov.length = 1;                                 // "Every provider", then each one a model is on
@@ -1565,6 +1588,7 @@
         var left = el('div');
         var title = el('div', {class: 'top'});
         title.appendChild(el('span', {class: 'alias'}, 'bench/' + row.name));
+        if (row.ad_hoc) title.appendChild(el('span', {class: 'chip'}, 'ad hoc: only when called by name'));
         row.tags.forEach(function (t) { title.appendChild(el('span', {class: 'chip'}, t)); });
         left.appendChild(title);
         if (row.description) left.appendChild(el('p', {class: 'desc'}, row.description));
@@ -1631,7 +1655,9 @@
     var name = c.name.value.trim().replace(/^bench\//, '');
     post('/api/collection', {name: name, members: picked(), shape: c.shape.value, description: c.desc.value,
                              tags: c.tags.value.split(','), reply: c.reply.value,
-                             merger: c.merger.value}).then(function (r) {
+                             merger: c.merger.value, ad_hoc: c.adHoc.checked,
+                             max_calls: c.ownCalls.value === '' ? null : Number(c.ownCalls.value),
+                             max_explorers: c.ownExp.value === '' ? null : Number(c.ownExp.value)}).then(function (r) {
       if (!r.ok) { listErrors(c.errors, r.errors); return; }
       finished('Saved bench/' + name + '.');
     }).catch(function (err) { c.errors.textContent = String(err); });