@wendellresearch-ai/benchwarmers 0.5.5 → 0.5.6

package/serve.py

@@ -554,7 +554,7 @@ def _return_to_collections(alias: str, was: dict, stamp: str, who: str) -> list[
     Returns the names it went back into."""
     if not was or bench_collections.unreadable(bench_collections.FILE):
         return []
-    data, cap = bench_collections.load(), bench_collections.settings()["max_calls"]
+    data, s = bench_collections.load(), bench_collections.settings()
     back = []
     for name, fields in was.items():
         c = data.get(name)
@@ -562,7 +562,7 @@ def _return_to_collections(alias: str, was: dict, stamp: str, who: str) -> list[
             continue
         old = json.loads(json.dumps(c))
         members = c.get("members") or []
-        if "members" in fields and alias not in members and len(members) < cap:
+        if "members" in fields and alias not in members and len(members) < bench_collections.limits(c, s)["max_calls"]:
             c["members"] = members + [alias]
         if "merger" in fields and not c.get("merger"):
             c["merger"] = alias
@@ -698,10 +698,11 @@ def dismiss_alert(key: str) -> dict:
 
 
 def retry_model(alias: str, who: str = "ui") -> dict:
-    """One fresh call to a model on a retiring alert. An answer is written to
+    """One fresh call to a model on Action needed. An answer is written to
     perf/history.csv as evidence (scripts/auto_retire.py reads it, so the
-    clock resets) and the alert is cleared now rather than at the next 5am
-    check. A failure writes nothing: the daily probe keeps the record.
+    clock resets) and its alerts are cleared now rather than at the next 5am
+    check (alerts.answered). A failure writes nothing: the daily probe keeps
+    the record.
 
     One request, never the client's retries: a 429's stated wait (UnoRouter's
     60 s) then a second request spent a minute and another call of a 1-a-minute
@@ -725,12 +726,10 @@ def retry_model(alias: str, who: str = "ui") -> dict:
                                  f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}", f"{seconds:.2f}",
                                  0.0, 1, 0, 0, 0, "", "retry from the dashboard", ""])
     import alerts as _alerts
-    served = {a: s for s, a in names.items()}.get(alias, "")
-    # Exactly this model's: auto_retire.py raises model:pending:<served> with its alias.
-    cleared = [k for k, a in _alerts.active().items()
-               if a.get("kind") == "retiring" and (a.get("alias") == alias or k == f"model:pending:{served}")]
-    for key in cleared:
-        _alerts.clear(key)
+    # Every alert the answer disproves, not only a retiring one: 2026-11-02,
+    # big-pickle answered and stayed on Action needed as "healthy", its
+    # unrecognised-failure alert still open.
+    cleared = _alerts.answered(alias, {a: s for s, a in names.items()}.get(alias, ""))
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, alias,
                 "retried", None, f"answered in {seconds:.1f}s")
     return {"ok": True, "errors": [], "sync": "",
@@ -1511,19 +1510,25 @@ LONG_CONTEXT = 128_000          # the long-context tag: members known to hold at
 
 
 def save_collection(name: str, members, shape: str, description: str, tags=None,
-                    reply: str = "all", merger: str = "", who: str = "ui") -> dict:
+                    reply: str = "all", merger: str = "", ad_hoc: bool = False, who: str = "ui",
+                    max_calls=None, max_explorers=None) -> dict:
     """Create or replace a collection. Every member must be a chat model on the
     bench (a decision model cannot answer a prompt), no more than max_calls.
     The description and tags are what an orchestrator reads on /v1/models to
-    choose a collection, so they say what it is for."""
+    choose a collection, so they say what it is for. `ad_hoc` takes it off
+    /v1/models: it runs only when a request names it. `max_calls` and
+    `max_explorers`: its own limits; None follows the gateway settings."""
     # The name is what follows "bench/"; one typed with the prefix is taken as meant.
     name, shape = (name or "").strip().removeprefix(bench_collections.PREFIX), (shape or "").strip()
     description = " ".join((description or "").split())
     if isinstance(tags, str):
         tags = tags.split(",")
     tags = list(dict.fromkeys(t.strip().lower() for t in (tags or []) if isinstance(t, str) and t.strip()))
-    s = bench_collections.settings()
-    errors = []
+    own = {k: v for k, v in (("max_calls", max_calls), ("max_explorers", max_explorers)) if v is not None}
+    s = bench_collections.limits(own)
+    errors = [f"{k.replace('_', ' ')} must be a whole number from {bench_collections.BOUNDS[k][0]} to"
+              f" {bench_collections.BOUNDS[k][1]}, or empty for the gateway setting"
+              for k, v in own.items() if not bench_collections.limit_ok(k, v)]
     if not bench_collections.NAME.match(name):
         errors.append("name: 2-32 lower-case letters, digits or '-', saying what it is for,"
                       " e.g. reviewers; callers ask for bench/<name>")
@@ -1534,7 +1539,7 @@ def save_collection(name: str, members, shape: str, description: str, tags=None,
     if not members:
         errors.append("a collection needs at least one member")
     if len(members) > s["max_calls"]:
-        errors.append(f"at most {s['max_calls']} members (max calls per request, in settings)")
+        errors.append(f"at most {s['max_calls']} members (max calls per request)")
     unknown = [m for m in members if m not in known]
     if unknown:
         errors.append(f"not chat models on the bench: {', '.join(unknown)}")
@@ -1558,7 +1563,7 @@ def save_collection(name: str, members, shape: str, description: str, tags=None,
     data = bench_collections.load()
     old = data.get(name)
     data[name] = {"members": members, "shape": shape, "description": description, "tags": tags,
-                  "reply": reply, "merger": merger}
+                  "reply": reply, "merger": merger, **({"ad_hoc": True} if ad_hoc else {}), **own}
     _write_atomic(bench_collections.FILE, _dump(data))
     _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
                 who, "", f"collection {name}", old, data[name])
@@ -1585,7 +1590,8 @@ def save_gateway_settings(max_calls, max_explorers, who: str = "ui") -> dict:
     if not isinstance(max_explorers, int) or isinstance(max_explorers, bool) or not 0 <= max_explorers <= 2:
         errors.append("max explorers must be 0, 1 or 2 (route.py seats at most two)")
     too_big = [n for n, c in bench_collections.load().items()
-               if isinstance(max_calls, int) and len(c.get("members", [])) > max_calls]
+               if isinstance(max_calls, int) and len(c.get("members", [])) > max_calls
+               and "max_calls" not in bench_collections.own_limits(c)]
     if too_big:
         errors.append(f"these collections have more members than {max_calls}: {', '.join(too_big)}")
     if bench_collections.unreadable(bench_collections.SETTINGS_FILE):
@@ -3132,8 +3138,8 @@ def _discover_rows() -> list[dict]:
     (Settings' free-only-in-their-own-app switches)."""
     import discover
     import sources
-    paths = [sources.CACHE / f"{n}.json" for n in sources.FEEDS] + [endpoints.YAML, endpoints.FILE,
-                                                                     endpoints.CONNECTIONS]
+    paths = [sources.CACHE / f"{n}.json" for n in [*sources.FEEDS, discover.LISTS]] + [
+        endpoints.YAML, endpoints.FILE, endpoints.CONNECTIONS]
     stamp = tuple(p.stat().st_mtime if p.exists() else 0 for p in paths)
     if _DISCOVER.get("stamp") != stamp:
         _DISCOVER.update(stamp=stamp, rows=discover.build())
@@ -3158,6 +3164,17 @@ def discover_model(provider: str, model: str) -> dict:
     return {"ok": True, "errors": errors, "model": row, "routes": routes}
 
 
+def discover_refresh(who: str = "ui") -> dict:
+    """Discover's Refresh: every list fetched now (discover.refresh), for a
+    model too new to have waited for the daily fetch. A list that would not
+    come keeps its last copy and is said; the rest are still used."""
+    import discover
+    said = discover.refresh(force=True)
+    _log_change(datetime.datetime.now().astimezone().isoformat(timespec="seconds"), who, "", "discover-refresh",
+                None, "; ".join(said) or "every list fetched")
+    return {"ok": True, "errors": said}
+
+
 def discover_card() -> str:
     """Models worth adding, from FreeLLMAPI and models.dev (metrics/discover.py),
     deduped. Read from the daily cache, so the page never waits on the network;
@@ -3174,7 +3191,7 @@ def discover_card() -> str:
         caps = [c for c, on in (("tools", r["tools"]), ("vision", r["vision"]),
                                 ("reasons", r["reasoning"])) if on]
         cost = r.get("cost") or {}
-        price = ("free" if r["free"] else
+        price = ("free" if r["free"] else "unknown" if r["free"] is None else
                  f"{cost.get('input', '?')} / {cost.get('output', '?')}" if cost else "paid")
         lim = r.get("limits") or {}
         lim_text = " · ".join([f"{v:,} {k}" for k, v in lim.items()]
@@ -3208,9 +3225,17 @@ def discover_card() -> str:
             f"<td>{add}</td></tr>")
     body.append("</tbody></table>")
     free = sum(1 for r in rows if r["free"])
+    import discover
+    import sources
+    ages = [a for a in map(sources.age_s, [*sources.FEEDS, discover.LISTS]) if a is not None]
     toolbar = ("<div class='toolbar'><input type='search' data-filter placeholder='Filter by model,"
                " provider or reach' aria-label='Filter rows'>"
-               "<label><input type='checkbox' data-paid> Show paid (connected providers)</label></div>")
+               "<label><input type='checkbox' data-paid> Show paid and unpriced (connected providers)</label>"
+               "<button class='btn small' type='button' id='d-refresh' title='Fetch FreeLLMAPI, models.dev and"
+               " every Open WebUI connection&#39;s own model list now, instead of waiting for the daily fetch'>"
+               "Refresh lists</button><span class='muted' id='d-refreshed'>"
+               + ("never fetched" if not ages else f"oldest list {max(ages) / 60:.0f} minutes old" if max(ages) < 5400
+                  else f"oldest list {max(ages) / 3600:.0f} hours old") + "</span></div>")
     key = """<dl>
       <dt>Rank</dt><dd>FreeLLMAPI's smartest-first order (1 is the smartest it lists); within
         each reach the list follows it. Hover for its fastest-first rank and size. A dash where
@@ -3223,7 +3248,8 @@ def discover_card() -> str:
         for its base URL); <em>other</em> a native API or an unvetted reseller.</dd>
       <dt>Price</dt><dd>Free if FreeLLMAPI lists it (a catalog of free tiers) or models.dev
         prices it at 0: models.dev shows the PAID price even where a free tier exists. Otherwise
-        input / output per million tokens.</dd>
+        input / output per million tokens. <em>Unknown</em> where only the provider's own list has
+        the model and it says nothing of price; those are behind <em>Show paid and unpriced</em>.</dd>
       <dt>Limits</dt><dd>The free tier's, from FreeLLMAPI: requests and tokens per minute (rpm,
         tpm) and per day (rpd, tpd), and roughly how many tokens a month that allows.</dd>
       <dt>New</dt><dd>First seen in either list in the last 7 days.</dd>
@@ -3241,11 +3267,13 @@ def discover_card() -> str:
       <dt>Sources</dt><dd>FreeLLMAPI's free catalog first: where it lists a model, its name,
         provider, context, tools, limits, ranks and quirks lead, and the popup links its
         freellmapi.co page. models.dev fills the rest and alone has the paid models, prices,
-        base URLs and modalities. A model one provider offers in both is shown once. Both are
-        refreshed at most daily.</dd>
+        base URLs and modalities; the popup links its page for the model. A model one provider
+        offers in both is shown once. Then each Open WebUI connection's own model list, for
+        what the provider serves that neither catalog has yet. All are fetched daily;
+        <em>Refresh lists</em> fetches them now.</dd>
     </dl>"""
-    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI"
-            f" and models.dev, deduped; {hidden:,} paid models on providers you have no route"
+    return (f"<div class='panel'><h2>Discover</h2><p class='sub'>{free} free models from FreeLLMAPI,"
+            f" models.dev and your providers' own lists, deduped; {hidden:,} paid models on providers you have no route"
             f" to are left out. Click a model for everything known about it and the routes it can be"
             f" added by.</p>{toolbar}"
             f"<div class='scroll'>{''.join(body)}</div>"
@@ -3287,7 +3315,7 @@ def collections_tab() -> str:
         " as <code>code</code> or <code>prose</code> the roles it does not admit; <code>free</code> hides"
         " the paid ones. Click a suggestion or type your own: comma-separated lower-case words, up to 8.</p>"
         # Not data-filter: that binds to the whole panel. The same match (any
-        # text, case-insensitive), on the member list only; ticked ones stay shown.
+        # text, case-insensitive), on the member list only; the seats show who is ticked.
         "<label class='field' for='c-member-filter'>Members</label>"
         # The panel as it will sit, drawn as the Collections list draws it
         # (bench.js recount): a seat per ticked member, explorer seats dashed.
@@ -3298,6 +3326,15 @@ def collections_tab() -> str:
         "<option value=''>Every provider</option></select></div>"
         "<div id='c-members' class='members' role='group' aria-label='Members'></div>"
         "<p class='rules' id='c-count'></p>"
+        "<div class='settings'>"
+        "<div><label class='field' for='c-max-calls'>Max calls per request</label>"
+        "<input id='c-max-calls' type='number' min='1' max='10'></div>"
+        "<div><label class='field' for='c-max-explorers'>Max explorers</label>"
+        "<input id='c-max-explorers' type='number' min='0' max='2'></div>"
+        "</div>"
+        "<p class='rules'>Empty follows the Gateway settings, shown greyed. A member out of quota stays"
+        " on the collection and an explorer sits in for it until the quota is back. Max explorers 0"
+        " means only the members ever run, stand-ins included.</p>"
         "<label class='field' for='c-shape'>Explorer shape</label><select id='c-shape'></select>"
         "<p class='rules'>Which models may fill the explorer seats. Automatic asks a decision"
         " model to label each prompt when one is onboarded, else general.</p>"
@@ -3316,6 +3353,10 @@ def collections_tab() -> str:
         "<label class='field' for='c-desc'>Description</label>"
         "<textarea id='c-desc' rows='2' maxlength='300'"
         " placeholder='e.g. Adversarial code review: finding bugs and broken guarantees in a diff'></textarea>"
+        "<label class='check'><input type='checkbox' id='c-ad-hoc'> Ad hoc: only when called by name</label>"
+        "<p class='rules'>Left off <code>GET /v1/models</code> and the list agents choose from, so an"
+        " agent picking for itself never spends it. For the expensive ones you decide to run: say"
+        " <code>bench/&lt;name&gt;</code> and it runs. Not a lock: anyone who knows the name can call it.</p>"
         "<div class='errors' id='c-errors' role='alert'></div><div class='done' id='c-done'></div>"
         "<div class='actions'><button class='btn danger' id='c-delete' hidden>Delete collection</button>"
         "<span class='spacer'></span><button class='btn' id='c-close'>Close</button>"
@@ -3329,7 +3370,8 @@ def collections_tab() -> str:
         "<input id='g-max-explorers' type='number' min='0' max='2'></div>"
         "</div>"
         "<p class='rules'>Explorers = min(max explorers, max calls − members): with 5 and 2, a"
-        " 2- or 3-model collection gets 2, a 4-model one 1, a 5-model one none. A request that"
+        " 2- or 3-model collection gets 2, a 4-model one 1, a 5-model one none. These apply to"
+        " every collection that does not set its own in its dialog. A request that"
         " names no collection is refused with each one's description and tags, so the caller"
         " can choose.</p>"
         "<div class='errors' id='g-errors' role='alert'></div><div class='done' id='g-done'></div>"
@@ -4439,6 +4481,7 @@ class Handler(gateway.Handler):
                "/api/remove": lambda p: remove_model(str(p.get("alias", "")), str(p.get("reason", ""))),
                "/api/restore": lambda p: restore_model(str(p.get("alias", ""))),
                "/api/retry": lambda p: retry_model(str(p.get("alias") or "")),
+               "/api/discover/refresh": lambda p: discover_refresh(),
                "/api/discover/add": lambda p: discover_add(str(p.get("connection") or ""),
                                                            str(p.get("model") or ""),
                                                            endpoint=str(p.get("endpoint") or ""),
@@ -4471,7 +4514,9 @@ class Handler(gateway.Handler):
                                                             str(p.get("shape") or ""),
                                                             str(p.get("description") or ""),
                                                             p.get("tags"), str(p.get("reply") or ""),
-                                                            str(p.get("merger") or "")),
+                                                            str(p.get("merger") or ""), p.get("ad_hoc") is True,
+                                                            max_calls=p.get("max_calls"),
+                                                            max_explorers=p.get("max_explorers")),
                "/api/collection/remove": lambda p: delete_collection(str(p.get("name") or "")),
                "/api/model-facts": lambda p: save_model_facts(str(p.get("alias") or ""), p.get("facts")),
                "/api/model-options": lambda p: save_model_options(str(p.get("alias") or ""), p.get("max_tokens")),

package/ui/bench.js

@@ -766,7 +766,8 @@
   var dd = $('discover-dialog');
   function price(x) {
     var c = x.cost || {};
-    return x.free ? 'free' : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
+    return x.free ? 'free' : x.free == null ? 'price unknown'
+      : (c.input != null ? '$' + c.input + ' in / $' + c.output + ' out per 1M tokens' : 'paid');
   }
   function limits(l) {
     return Object.keys(l || {}).map(function (k) { return Number(l[k]).toLocaleString() + ' ' + k; }).join(' · ');
@@ -835,13 +836,15 @@
         if (res.errors.length) listErrors($('d-errors'), res.errors);   // a hub that could not be asked
         var r = res.model, p = $('d-profile'), list = $('d-routes');
         $('d-title').textContent = r.name;
-        // Each source a link: FreeLLMAPI's page for the model where its sitemap has one, else its catalog.
+        // Each source a link to its page for the model: FreeLLMAPI's where its sitemap has one, models.dev's by its
+        // canonical id. The provider's own list has no page, only its connection.
         var sv = $('d-served');
         sv.textContent = r.provider_name + ' · ' + r.model + ' · listed by ';
         r.sources.forEach(function (s, i) {
-          var fl = s === 'freellmapi';
           if (i) sv.appendChild(document.createTextNode(' and '));
-          sv.appendChild(el('a', {href: fl ? r.page || 'https://freellmapi.co/models' : 'https://models.dev/',
+          if (s === 'provider') { sv.appendChild(document.createTextNode(r.provider_name + '\'s own list (' + r.connection + ' connection)')); return; }
+          var fl = s === 'freellmapi';
+          sv.appendChild(el('a', {href: fl ? r.page || 'https://freellmapi.co/models' : r.md_page || 'https://models.dev/',
                                   target: '_blank', rel: 'noopener'}, fl ? 'FreeLLMAPI' : 'models.dev'));
         });
         if (r.on_bench_as) sv.appendChild(document.createTextNode(' · on the bench as ' + r.on_bench_as));
@@ -905,6 +908,16 @@
     e.preventDefault(); openDiscover(row.dataset.provider, row.dataset.model);
   });
   $('d-close').addEventListener('click', function () { dd.close(); });
+  // Refresh lists: every list fetched now (serve.discover_refresh), then the page with them.
+  var dr = $('d-refresh');
+  if (dr) dr.addEventListener('click', function () {
+    var said = $('d-refreshed');
+    dr.disabled = true; dr.textContent = 'Fetching…'; said.textContent = 'about half a minute';
+    post('/api/discover/refresh', {}).then(function (res) {
+      said.textContent = (res.errors.length ? res.errors.join(' · ') + ' · ' : '') + 'Reloading…';
+      setTimeout(function () { location.reload(); }, res.errors.length ? 4000 : 500);
+    }).catch(function (err) { said.textContent = String(err); dr.disabled = false; dr.textContent = 'Refresh lists'; });
+  });
   function follow(log, title, before) {
     runView(title, before + 'Running in the background; closing this does not stop it. Log: runs/' + log);
     timer = setInterval(function () {
@@ -1331,10 +1344,14 @@
            head: $('c-form-head'), maxCalls: $('g-max-calls'), maxExp: $('g-max-explorers'),
            gErrors: $('g-errors'), gDone: $('g-done'), form: $('c-form'), status: $('c-status'),
            del: $('c-delete'), reply: $('c-reply'), merger: $('c-merger'), find: $('c-member-filter'),
-           seats: $('c-seats'), prov: $('c-member-provider')};
+           seats: $('c-seats'), prov: $('c-member-provider'), adHoc: $('c-ad-hoc'),
+           ownCalls: $('c-max-calls'), ownExp: $('c-max-explorers')};
   var C = null, open = null;         // the collection whose details are showing
+  // What the dialog's two limits come to: empty follows the gateway settings.
+  function limit(input, fallback) { return input.value === '' ? fallback : parseInt(input.value, 10); }
+  function maxCalls() { return limit(c.ownCalls, C.settings.max_calls); }
   function explorersFor(n) {
-    return Math.max(0, Math.min(C.settings.max_explorers, C.settings.max_calls - n));
+    return Math.max(0, Math.min(limit(c.ownExp, C.settings.max_explorers), maxCalls() - n));
   }
   function picked() {
     return Array.prototype.filter.call(c.members.querySelectorAll('input'), function (x) { return x.checked; })
@@ -1356,7 +1373,7 @@
         return ' · ' + t.name + ' is ticked on ' + t.on.join(' and ') + ': one model agrees with itself, it does not check itself';
       }).join('');
     c.members.querySelectorAll('input').forEach(function (x) {
-      x.disabled = !x.checked && (n >= C.settings.max_calls || !!x.dataset.blocked);
+      x.disabled = !x.checked && (n >= maxCalls() || !!x.dataset.blocked);
     });
     drawSeats(ex);
   }
@@ -1435,9 +1452,9 @@
   }
   // The member search: the filter boxes' match (any text, case-insensitive),
   // over the model's name and each chip's alias, served name and provider,
-  // and the provider select beside it. It only hides chips, and never a ticked
-  // one, so what is picked is always in view and every count still includes
-  // it; a row goes when all its chips do.
+  // and the provider select beside it. It only hides chips, ticked ones too:
+  // the seats above the list always show what is picked, each with its ×, and
+  // every count still includes it. A row goes when all its chips do.
   function filterMembers() {
     var q = c.find.value.trim().toLowerCase(), prov = c.prov.value;
     c.members.querySelectorAll('.m-row').forEach(function (row) {
@@ -1446,8 +1463,8 @@
         var box = lab.querySelector('input');
         var text = (box.value + ' ' + (lab.dataset.served || '') + ' ' + (fact(box.value).connection || '') + ' ' +
                     lab.textContent).toLowerCase();
-        lab.hidden = !box.checked && (!!lab.dataset.paid || !named && text.indexOf(q) === -1
-                                      || !!prov && (fact(box.value).provider || '') !== prov);
+        lab.hidden = !box.checked && !!lab.dataset.paid || !named && text.indexOf(q) === -1
+                     || !!prov && (fact(box.value).provider || '') !== prov;
         any = any || !lab.hidden;
       });
       row.hidden = !any;
@@ -1483,6 +1500,8 @@
   function showMerger() { $('c-merger-row').hidden = c.reply.value !== 'merged'; }
   c.reply.addEventListener('change', showMerger);
   c.shape.addEventListener('change', recount);          // the explorer seats say their shape
+  c.ownCalls.addEventListener('input', recount);
+  c.ownExp.addEventListener('input', recount);
   // A collection opens in a popup, as every other edit on the dashboard does:
   // the one clicked, or a new one from New collection.
   function closeForm() {
@@ -1497,6 +1516,10 @@
     if (!c.form.open) c.form.showModal();
     c.name.value = row ? row.name : ''; c.name.readOnly = !!row;
     c.desc.value = row ? row.description : ''; c.tags.value = row ? row.tags.join(', ') : '';
+    c.adHoc.checked = !!row && row.ad_hoc;
+    c.ownCalls.value = row && row.max_calls !== undefined ? row.max_calls : '';
+    c.ownExp.value = row && row.max_explorers !== undefined ? row.max_explorers : '';
+    c.ownCalls.placeholder = C.settings.max_calls; c.ownExp.placeholder = C.settings.max_explorers;
     drawTagPicks();
     c.members.textContent = ''; c.find.value = '';
     c.prov.length = 1;                                 // "Every provider", then each one a model is on
@@ -1565,6 +1588,7 @@
         var left = el('div');
         var title = el('div', {class: 'top'});
         title.appendChild(el('span', {class: 'alias'}, 'bench/' + row.name));
+        if (row.ad_hoc) title.appendChild(el('span', {class: 'chip'}, 'ad hoc: only when called by name'));
         row.tags.forEach(function (t) { title.appendChild(el('span', {class: 'chip'}, t)); });
         left.appendChild(title);
         if (row.description) left.appendChild(el('p', {class: 'desc'}, row.description));
@@ -1631,7 +1655,9 @@
     var name = c.name.value.trim().replace(/^bench\//, '');
     post('/api/collection', {name: name, members: picked(), shape: c.shape.value, description: c.desc.value,
                              tags: c.tags.value.split(','), reply: c.reply.value,
-                             merger: c.merger.value}).then(function (r) {
+                             merger: c.merger.value, ad_hoc: c.adHoc.checked,
+                             max_calls: c.ownCalls.value === '' ? null : Number(c.ownCalls.value),
+                             max_explorers: c.ownExp.value === '' ? null : Number(c.ownExp.value)}).then(function (r) {
       if (!r.ok) { listErrors(c.errors, r.errors); return; }
       finished('Saved bench/' + name + '.');
     }).catch(function (err) { c.errors.textContent = String(err); });