@wendellresearch-ai/benchwarmers 0.6.1 → 0.6.2

package/VERSION

@@ -1 +1 @@
-0.6.1
+0.6.2

package/metrics/route.py

@@ -148,12 +148,14 @@ def _full_tag_of(alias: str) -> str:
     import endpoints
     # Tags are keyed by the served name. A decision model is not in the llm
     # yaml, so its served name comes from config/decision-models.json.
-    served = (_registered_names().get(alias)
-              or (_json(DATA / "config" / "decision-models.json").get(alias) or {}).get("served")
-              or alias)
+    decision = _json(DATA / "config" / "decision-models.json").get(alias) or {}
+    served = _registered_names().get(alias) or decision.get("served") or alias
     tag = _json(DATA / "config" / "provider-tags.json").get(served)
     if not tag:
-        name = endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+        # A decision model is in no registration either: its own endpoint is its `via`
+        # (Cloudflare's clef, onboarded from a direct cloudflare endpoint).
+        name = (endpoints.endpoint_name(endpoints.registrations().get(alias) or {})
+                or decision.get("via") or decision.get("endpoint"))
         tag = name if name and not endpoints.hub(name) else None
     return tag or "?"
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wendellresearch-ai/benchwarmers",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "Self-hosted panel gateway for LLMs: ask a collection of models at once, keep every answer side by side, rotate explorer models in, and let your verdicts pick the next panel.",
   "mcpName": "io.github.WendellResearch-AI/benchwarmers",
   "license": "MIT",

package/scripts/test_no_owui.py

@@ -236,6 +236,11 @@ def owner(fake: str) -> None:
     assert tags["fast-1"] == "groq" and tags["llm7.codestral"] == "llm7-api" and tags["vendor/x:free"] == "openrouter", tags
     assert "never-read" not in json.dumps(tags)
     assert route.connection_of("fast-1") == "groq" and route.connection_of("x-free") == "openrouter"
+    # A decision model onboarded from the direct endpoint: in no registration, untagged, its `via` is its connection.
+    write("decision-models.json", {"dec-1": {"served": "vendor/decide:free", "endpoint": "openrouter", "via": "openrouter"},
+                                   "dec-hub": {"served": "hub-decide", "endpoint": "openwebui", "via": "openwebui"}})
+    assert route.connection_of("dec-1") == "openrouter" and route.connection_of("dec-hub") == "?"
+    (TMP / "config" / "decision-models.json").unlink()
 
     # The default hub is recorded whole; the direct endpoint, its registered models only.
     assert check_deprecations.served() == {"fast-1", "llm7.codestral", "fresh-2", "mercury-decide",

package/serve.py

@@ -4956,11 +4956,12 @@ def move_card() -> str:
     """Settings' Export and import (export_zip, import_zip; bench.js)."""
     return ("<div class='panel form-panel'><h2>Export and import</h2><p class='muted'>Take this install to another"
             " machine, or keep a copy of it: one .zip file, imported on the other install's Settings page.</p>"
-            "<div class='pick'><label class='check'><input type='radio' name='x-scope' value='settings' checked"
+            # No wrapper: `.pick` is the onboarding picker's 3-column grid row, and squeezed these into it.
+            "<label class='check'><input type='radio' name='x-scope' value='settings' checked"
             " data-kept> <span><b>Settings only</b>: endpoints, collections, roles, model options, the schedule,"
             " gateway keys, the theme (<code>config/</code>)</span></label>"
             "<label class='check'><input type='radio' name='x-scope' value='all' data-kept> <span><b>Everything</b>:"
-            " the settings, and every result, history, log and saved answer the bench judges by</span></label></div>"
+            " the settings, and every result, history, log and saved answer the bench judges by</span></label>"
             "<label class='check'><input type='checkbox' id='x-keys' data-kept> <span>With the endpoints' API keys."
             " The file then holds them: keep it like a password, and delete it once imported.</span></label>"
             "<div class='actions'><button class='btn' type='button' id='x-export'>Download export</button></div>"
@@ -5005,23 +5006,28 @@ def access_card() -> str:
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td>{esc(v)}</td>"
                    f"<td class='mono set-by'>{'<br>'.join(esc(s) for s in src.split(', '))}</td></tr>"
                    for k, v, src in rows)
-    running, disk = on_disk()
-    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
-           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
     return ("<div class='panel'><h2>Server and access</h2><p class='muted'>How this install is reached and who"
             " may call it. Set in the environment, read at start: change one and restart. Shown here, never"
             " set here, since a page that could change its own login could open itself to anyone. No secret"
             " is shown, only whether one is set.</p>"
             f"<table class='facts'><thead><tr>{th('Setting')}{th('Now')}{th('Set by')}</tr></thead>"
-            f"<tbody>{body}</tbody></table>"
-            # restart(): stopped and started again by serve.py's own parent process.
-            f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
-            " for an update or a setting"
-            " above, whatever runs it (a service, the launcher, a container or a terminal). It waits for"
-            " onboarding, a scheduled job and gateway calls in progress, or cuts them off if you insist; the"
-            " page comes back in a few seconds.</p>"
-            "<p><button class='btn' type='button' id='restart'>Restart Benchwarmers</button>"
-            " <span id='restart-out' role='status'></span></p></div>")
+            f"<tbody>{body}</tbody></table>{restart_block('restart', 'a setting above')}</div>")
+
+
+def restart_block(id_: str, what: str) -> str:
+    """The Restart heading, what runs now, and its button (bench.js restart()).
+    On Settings (Server and access) and Info, each with its own ids, since both
+    tabs can be in the page at once."""
+    running, disk = on_disk()
+    now = (f"Running {esc(RELEASE)} ({esc(running)}), the code on disk." if running == disk else
+           f"<strong>Newer code is on disk</strong> ({esc(disk)}; running {esc(RELEASE)}, {esc(running)}).")
+    # restart(): stopped and started again by serve.py's own parent process.
+    return (f"<h3>Restart</h3><p class='muted'>{now} A restart starts the dashboard again on the code on disk,"
+            f" for an update or {what}, whatever runs it (a service, the launcher, a container or a terminal)."
+            " It waits for onboarding, a scheduled job and gateway calls in progress, or cuts them off if you"
+            " insist; the page comes back in a few seconds.</p>"
+            f"<p><button class='btn' type='button' id='{id_}'>Restart Benchwarmers</button>"
+            f" <span id='{id_}-out' role='status'></span></p>")
 
 
 RESTART = 75                       # the exit code serve.py's parent reads as "start me again"
@@ -5337,7 +5343,8 @@ def models_page() -> str:
     # Every model, decision models too (connections_view is chat models only,
     # for what a collection may hold): they are on the bench like any other.
     by: dict[str, list[str]] = {}
-    for a in model_directory():
+    directory = model_directory()
+    for a in directory:
         by.setdefault(catalog.connection_of(a), []).append(a)
     for c in ({"name": n, "models": sorted(m)} for n, m in sorted(by.items())):
         rows = []
@@ -5348,6 +5355,9 @@ def models_page() -> str:
             ep = through.get(a, "?")
             kind = (" " + chip("decision", "", "a decision model: answers typed questions, not chat")
                     if a in decisions else "")
+            if directory[a].get("ad_hoc"):
+                kind += " " + chip("ad hoc", "", "ad hoc only: never chosen by the bench on its own, only when you"
+                                   " name it (a collection member, with, --model)")
             rows.append(f"<tr><td><a href='#' class='model' data-alias='{esc(a)}'>"
                         f"<span class='alias'>{esc(a)}</span></a>{kind}</td>"
                         f"<td data-sort='{esc(ep)}'>{chip(ep, 'via', f'reached through the {ep} endpoint')}</td>"
@@ -5604,6 +5614,7 @@ def info_page() -> str:
         ("Theme", saved_theme())]]
     body = "".join(f"<tr><th scope='row'>{esc(k)}</th><td class='mono'>{v}</td></tr>" for k, v in rows)
     return (f"<div class='panel'><h2>Info</h2><table class='facts'><tbody>{body}</tbody></table>"
+            f"{restart_block('restart-info', 'a setting')}"
             "<p class='muted'>Benchwarmers · by Wendell Research · MIT licence"
             " (<code>LICENSE</code>, credits in <code>NOTICE</code>). Every response carries"
             " <code>X-Benchwarmers-Version</code>.</p></div>")
@@ -5668,8 +5679,7 @@ def editor() -> str:
         + "".join(f"<label>{k} <input type='number' min='1' data-k='{k}' placeholder='—'></label>"
                   for k in ("rpm", "rpd", "tpm", "tpd"))
         + "</div><div class='src' id='s-limits'></div></div>"
-        "</div><div class='errors' id='f-errors' role='alert'></div><div class='done' id='f-done'></div>"
-        "<div class='actions'><button class='btn' id='f-save' type='button'>Save facts</button></div>"
+        "</div><div class='errors' id='f-errors' role='alert'></div>"
         "</details>"
         # How the bench calls it (serve.save_model_options): every caller, gateway, suite and probes.
         "<details id='m-opts' class='facts-box'><summary>Call options</summary>"
@@ -5679,8 +5689,7 @@ def editor() -> str:
         " makes to it. A reasoning model thinks out of the same budget and answers with an empty string"
         " when it runs out, which looks like a dead model: raise it then. Blank leaves it to the"
         " endpoint. <span id='co-ceiling'></span></p>"
-        "<div class='errors' id='co-errors' role='alert'></div><div class='done' id='co-done'></div>"
-        "<div class='actions'><button class='btn' id='co-save' type='button'>Save options</button></div>"
+        "<div class='errors' id='co-errors' role='alert'></div>"
         "</details>"
         "<label class='check'><input type='checkbox' id='m-adhoc'> Ad hoc only</label>"
         "<p class='rules'>Never chosen by the bench on its own: no explorer seat, no router"

package/ui/bench.js

@@ -55,30 +55,33 @@
   askD.addEventListener('close', function () { answer(false); });   // Escape
 
   // ---- restart (serve.restart): the page reloads once the new one answers ----
-  function restart(force) {
-    var rs = $('restart'), out = $('restart-out');
+  // One button on Settings (restart) and one on Info (restart-info), each with its own status line.
+  function restart(id, force) {
+    var rs = $(id), out = $(id + '-out');
     rs.disabled = true; out.textContent = 'Restarting…';
     post('/api/restart', {force: force}).then(function (res) {
-      if (res.ok) return back(Date.now());
+      if (res.ok) return back(out, Date.now());
       rs.disabled = false; out.textContent = '';
       if (!res.busy) { listErrors(out, res.errors); return; }
       ask('Still in progress: ' + res.errors.join('; ') + '. Restart anyway and cut them off?', 'Restart anyway', true)
-        .then(function (yes) { if (yes) restart(true); });
+        .then(function (yes) { if (yes) restart(id, true); });
     }).catch(function (err) { rs.disabled = false; out.textContent = String(err); });
   }
   // The old one stops half a second after answering; /health answers again once the new one is up.
-  function back(since) {
+  function back(out, since) {
     setTimeout(function () {
       fetch('/health', {cache: 'no-store'}).then(function (r) { if (!r.ok) throw r.status; location.reload(); })
         .catch(function () {
-          if (Date.now() - since < 120000) back(since);
-          else $('restart-out').textContent = 'Not back after two minutes: look at whatever runs it.';
+          if (Date.now() - since < 120000) back(out, since);
+          else out.textContent = 'Not back after two minutes: look at whatever runs it.';
         });
     }, 1500);
   }
-  on('click', 'restart', function () {
-    ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
-      .then(function (yes) { if (yes) restart(false); });
+  ['restart', 'restart-info'].forEach(function (id) {
+    on('click', id, function () {
+      ask('Restart Benchwarmers? The dashboard and the gateway on this port stop for a few seconds.', 'Restart')
+        .then(function (yes) { if (yes) restart(id, false); });
+    });
   });
 
   // ---- Export and import (serve.export_zip, import_zip) ----------------
@@ -490,7 +493,7 @@
            reason: $('m-reason'), remove: $('m-remove'), onboard: $('m-onboard'),
            qtypes: $('m-qtypes'), aliasWhy: $('m-alias-why'), roleHint: $('m-role-hint')};
   m.aliasWhy.dataset.chat = m.aliasWhy.textContent;
-  var editing = null;
+  var editing = null, shown = {};
 
   function openModel(alias) {
     var model = D.models[alias], gone = (D.retired || {})[alias];
@@ -539,6 +542,8 @@
         + (hint.at || '').slice(0, 10) + '). Only a suggestion: the role is the one you pick above.' : '';
       m.save.disabled = false; m.save.textContent = 'Save';
       m.reason.value = ''; m.removeBox.open = false; m.remove.disabled = false; refresh();
+      // What the popup opened with: Save sends only the parts that differ from it.
+      shown = {model: JSON.stringify(modelBody()), facts: factsBody(), opts: optsBody()};
     }
     if (!md.open) md.showModal();      // open already: shown again as it now is (shownAgain)
   }
@@ -577,7 +582,7 @@
     var box = $('m-facts'), f = (D.facts || {})[alias], profile = $('m-profile');
     box.hidden = profile.hidden = decision || !f;
     profile.textContent = '';
-    $('f-errors').textContent = ''; $('f-done').textContent = '';
+    $('f-errors').textContent = '';
     if (box.hidden) return;
     var src = f.sources || {}, mine = function (k) { return src[k] === 'you'; };
     // At a glance, as models.dev shows a model: what goes in, what comes out.
@@ -621,40 +626,25 @@
     var lsrc = Object.keys(src).filter(function (k) { return k.indexOf('limits') === 0; }).map(function (k) { return src[k]; });
     $('s-limits').textContent = lsrc.length ? lsrc.join(' · ') : 'none known';
   }
-  $('f-save').addEventListener('click', function () {
-    if (!editing) return;
+  // The facts as typed, as /api/model-facts takes them; null when the box is not shown.
+  function factsBody() {
+    if ($('m-facts').hidden) return null;
     var facts = {limits: {}};
     FACT_NUM.forEach(function (k) { if ($('f-' + k).value) facts[k] = parseInt($('f-' + k).value, 10); });
     FACT_CAP.forEach(function (k) { var v = $('f-' + k).value; if (v) facts[k] = v === 'yes'; });
     if ($('f-knowledge_cutoff').value.trim()) facts.knowledge_cutoff = $('f-knowledge_cutoff').value.trim();
     document.querySelectorAll('#f-limits input').forEach(function (b) { if (b.value) facts.limits[b.dataset.k] = parseInt(b.value, 10); });
-    $('f-errors').textContent = ''; $('f-done').textContent = '';
-    var alias = editing;
-    post('/api/model-facts', {alias: alias, facts: facts}).then(function (r) {
-      if (!r.ok) { listErrors($('f-errors'), r.errors); return; }
-      // The tables that show facts (At a glance, Models, Quotas) follow once the popup closes (live).
-      shownAgain(alias, 'f-done', 'Saved. The tables show it once this closes.');
-    });
-  });
+    return JSON.stringify(facts);
+  }
   // ---- call options: how the bench asks this model (serve.save_model_options)
   function fillOptions(alias, decision) {
     var o = (D.options || {})[alias] || {}, ceiling = ((D.facts || {})[alias] || {}).max_output;
     $('m-opts').hidden = decision;
-    $('co-errors').textContent = ''; $('co-done').textContent = '';
+    $('co-errors').textContent = '';
     $('co-max').value = o.max_tokens || ''; $('co-max').placeholder = "the endpoint's default";
     $('co-ceiling').textContent = ceiling ? 'At most its max output, ' + Number(ceiling).toLocaleString() + '.' : '';
   }
-  $('co-save').addEventListener('click', function () {
-    if (!editing) return;
-    var v = $('co-max').value.trim();
-    $('co-errors').textContent = ''; $('co-done').textContent = 'Saving…';   // the llm CLI is told too: seconds
-    var alias = editing;
-    post('/api/model-options', {alias: alias, max_tokens: v ? Number(v) : null}).then(function (r) {
-      if (!r.ok) { $('co-done').textContent = ''; listErrors($('co-errors'), r.errors); return; }
-      ownChange();                         // only this popup shows them
-      shownAgain(alias, 'co-done', 'Saved' + (r.note ? '; ' + r.note : '') + '.');
-    });
-  });
+  function optsBody() { return $('m-opts').hidden ? null : $('co-max').value.trim(); }
 
   function refresh() {
     m.danger.hidden = !(editing && m.alias.value.trim() !== editing);
@@ -690,22 +680,57 @@
       then(res);
     };
   }
-  m.save.addEventListener('click', function () {
-    if (!editing) return;
-    var body = {alias: editing, new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
+  function modelBody() {
+    var body = {new_alias: m.alias.value.trim(), new_role: m.role.value, new_ad_hoc: m.adhoc.checked};
     if (!m.qtypes.hidden) {
       body.new_question_types = Array.prototype.filter.call(m.qtypes.querySelectorAll('input'),
         function (c) { return c.checked; }).map(function (c) { return c.value; });
     }
-    (body.new_alias === editing ? Promise.resolve(true)
-      : ask('Rename ' + editing + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
+    return body;
+  }
+  // The one Save: facts, then call options, then the model itself (alias, role, ad hoc,
+  // question types), each only if changed. The model goes last: a rename moves the
+  // facts and options saved under the old alias with it. The first refusal stops it.
+  m.save.addEventListener('click', function () {
+    if (!editing) return;
+    var alias = editing, body = modelBody(), facts = factsBody(), opts = optsBody(), said = [];
+    var steps = [];
+    if (facts !== shown.facts) steps.push(function () {
+      return post('/api/model-facts', {alias: alias, facts: JSON.parse(facts)}).then(function (r) {
+        if (!r.ok) { listErrors($('f-errors'), r.errors); $('m-facts').open = true; throw 'refused'; }
+        said.push('model facts');
+      });
+    });
+    if (opts !== shown.opts) steps.push(function () {   // the llm CLI is told too: seconds
+      return post('/api/model-options', {alias: alias, max_tokens: opts ? Number(opts) : null}).then(function (r) {
+        if (!r.ok) { listErrors($('co-errors'), r.errors); $('m-opts').open = true; throw 'refused'; }
+        ownChange();                         // only this popup shows them
+        said.push('call options' + (r.note ? ' (' + r.note + ')' : ''));
+      });
+    });
+    if (JSON.stringify(body) !== shown.model) steps.push(function () {
+      body.alias = alias;
+      return post('/api/model', body).then(function (r) {
+        if (!r.ok) { listErrors(m.errors, r.errors); throw 'refused'; }
+        said.push('the model (' + done(r).replace(/^Done: |\.$/g, '') + ')');
+        if (body.new_alias !== alias) { m.title.textContent = body.new_alias; refreshPanes(); }
+      });
+    });
+    m.errors.textContent = ''; m.done.textContent = ''; $('f-errors').textContent = ''; $('co-errors').textContent = '';
+    if (!steps.length) { m.done.textContent = 'Nothing changed.'; return; }
+    (body.new_alias === alias ? Promise.resolve(true)
+      : ask('Rename ' + alias + ' to ' + body.new_alias + '? Every record keyed by the old alias follows it.', 'Rename')
     ).then(function (yes) {
       if (!yes) return;
-      m.save.disabled = true; m.save.textContent = 'Saving…'; m.errors.textContent = '';
-      post('/api/model', body).then(after(m.save, 'Save', function (res) {
-        if (body.new_alias !== body.alias) { m.title.textContent = body.new_alias; refreshPanes(); }
-        shownAgain(body.new_alias, 'm-done', done(res));
-      })).catch(function (err) { m.errors.textContent = String(err); m.save.disabled = false; m.save.textContent = 'Save'; });
+      m.save.disabled = true; m.save.textContent = 'Saving…';
+      steps.reduce(function (p, step) { return p.then(step); }, Promise.resolve()).then(function () {
+        // The tables that show facts follow once the popup closes (live).
+        shownAgain(body.new_alias, 'm-done', 'Saved: ' + said.join(' · ') + '.');
+      }).catch(function (err) {
+        if (err !== 'refused') m.errors.textContent = String(err);
+        if (said.length) m.done.textContent = 'Saved before that: ' + said.join(' · ') + '.';
+        m.save.disabled = false; m.save.textContent = 'Save';
+      });
     });
   });
   // The confirm is the last word: the popup closes at once and the page