@wendellresearch-ai/benchwarmers 0.5.6 → 0.6.0

package/metrics/endpoints.py

@@ -131,8 +131,9 @@ def load() -> dict[str, dict]:
     else's machine is a timeout with no explanation. Whatever needs an
     endpoint asks default() (base, key, models and yaml_block all do), and
     that refuses, so a job run before any endpoint exists says what to do."""
+    import locked
     try:
-        data = json.loads(FILE.read_text(encoding="utf-8"))
+        data = json.loads(locked.read_text(FILE))      # inside a render, from its snapshot
     except FileNotFoundError:
         return {}
     except (OSError, ValueError) as exc:
@@ -147,11 +148,8 @@ def connections() -> dict:
     ties, names for bare-IP hosts, models.dev ids, providers whose free tier
     works only in their own client. {} when absent: a fresh
     install has no such facts, and none are assumed."""
-    try:
-        data = json.loads(CONNECTIONS.read_text(encoding="utf-8"))
-    except (OSError, ValueError):
-        return {}
-    return data if isinstance(data, dict) else {}
+    import locked
+    return locked.read_json(CONNECTIONS)               # inside a render, from its snapshot
 
 
 _default: tuple = (None, "")

package/metrics/guard.py

@@ -146,9 +146,11 @@ def cors_origin(headers, port: int) -> str:
     return origin if origin.lower() in origins(port) else ""
 
 
-def check(headers, port: int, post: bool, max_body: int, cors: bool = False) -> int:
+def check(headers, port: int, post: bool, max_body: int, cors: bool = False,
+          kind: str = "application/json") -> int:
     """Raise Refused, or return how many body bytes to read (0 for a GET).
-    `cors`: a gateway route, where an allowed origin may be cross-site."""
+    `cors`: a gateway route, where an allowed origin may be cross-site.
+    `kind`: the body's type; never one a plain form can send (an import's zip)."""
     host = (headers.get("Host") or "").strip().lower()
     if host not in hosts(port):
         raise Refused(403, f"unknown Host {host or '(none)'!r}; set BENCHWARMERS_PUBLIC_URL"
@@ -161,8 +163,8 @@ def check(headers, port: int, post: bool, max_body: int, cors: bool = False) ->
         raise Refused(403, "cross-site requests are refused")
     if origin and origin not in allowed:
         raise Refused(403, f"Origin {origin!r} is not allowed; set BENCH_ALLOWED_ORIGINS to add one")
-    if not (headers.get("Content-Type") or "").lower().startswith("application/json"):
-        raise Refused(415, "send Content-Type: application/json")
+    if not (headers.get("Content-Type") or "").lower().startswith(kind):
+        raise Refused(415, f"send Content-Type: {kind}")
     try:
         length = int(headers.get("Content-Length") or 0)
     except ValueError:

package/metrics/ledger.py

@@ -17,7 +17,9 @@ are what a provider's request limits count. `status` is ok, empty, error,
 timeout or refused (not sent: pacing, or the free-tier guard). Tokens are the
 provider's own figures from the reply's usage, absent when it gave none.
 `caller` is the script that made the call (gateway, perf/run, suite/run, ...)
-and `purpose` what the caller said it was for, when it said.
+and `purpose` what the caller said it was for, when it said. A failed call
+(status error) carries its `error`, start and end (client.clip), from
+2026-11-08: the uptime report classifies it (suite/run.uptime_class).
 
 A FILE A MONTH
 --------------

package/metrics/locked.py

@@ -26,6 +26,7 @@ import json
 import os
 import pathlib
 import tempfile
+import threading
 import time
 
 if os.name == "nt":
@@ -71,6 +72,7 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
     writers never share it (a fixed `.part` name let one writer move another's
     half-written file into place)."""
     path = pathlib.Path(path)
+    _forget(path)
     path.parent.mkdir(parents=True, exist_ok=True)
     fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
     try:
@@ -90,10 +92,85 @@ def write_atomic(path: pathlib.Path, text: str) -> None:
             os.remove(tmp)
 
 
+_seen = threading.local()
+
+
+@contextlib.contextmanager
+def snapshot():
+    """Inside it, this thread reads each file from disk once and reuses the
+    text: one dashboard render asked for the same dozen config files 11,000
+    times, seconds of opens on Windows. Text, not the parsed object, so a
+    caller changing what it parsed changes nothing for the next one. Other
+    threads (the gateway, a POST) read the disk as before; a write from this
+    thread drops its copy."""
+    if getattr(_seen, "texts", None) is not None:      # already inside one
+        yield
+        return
+    _seen.texts = {}
+    try:
+        yield
+    finally:
+        _seen.texts = None
+
+
+def once(key, compute):
+    """compute(); inside snapshot(), its first answer for `key` (an OSError as
+    first raised). Only for answers no caller can change, such as a string."""
+    texts = getattr(_seen, "texts", None)
+    if texts is None:
+        return compute()
+    if key not in texts:
+        try:
+            texts[key] = compute()
+        except OSError as exc:
+            texts[key] = exc                           # missing stays missing, as the same error
+    if isinstance(texts[key], OSError):
+        raise texts[key]
+    return texts[key]
+
+
+def read_text(path) -> str:
+    """The file's text (OSError if it cannot be read); inside snapshot(), as
+    first read."""
+    return once(str(path), lambda: pathlib.Path(path).read_text(encoding="utf-8"))
+
+
+def appended(path, mark: dict) -> tuple[bool, list[str]]:
+    """(start over?, lines) for a log that is only ever appended to: the
+    whole lines added since `mark` last saw it. The first time, or when the
+    file was replaced or rewritten rather than appended to (shorter, or the
+    bytes before where it stopped differ), it is True and every line. A last
+    line still missing its newline waits for the next call. `mark` is the
+    caller's own dict, kept between calls; hold one lock around this call
+    and whatever folds the lines in."""
+    path = str(path)
+    at, before = (mark.get("at", 0), mark.get("before", b"")) if mark.get("path") == path else (0, b"")
+    try:
+        with open(path, "rb") as fh:
+            size = fh.seek(0, os.SEEK_END)
+            if at:
+                fh.seek(max(at - len(before), 0))
+                if size < at or fh.read(len(before)) != before:
+                    at, before = 0, b""
+            fh.seek(at)
+            data = fh.read()
+    except OSError:
+        mark.clear()
+        return True, []
+    end = data.rfind(b"\n") + 1
+    mark.update(path=path, at=at + end, before=(before + data[:end])[-64:])
+    return at == 0, data[:end].decode("utf-8", "replace").splitlines()
+
+
+def _forget(path) -> None:
+    if (texts := getattr(_seen, "texts", None)) is not None:
+        texts.pop(str(path), None)
+
+
 def read_json(path: pathlib.Path) -> dict:
     """The file's object, or {} when it is missing, unreadable or not an object."""
     try:
-        data = json.loads(pathlib.Path(path).read_text(encoding="utf-8"))
+        data = json.loads(read_text(path))
     except (OSError, ValueError):
         return {}
     return data if isinstance(data, dict) else {}
@@ -105,6 +182,7 @@ def update(path: pathlib.Path, change, **dump):
     json.dumps (indent defaults to 2)."""
     dump.setdefault("indent", 2)
     with lock(path):
+        _forget(path)                                  # under the lock, the disk's copy
         data = read_json(path)
         before = json.dumps(data, sort_keys=True)
         result = change(data)
@@ -135,6 +213,29 @@ def _selftest() -> None:
         assert data == {"t": 200, "p": 150}, data
         assert update(f, lambda d: "same") == "same"          # no change, no write, result passed back
         assert not [p for p in f.parent.iterdir() if p.suffix == ".part"], "temp file left behind"
+        # A snapshot reads each file once; its own writes, and leaving it, read the disk again.
+        g = pathlib.Path(tmp) / "g.json"
+        with snapshot():
+            assert read_json(g) == {}                           # missing stays missing
+            g.write_text('{"a": 1}', encoding="utf-8")
+            assert read_json(g) == {}
+            write_atomic(g, '{"a": 2}')
+            assert read_json(g) == {"a": 2}
+            g.write_text('{"a": 3}', encoding="utf-8")
+            read_json(g)["a"] = 9                               # changing what it parsed changes nothing
+            assert read_json(g) == {"a": 2}
+        assert read_json(g) == {"a": 3}
+        # A log read in pieces: only what was added, a half-written line waits,
+        # and a rewrite starts over.
+        log, mark = pathlib.Path(tmp) / "l.jsonl", {}
+        assert appended(log, mark) == (True, [])
+        log.write_bytes(b"a\nb\nc")
+        assert appended(log, mark) == (True, ["a", "b"])
+        with log.open("ab") as fh:
+            fh.write(b"\nd\n")
+        assert appended(log, mark) == (False, ["c", "d"]) and appended(log, mark) == (False, [])
+        log.write_bytes(b"x\ny\nz\nw\n")                     # longer, but not what was there
+        assert appended(log, mark) == (True, ["x", "y", "z", "w"])
     print("ok")
 
 

package/metrics/offload.py

@@ -29,34 +29,51 @@ import collections
 import datetime as dt
 import json
 import pathlib
+import threading
 
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
 LOG = DATA / "metrics" / "offloads.jsonl"
 
 
+_lock = threading.Lock()
+_seen: dict = {"mark": {}, "renames": None, "out": [], "by_call": {}}
+
+
 def entries() -> list[dict]:
     """Every offload, oldest first. An entry with a `call` id (written by the
     bench gateway) is superseded by a later one with the same id: the gateway
-    records each seat unjudged as it answers, and the verdict comes after."""
-    if not LOG.exists():
-        return []
-    out, by_call = [], {}
-    from aliases import current
-    for line in LOG.read_text(encoding="utf-8").splitlines():
-        if line.strip():
-            try:
-                e = json.loads(line)
-            except ValueError:
-                continue
-            e["model"] = current(e.get("model", ""))   # a renamed model keeps its record
-            if e.get("call") in by_call:
-                out[by_call[e["call"]]] = e
-                continue
-            if e.get("call"):
-                by_call[e["call"]] = len(out)
-            out.append(e)
-    return out
+    records each seat unjudged as it answers, and the verdict comes after.
+
+    Only the lines added since the last call are read (all of them again
+    after a rename): one dashboard render asked seven times, a log of 44,000
+    lines took most of a second each, and the gateway adds to it on every
+    call. A new list each call; the entries in it are shared, so never change
+    one."""
+    import aliases
+    import locked
+    with _lock:
+        s = _seen
+        if s["renames"] is not aliases.history():  # a rename re-reads the history as a new dict
+            s.update(mark={}, renames=aliases.history())
+        fresh, lines = locked.appended(LOG, s["mark"])
+        if fresh:
+            s.update(out=[], by_call={})
+        out, by_call = s["out"], s["by_call"]
+        for line in lines:
+            if line.strip():
+                try:
+                    e = json.loads(line)
+                except ValueError:
+                    continue
+                e["model"] = aliases.current(e.get("model", ""), s["renames"])   # a renamed model keeps its record
+                if e.get("call") in by_call:
+                    out[by_call[e["call"]]] = e
+                    continue
+                if e.get("call"):
+                    by_call[e["call"]] = len(out)
+                out.append(e)
+        return list(out)
 
 
 def add(model: str, task: str, useful: bool | None, note: str, call: str | None = None,

package/metrics/route.py

@@ -101,8 +101,9 @@ from aliases import current as current_alias  # noqa: E402
 
 
 def _json(path: pathlib.Path) -> dict:
+    import locked
     try:
-        return json.loads(path.read_text(encoding="utf-8"))
+        return json.loads(locked.read_text(path))      # inside a render, from its snapshot
     except (OSError, ValueError):
         return {}
 
@@ -135,7 +136,14 @@ def full_tag_of(alias: str) -> str:
     name, else "?": a hub fronts many connections, and an untagged model
     behind it (Open WebUI stopped listing it) is not on a connection called
     `openwebui` with every other untagged one.
+
+    Worked out once per alias in a dashboard render, which asked 2,600 times.
     """
+    import locked
+    return locked.once(("full_tag_of", alias), lambda: _full_tag_of(alias))
+
+
+def _full_tag_of(alias: str) -> str:
     from run import _registered_names
     import endpoints
     # Tags are keyed by the served name. A decision model is not in the llm

package/metrics/scheduler.py

@@ -6,7 +6,8 @@
 RELEASE-PLAN.md Phase 3. The checks used to exist only as Windows scheduled
 tasks (scripts/schedule.ps1), which macOS, Linux and Docker do not have. serve.py
 starts this in a background thread; the jobs below are the same seven, at the
-same times, with the same arguments (schedule.ps1 explains each choice).
+same times, with the same arguments (schedule.ps1 explains each choice), and
+since 2026-11-08 the decision benchmark, which has no Windows task.
 
 MODE, from config/scheduler.json:
   absent / "off"   nothing runs. A fresh install spends no model calls until
@@ -55,6 +56,11 @@ JOBS = {
     "perf": (["perf/run.py", "-n", "3", "--timeout", "90", "--client", "builtin"], [(0, "07:00")]),
     "capabilities": (["metrics/probe_caps.py", "--gap", "30"], [(6, "07:00")]),
     "suite": (["suite/run.py", "--next"], [(None, t) for t in ("03:00", "09:00", "15:00", "21:00")]),
+    # Not in schedule.ps1 (2026-11-08). One question per model per set, 30 s
+    # apart round-robin: most decision models are OpenRouter `:free`, whose
+    # daily budget the suite and the probe share (metrics/decide.py).
+    "decisions": (["metrics/decide.py", "bench", "--per", "1", "--gap", "30", "--new-errors", "3"],
+                  [(None, "04:30")]),
 }
 # Each mode as the Settings page names and explains it (the docstring above).
 MODES = {"off": ("Off", "Nothing runs. A fresh install spends no model calls until someone opts in."),
@@ -77,6 +83,9 @@ ABOUT = {
                     " per model, spaced out.",
     "suite": "Scores one paid model on the review suite every six hours: up to six tasks it has not"
              " answered yet. These scores are what the router ranks by.",
+    "decisions": "Measures the decision models: one labelled question per model on each set (error classes,"
+                 " collections, verdicts), 30 s apart, stopping a model at its first failure; and asks"
+                 " about up to three new unrecognised error shapes. The gateway's picks ask the best one.",
 }
 
 
@@ -213,6 +222,7 @@ def _selftest() -> None:
     assert next_due(JOBS["perf"][1], mon) == dt.datetime(2026, 10, 19, 7, 0)
     assert next_due(JOBS["perf"][1], mon.replace(hour=8)) == dt.datetime(2026, 10, 26, 7, 0)
     assert next_due(JOBS["capabilities"][1], mon) == dt.datetime(2026, 10, 25, 7, 0)
+    assert next_due(JOBS["decisions"][1], mon) == dt.datetime(2026, 10, 19, 4, 30) and set(ABOUT) == set(JOBS)
     conf = {"mode": "log-only", "jobs": {n: n == "availability" for n in JOBS}}
     state: dict = {}
     global _log