@wendellresearch-ai/benchwarmers 0.2.0 → 0.2.1

package/metrics/alerts.py

@@ -1,116 +1,116 @@
-"""Things that went wrong and are still wrong, where they will be seen.
-
-    python metrics/alerts.py            # what is currently wrong
-
-The scheduled checks already find problems. Until now they announced them by
-exiting non-zero, which Windows records and nobody reads — a model could be
-retired for a fortnight and the only trace would be a number in Task
-Scheduler. Work that reports only to a log nobody opens has not reported.
-
-KEYED AND SELF-CLEARING, NOT AN APPEND LOG
-------------------------------------------
-An alert has a stable `key`, so a daily check that keeps finding the same
-retired model updates one row rather than writing thirty. The first sighting is
-kept, because "missing since Tuesday" is the useful part — a model absent for a
-day may be an outage and one absent for a week is gone.
-
-**A check that finds nothing CLEARS its alerts.** Otherwise the file only ever
-grows and the dashboard turns into a wall of stale red that everyone learns to
-ignore, which is worse than no alerting at all. `sweep()` exists so a caller
-can say "these are all the problems I found" and have the rest retired
-automatically.
-
-WHAT IS NOT AN ALERT
---------------------
-Anything transient. A rate limit, a timeout, one failed call — those are
-`capacity` and they resolve themselves. An alert is for a state that will not
-fix itself and that somebody has to decide about: a model retired upstream, a
-registration pointing at a name that does not exist, a subscription that no
-longer covers what it did.
-"""
-
-from __future__ import annotations
-
-import datetime as dt
-import pathlib
-
-import locked
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-FILE = DATA / "metrics" / "alerts.json"
-
-
-def _load() -> dict:
-    return locked.read_json(FILE)
-
-
-def _update(change):
-    """Read-change-write under a lock: the gateway's seat threads, the
-    scheduled checks and the dashboard all raise and clear alerts."""
-    return locked.update(FILE, change, sort_keys=True)
-
-
-def raise_alert(key: str, kind: str, message: str, action: str = "",
-                **extra) -> None:
-    """Record a problem, keeping the date it was first seen.
-
-    `extra` carries what the dashboard needs to ACT on it rather than just
-    show it: `alias` for a one-model action, `aliases` for retire-all.
-    """
-    today = dt.date.today().isoformat()
-
-    def put(data: dict) -> None:
-        data[key] = {
-            "kind": kind,
-            "message": message,
-            "action": action,
-            "first_seen": data.get(key, {}).get("first_seen", today),
-            "last_seen": today,
-            **extra,
-        }
-    _update(put)
-
-
-def clear(key: str) -> bool:
-    return _update(lambda data: data.pop(key, None) is not None)
-
-
-def sweep(prefix: str, still_wrong: set[str]) -> list[str]:
-    """Retire every alert under `prefix` that the caller did not re-raise.
-
-    This is what stops the file growing for ever. A check reports the complete
-    set of problems it found, and anything it used to find but no longer does
-    is resolved — a model that came back should not leave a red row behind it.
-    """
-    def retire(data: dict) -> list[str]:
-        gone = [k for k in data
-                if k.startswith(prefix) and k not in still_wrong]
-        for key in gone:
-            del data[key]
-        return gone
-    return _update(retire)
-
-
-def active() -> dict:
-    return _load()
-
-
-def main() -> None:
-    data = active()
-    if not data:
-        print("Nothing wrong.")
-        return
-    today = dt.date.today()
-    print(f"{len(data)} open:\n")
-    for key, a in sorted(data.items(),
-                         key=lambda kv: kv[1]["first_seen"]):
-        days = (today - dt.date.fromisoformat(a["first_seen"])).days
-        age = "today" if days == 0 else f"{days}d"
-        print(f"  [{a['kind']}] {a['message']}")
-        print(f"      first seen {a['first_seen']} ({age})"
-              + (f" — {a['action']}" if a["action"] else ""))
-
-
-if __name__ == "__main__":
-    main()
+"""Things that went wrong and are still wrong, where they will be seen.
+
+    python metrics/alerts.py            # what is currently wrong
+
+The scheduled checks already find problems. Until now they announced them by
+exiting non-zero, which Windows records and nobody reads — a model could be
+retired for a fortnight and the only trace would be a number in Task
+Scheduler. Work that reports only to a log nobody opens has not reported.
+
+KEYED AND SELF-CLEARING, NOT AN APPEND LOG
+------------------------------------------
+An alert has a stable `key`, so a daily check that keeps finding the same
+retired model updates one row rather than writing thirty. The first sighting is
+kept, because "missing since Tuesday" is the useful part — a model absent for a
+day may be an outage and one absent for a week is gone.
+
+**A check that finds nothing CLEARS its alerts.** Otherwise the file only ever
+grows and the dashboard turns into a wall of stale red that everyone learns to
+ignore, which is worse than no alerting at all. `sweep()` exists so a caller
+can say "these are all the problems I found" and have the rest retired
+automatically.
+
+WHAT IS NOT AN ALERT
+--------------------
+Anything transient. A rate limit, a timeout, one failed call — those are
+`capacity` and they resolve themselves. An alert is for a state that will not
+fix itself and that somebody has to decide about: a model retired upstream, a
+registration pointing at a name that does not exist, a subscription that no
+longer covers what it did.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import pathlib
+
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+FILE = DATA / "metrics" / "alerts.json"
+
+
+def _load() -> dict:
+    return locked.read_json(FILE)
+
+
+def _update(change):
+    """Read-change-write under a lock: the gateway's seat threads, the
+    scheduled checks and the dashboard all raise and clear alerts."""
+    return locked.update(FILE, change, sort_keys=True)
+
+
+def raise_alert(key: str, kind: str, message: str, action: str = "",
+                **extra) -> None:
+    """Record a problem, keeping the date it was first seen.
+
+    `extra` carries what the dashboard needs to ACT on it rather than just
+    show it: `alias` for a one-model action, `aliases` for retire-all.
+    """
+    today = dt.date.today().isoformat()
+
+    def put(data: dict) -> None:
+        data[key] = {
+            "kind": kind,
+            "message": message,
+            "action": action,
+            "first_seen": data.get(key, {}).get("first_seen", today),
+            "last_seen": today,
+            **extra,
+        }
+    _update(put)
+
+
+def clear(key: str) -> bool:
+    return _update(lambda data: data.pop(key, None) is not None)
+
+
+def sweep(prefix: str, still_wrong: set[str]) -> list[str]:
+    """Retire every alert under `prefix` that the caller did not re-raise.
+
+    This is what stops the file growing for ever. A check reports the complete
+    set of problems it found, and anything it used to find but no longer does
+    is resolved — a model that came back should not leave a red row behind it.
+    """
+    def retire(data: dict) -> list[str]:
+        gone = [k for k in data
+                if k.startswith(prefix) and k not in still_wrong]
+        for key in gone:
+            del data[key]
+        return gone
+    return _update(retire)
+
+
+def active() -> dict:
+    return _load()
+
+
+def main() -> None:
+    data = active()
+    if not data:
+        print("Nothing wrong.")
+        return
+    today = dt.date.today()
+    print(f"{len(data)} open:\n")
+    for key, a in sorted(data.items(),
+                         key=lambda kv: kv[1]["first_seen"]):
+        days = (today - dt.date.fromisoformat(a["first_seen"])).days
+        age = "today" if days == 0 else f"{days}d"
+        print(f"  [{a['kind']}] {a['message']}")
+        print(f"      first seen {a['first_seen']} ({age})"
+              + (f" — {a['action']}" if a["action"] else ""))
+
+
+if __name__ == "__main__":
+    main()

package/metrics/aliases.py

@@ -1,41 +1,41 @@
-"""One place that knows an alias's current name.
-
-`config/alias-history.json` maps an old alias to the name it became, because
-aliases get renamed (`laguna-xs` became `laguna-xs-2.1` when a 2.2 was
-coming) and every record here is keyed by alias: suite outputs are named
-`<alias>.md`, offloads log `"model": "<alias>"`, the perf history has a
-`model` column, the run notes quote the alias in a table. The notes are
-narrative and are never retro-edited; the rest is data and is never rewritten
-either — instead every reader maps through this, so a renamed model keeps
-its whole record under its new name.
-
-Chains are followed (a -> b -> c resolves a to c) and cycles are refused.
-"""
-
-from __future__ import annotations
-
-import json
-import pathlib
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-HISTORY = DATA / "config" / "alias-history.json"
-
-
-def history() -> dict[str, str]:
-    try:
-        return json.loads(HISTORY.read_text(encoding="utf-8"))
-    except (OSError, ValueError):
-        return {}
-
-
-def current(alias: str, table: dict[str, str] | None = None) -> str:
-    """The name this alias goes by today."""
-    table = history() if table is None else table
-    seen = {alias}
-    while alias in table:
-        alias = table[alias]
-        if alias in seen:
-            raise ValueError(f"alias history loops at {alias!r}")
-        seen.add(alias)
-    return alias
+"""One place that knows an alias's current name.
+
+`config/alias-history.json` maps an old alias to the name it became, because
+aliases get renamed (`laguna-xs` became `laguna-xs-2.1` when a 2.2 was
+coming) and every record here is keyed by alias: suite outputs are named
+`<alias>.md`, offloads log `"model": "<alias>"`, the perf history has a
+`model` column, the run notes quote the alias in a table. The notes are
+narrative and are never retro-edited; the rest is data and is never rewritten
+either — instead every reader maps through this, so a renamed model keeps
+its whole record under its new name.
+
+Chains are followed (a -> b -> c resolves a to c) and cycles are refused.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+HISTORY = DATA / "config" / "alias-history.json"
+
+
+def history() -> dict[str, str]:
+    try:
+        return json.loads(HISTORY.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def current(alias: str, table: dict[str, str] | None = None) -> str:
+    """The name this alias goes by today."""
+    table = history() if table is None else table
+    seen = {alias}
+    while alias in table:
+        alias = table[alias]
+        if alias in seen:
+            raise ValueError(f"alias history loops at {alias!r}")
+        seen.add(alias)
+    return alias

package/metrics/bench_collections.py

@@ -1,141 +1,141 @@
-"""Bench-gateway collections: named groups of models a caller asks for by name.
-
-    config/collections.json   {"core": {"members": [...], "shape": "",
-                                         "description": "...", "tags": ["general"],
-                                         "reply": "all", "merger": ""}}
-    config/gateway.json       {"max_calls": 5, "max_explorers": 2}
-
-A request for `bench/core` runs every member, plus explorer seats that
-`route.py` fills by staleness so the roster keeps getting evidence even when
-the same collection is called every day:
-
-    explorers = min(max_explorers, max_calls - members)
-
-`shape` (review, design, …) decides which models may fill those explorer
-seats; empty means a decision model labels each prompt, or `general` without
-one. `reply` is what the caller reads: `all` (every answer, the default) or
-`merged` (one answer a `merger` model writes from them; empty means chosen by
-verdict record, gateway.py). A request may override it with a `:merged` or
-`:all` suffix on the model name, or `"bench": {"reply": ...}` in the body.
-Named `bench_collections` because `collections` would shadow the standard
-library for everything importing from metrics/.
-"""
-
-from __future__ import annotations
-
-import json
-import pathlib
-import re
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-FILE = DATA / "config" / "collections.json"
-SETTINGS_FILE = DATA / "config" / "gateway.json"
-DEFAULTS = {"max_calls": 5, "max_explorers": 2}
-NAME = re.compile(r"^[a-z0-9][a-z0-9-]{1,31}$")
-PREFIX = "bench/"
-REPLIES = ("all", "merged")
-
-
-def _read(path: pathlib.Path) -> dict:
-    try:
-        data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, ValueError):
-        return {}
-    return data if isinstance(data, dict) else {}
-
-
-def unreadable(path: pathlib.Path) -> bool:
-    """True if the file exists but is not a JSON object. A writer must refuse
-    then: `_read` would hand it {} and the save would wipe the file."""
-    if not path.exists():
-        return False
-    try:
-        return not isinstance(json.loads(path.read_text(encoding="utf-8")), dict)
-    except (OSError, ValueError):
-        return True
-
-
-def load() -> dict[str, dict]:
-    return _read(FILE)
-
-
-def settings() -> dict:
-    return {**DEFAULTS, **{k: v for k, v in _read(SETTINGS_FILE).items() if k in DEFAULTS}}
-
-
-def explorers_for(members: int, s: dict | None = None) -> int:
-    s = s or settings()
-    return max(0, min(s["max_explorers"], s["max_calls"] - members))
-
-
-TAG = re.compile(r"^[a-z0-9][a-z0-9-]{0,23}$")
-
-
-def described(name: str, c: dict, s: dict | None = None) -> dict:
-    """What a caller needs to choose a collection: purpose, tags and make-up.
-    Served on GET /v1/models, so an orchestrator can pick by task."""
-    members = c.get("members", [])
-    out = {"description": c.get("description") or c.get("note", ""),
-           "tags": c.get("tags", []), "members": members, "shape": c.get("shape", ""),
-           "explorers": explorers_for(len(members), s),
-           "reply": c.get("reply") or "all", "merger": c.get("merger", "")}
-    # What the members can handle together (metrics/catalog.py): the smallest
-    # context window is what the gateway refuses above.
-    try:
-        import catalog
-        out["together"] = catalog.consolidate(members)
-    except Exception:                                  # noqa: BLE001
-        out["together"] = None
-    return out
-
-
-def resolve(model: str | None) -> str | None:
-    """The collection a request's `model` names: `bench/core` or `core`; None
-    when it names nothing here. There is no default: a caller that did not
-    choose is told what each collection is for and asked to choose."""
-    name = (model or "").strip().removeprefix(PREFIX)   # "bench/" alone names nothing
-    return name if name and name in load() else None
-
-
-def split_reply(model: str) -> tuple[str, str]:
-    """`bench/core:merged` -> ("bench/core", "merged"), for clients that can
-    only pick a model name. No known suffix -> (model, ""): a collection name
-    cannot hold ':', so anything else stays and is refused as unknown."""
-    base, sep, mode = model.rpartition(":")
-    return (base, mode) if sep and mode in REPLIES else (model, "")
-
-
-def reply_errors(reply: str, merger: str, known) -> list[str]:
-    """What is wrong with a collection's reply mode and merger. `known`: the
-    chat models on the bench; the merger is any of them, member or not."""
-    errors = []
-    if reply not in REPLIES:
-        errors.append(f"reply must be one of {', '.join(REPLIES)}")
-    if merger and merger not in known:
-        errors.append(f"merger: {merger!r} is not a chat model on the bench; leave it empty to choose by verdicts")
-    return errors
-
-
-def choose_message(model: str | None) -> str:
-    """The refusal for a request that names no collection, or the wrong one:
-    every collection with its description and tags, so the caller can choose."""
-    asked = f"{model!r} is not a collection here." if model else "No collection was named."
-    rows = [f"- {PREFIX}{n}: {c.get('description') or 'no description'}"
-            + (f" [{', '.join(c.get('tags', []))}]" if c.get("tags") else "")
-            for n, c in sorted(load().items())]
-    return (f"{asked} Set \"model\" to the collection that fits the task"
-            " (GET /v1/models lists them with description and tags):\n"
-            + ("\n".join(rows) or "- none yet: create one in the dashboard's Collections dialog"))
-
-
-if __name__ == "__main__":
-    # DEFAULTS, not settings(): the install's gateway.json must not decide the check.
-    assert [explorers_for(n, DEFAULTS) for n in range(1, 7)] == [2, 2, 2, 1, 0, 0]
-    assert explorers_for(3, {"max_calls": 4, "max_explorers": 2}) == 1
-    assert resolve("bench/") is None and resolve("bench/no-such") is None and resolve("") is None
-    assert split_reply("bench/core:merged") == ("bench/core", "merged") and split_reply("core:all") == ("core", "all")
-    assert split_reply("bench/core") == ("bench/core", "") and split_reply("bench/core:x") == ("bench/core:x", "")
-    assert not reply_errors("merged", "", []) and not reply_errors("all", "m", ["m"])
-    assert len(reply_errors("", "m", [])) == 2
-    print("ok")
+"""Bench-gateway collections: named groups of models a caller asks for by name.
+
+    config/collections.json   {"core": {"members": [...], "shape": "",
+                                         "description": "...", "tags": ["general"],
+                                         "reply": "all", "merger": ""}}
+    config/gateway.json       {"max_calls": 5, "max_explorers": 2}
+
+A request for `bench/core` runs every member, plus explorer seats that
+`route.py` fills by staleness so the roster keeps getting evidence even when
+the same collection is called every day:
+
+    explorers = min(max_explorers, max_calls - members)
+
+`shape` (review, design, …) decides which models may fill those explorer
+seats; empty means a decision model labels each prompt, or `general` without
+one. `reply` is what the caller reads: `all` (every answer, the default) or
+`merged` (one answer a `merger` model writes from them; empty means chosen by
+verdict record, gateway.py). A request may override it with a `:merged` or
+`:all` suffix on the model name, or `"bench": {"reply": ...}` in the body.
+Named `bench_collections` because `collections` would shadow the standard
+library for everything importing from metrics/.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import re
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+FILE = DATA / "config" / "collections.json"
+SETTINGS_FILE = DATA / "config" / "gateway.json"
+DEFAULTS = {"max_calls": 5, "max_explorers": 2}
+NAME = re.compile(r"^[a-z0-9][a-z0-9-]{1,31}$")
+PREFIX = "bench/"
+REPLIES = ("all", "merged")
+
+
+def _read(path: pathlib.Path) -> dict:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def unreadable(path: pathlib.Path) -> bool:
+    """True if the file exists but is not a JSON object. A writer must refuse
+    then: `_read` would hand it {} and the save would wipe the file."""
+    if not path.exists():
+        return False
+    try:
+        return not isinstance(json.loads(path.read_text(encoding="utf-8")), dict)
+    except (OSError, ValueError):
+        return True
+
+
+def load() -> dict[str, dict]:
+    return _read(FILE)
+
+
+def settings() -> dict:
+    return {**DEFAULTS, **{k: v for k, v in _read(SETTINGS_FILE).items() if k in DEFAULTS}}
+
+
+def explorers_for(members: int, s: dict | None = None) -> int:
+    s = s or settings()
+    return max(0, min(s["max_explorers"], s["max_calls"] - members))
+
+
+TAG = re.compile(r"^[a-z0-9][a-z0-9-]{0,23}$")
+
+
+def described(name: str, c: dict, s: dict | None = None) -> dict:
+    """What a caller needs to choose a collection: purpose, tags and make-up.
+    Served on GET /v1/models, so an orchestrator can pick by task."""
+    members = c.get("members", [])
+    out = {"description": c.get("description") or c.get("note", ""),
+           "tags": c.get("tags", []), "members": members, "shape": c.get("shape", ""),
+           "explorers": explorers_for(len(members), s),
+           "reply": c.get("reply") or "all", "merger": c.get("merger", "")}
+    # What the members can handle together (metrics/catalog.py): the smallest
+    # context window is what the gateway refuses above.
+    try:
+        import catalog
+        out["together"] = catalog.consolidate(members)
+    except Exception:                                  # noqa: BLE001
+        out["together"] = None
+    return out
+
+
+def resolve(model: str | None) -> str | None:
+    """The collection a request's `model` names: `bench/core` or `core`; None
+    when it names nothing here. There is no default: a caller that did not
+    choose is told what each collection is for and asked to choose."""
+    name = (model or "").strip().removeprefix(PREFIX)   # "bench/" alone names nothing
+    return name if name and name in load() else None
+
+
+def split_reply(model: str) -> tuple[str, str]:
+    """`bench/core:merged` -> ("bench/core", "merged"), for clients that can
+    only pick a model name. No known suffix -> (model, ""): a collection name
+    cannot hold ':', so anything else stays and is refused as unknown."""
+    base, sep, mode = model.rpartition(":")
+    return (base, mode) if sep and mode in REPLIES else (model, "")
+
+
+def reply_errors(reply: str, merger: str, known) -> list[str]:
+    """What is wrong with a collection's reply mode and merger. `known`: the
+    chat models on the bench; the merger is any of them, member or not."""
+    errors = []
+    if reply not in REPLIES:
+        errors.append(f"reply must be one of {', '.join(REPLIES)}")
+    if merger and merger not in known:
+        errors.append(f"merger: {merger!r} is not a chat model on the bench; leave it empty to choose by verdicts")
+    return errors
+
+
+def choose_message(model: str | None) -> str:
+    """The refusal for a request that names no collection, or the wrong one:
+    every collection with its description and tags, so the caller can choose."""
+    asked = f"{model!r} is not a collection here." if model else "No collection was named."
+    rows = [f"- {PREFIX}{n}: {c.get('description') or 'no description'}"
+            + (f" [{', '.join(c.get('tags', []))}]" if c.get("tags") else "")
+            for n, c in sorted(load().items())]
+    return (f"{asked} Set \"model\" to the collection that fits the task"
+            " (GET /v1/models lists them with description and tags):\n"
+            + ("\n".join(rows) or "- none yet: create one in the dashboard's Collections dialog"))
+
+
+if __name__ == "__main__":
+    # DEFAULTS, not settings(): the install's gateway.json must not decide the check.
+    assert [explorers_for(n, DEFAULTS) for n in range(1, 7)] == [2, 2, 2, 1, 0, 0]
+    assert explorers_for(3, {"max_calls": 4, "max_explorers": 2}) == 1
+    assert resolve("bench/") is None and resolve("bench/no-such") is None and resolve("") is None
+    assert split_reply("bench/core:merged") == ("bench/core", "merged") and split_reply("core:all") == ("core", "all")
+    assert split_reply("bench/core") == ("bench/core", "") and split_reply("bench/core:x") == ("bench/core:x", "")
+    assert not reply_errors("merged", "", []) and not reply_errors("all", "m", ["m"])
+    assert len(reply_errors("", "m", [])) == 2
+    print("ok")