@wendellresearch-ai/benchwarmers 0.1.1 → 0.2.1

package/launcher.py

@@ -0,0 +1,120 @@
+"""The `benchwarmers` and `bench` commands, however Benchwarmers was installed.
+
+    pipx install benchwarmers        # PyPI: this file is the entry point (pyproject.toml)
+    npm install -g @wendellresearch-ai/benchwarmers    # npm: bin/benchwarmers.js finds Python and runs this
+    python launcher.py [command]     # a checkout
+
+It keeps the data in the user's profile (scripts/init.py's default_folder, or
+BENCHWARMERS_HOME), sets that folder up on the first run, then runs the
+command: a server (serve, gateway, mcp, init) or, for everything else, the
+client (cli.py) in the caller's own folder. One launcher for every install, so
+the commands, the help and the first run are written once.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent                 # the app: a checkout, the npm package or the wheel's folder
+SERVERS = {"serve": "serve.py", "gateway": "gateway.py", "mcp": "mcp_server.py",
+           "init": str(Path("scripts") / "init.py")}
+# Everything but running a server is cli.py, a client of a running server.
+CLIENT = ("status", "ask", "verdict", "models", "model", "candidates", "onboard", "dismiss",
+          "collections", "collection", "endpoints", "endpoint", "keys", "key", "settings")
+
+
+def version() -> str:
+    return (ROOT / "VERSION").read_text(encoding="utf-8").strip()
+
+
+def default_home() -> Path:
+    sys.path.insert(0, str(ROOT / "scripts"))
+    try:
+        import init
+        return init.default_folder()
+    finally:
+        sys.path.pop(0)
+
+
+def prog() -> str:
+    """`bench` or `benchwarmers`, as typed (npm passes it, being the one that saw it)."""
+    named = os.environ.get("BENCHWARMERS_PROG") or Path(sys.argv[0]).stem
+    return named if named in ("bench", "benchwarmers") else "benchwarmers"
+
+
+def usage() -> str:
+    return "\n".join([
+        f"Usage: {prog()} [command]",
+        "",
+        "Running it:",
+        "  serve        the dashboard, with the gateway at /v1: http://localhost:8122 (the default)",
+        "  gateway      the gateway alone: http://127.0.0.1:8123/v1",
+        "  init [...]   set up a data folder, optionally with a first endpoint (init --help)",
+        "  mcp          an MCP server on stdio for agents",
+        "",
+        "Using it (a client of a running server, BENCHWARMERS_URL; --help on any):",
+        "  status                          what is running, and where",
+        "  ask <collection> [prompt]       ask a collection; -f FILE adds a file, a pipe reads stdin",
+        "  verdict <call> --useful|--not-useful [--real ...] [--false ...]",
+        "  models | model <action>         list, rename, role, retire, restore, max-tokens, ...",
+        "  candidates | onboard | dismiss  what could be onboarded, and onboarding it",
+        "  collections | collection <action>   list, show, save, rm",
+        "  endpoints | endpoint <action>   list, add, test, key, rm, default",
+        "  keys | key <action>             gateway keys: list, create, revoke",
+        "  settings <gateway|client|scheduler>",
+        "  --version    the version",
+        "",
+        f"Your data lives in BENCHWARMERS_HOME, else {default_home()}.",
+        "BENCH_PORT changes the port. Docs: https://github.com/WendellResearch-AI/benchwarmers",
+    ])
+
+
+def plan(argv: list[str]) -> dict:
+    """What a command line means: {"say", "code"} to print and stop, or
+    {"script", "args", "cwd", "first_run"} to run. Pure, so it can be tested."""
+    first = argv[0] if argv else ""
+    command = first if first and not first.startswith("-") else "serve"
+    client = command in CLIENT
+    if first in ("--version", "-v"):
+        return {"say": version(), "code": 0}
+    # A client command's own --help is cli.py's, which knows its options; init has its own.
+    if command != "init" and not client and ("--help" in argv or "-h" in argv):
+        return {"say": usage(), "code": 0}
+    if not client and command not in SERVERS:
+        return {"say": f'{prog()}: no command "{command}"\n\n{usage()}', "code": 2, "stderr": True}
+    if client:                                         # cli.py takes the command itself
+        return {"script": "cli.py", "args": argv, "cwd": os.getcwd(), "first_run": False}
+    # The first run makes the data folder; not for mcp, whose stdout carries protocol messages only.
+    return {"script": SERVERS[command], "args": argv[1:] if command == first else argv, "cwd": str(ROOT),
+            "first_run": command not in ("init", "mcp")}
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    p = plan(argv)
+    if "say" in p:
+        print(p["say"], file=sys.stderr if p.get("stderr") else sys.stdout)
+        return p["code"]
+    home = Path(os.environ.get("BENCHWARMERS_HOME", "").strip() or default_home()).expanduser()
+    # No bytecode beside the code: an installed folder may not be writable, and
+    # an upgrade replaces it anyway.
+    env = {**os.environ, "BENCHWARMERS_HOME": str(home), "PYTHONDONTWRITEBYTECODE": "1", "PYTHONUTF8": "1",
+           "BENCHWARMERS_LAUNCHER": os.environ.get("BENCHWARMERS_LAUNCHER") or "pip"}
+    if p["first_run"] and not home.exists():
+        print(f"First run: making your data folder at {home}")
+        done = subprocess.run([sys.executable, str(ROOT / SERVERS["init"]), str(home)], env=env, cwd=ROOT)
+        if done.returncode:
+            return done.returncode
+    child = subprocess.Popen([sys.executable, str(ROOT / p["script"]), *p["args"]], env=env, cwd=p["cwd"])
+    while True:
+        try:
+            return child.wait()
+        except KeyboardInterrupt:                      # the child has it too; let it finish stopping
+            continue
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/mcp_server.py

@@ -0,0 +1,266 @@
+"""Benchwarmers as an MCP server, so an agent (Claude Code, or any MCP client)
+can ask a collection and send its verdicts.
+
+    benchwarmers mcp                 # from npm: npx -y @wendellresearch-ai/benchwarmers mcp
+    python mcp_server.py             # from a checkout
+
+MCP over stdio: JSON-RPC 2.0, one message per line, nothing else on stdout.
+A THIN CLIENT of a Benchwarmers server that is already running, over the same
+HTTP routes any other caller uses (GATEWAY.md), so the panel, the call log and
+the verdicts are that server's, whichever machine it is on:
+
+    BENCHWARMERS_URL       the server, default http://127.0.0.1:8122 (a /v1 on the end is fine)
+    BENCHWARMERS_KEY       a gateway key, sent as `Authorization: Bearer`; needed beyond
+                           this machine, or where BENCHWARMERS_REQUIRE_KEYS is set (metrics/guard.py)
+    BENCHWARMERS_TIMEOUT   seconds to wait for a panel, default 900
+
+Tools: list_collections (GET /v1/models), ask (POST /v1/chat/completions),
+verdict (POST /bench/verdict). A refusal from the server, or no server at
+all, comes back as a tool error carrying its message. Standard library only,
+and nothing from metrics/: the data folder is the server's, not this one's.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import threading
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+# Newest first: the client's version when it is one of these, else the newest.
+PROTOCOLS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05")
+PREFIX = "bench/"
+VERSION = (Path(__file__).resolve().parent / "VERSION").read_text(encoding="utf-8").strip()
+
+RULES = (
+    "Benchwarmers asks a COLLECTION of models at once (every member plus explorer seats it rotates in)"
+    " and keeps every answer. Rules: (1) ask a collection, not a favourite model; list_collections says"
+    " what each is for. (2) Verify before acting on anything a seat says: seats fail together, so"
+    " agreement is not evidence. (3) Send a verdict per seat you judged, with the claims it got right"
+    " in `real` and the untrue ones in `false`. (4) Name every seat that answered, by model and seat,"
+    " including errors and empty replies. An empty reply from a reasoning model is usually its token"
+    " budget, not incapacity. Good uses: exhaustive enumeration over real files, adversarial review."
+    " Keep the design and the decisions yourself.")
+
+TOOLS = [
+    {"name": "list_collections",
+     "title": "List collections",
+     "description": "The collections this Benchwarmers server offers: name, what each is for, tags, members,"
+                    " explorer seats, reply mode, and what the members handle together (smallest context"
+                    " window, shared capabilities, tightest rate limits). Pick by task before calling ask.",
+     "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False},
+     "annotations": {"readOnlyHint": True, "openWorldHint": False}},
+    {"name": "ask",
+     "title": "Ask a collection",
+     "description": "Put a prompt to every seat of one collection at once and get every answer back, each"
+                    " under its seat and model, plus the seats as data (alias, seat, status, call id). Can"
+                    " take minutes. Ask a collection, not a favourite model. Give the seats the real text"
+                    " (code, diff, file contents) in the prompt: they cannot read your files. Verify every"
+                    " claim against the source before acting on it, name every seat that answered in what"
+                    " you tell the user, then send a verdict per seat you judged.",
+     "inputSchema": {"type": "object", "required": ["collection", "prompt"], "additionalProperties": False,
+                     "properties": {
+                         "collection": {"type": "string",
+                                        "description": "A collection name, `core` or `bench/core` (list_collections)."},
+                         "prompt": {"type": "string", "description": "The question, with everything it needs."},
+                         "system": {"type": "string", "description": "A system message for every seat."},
+                         "merged": {"type": "boolean",
+                                    "description": "true: one reconciled answer, every seat still called and"
+                                                   " listed; false: every answer. Omit for the collection's own."},
+                         "with": {"type": "array", "items": {"type": "string"},
+                                  "description": "Model aliases to seat in explorer chairs, this request only."}}},
+     "annotations": {"readOnlyHint": False, "destructiveHint": False, "openWorldHint": True}},
+    {"name": "verdict",
+     "title": "Record a verdict",
+     "description": "Judge one seat's answer by its call id, once you have checked it against the source. One"
+                    " per seat you judged. List the claims it got right in `real` and the claims that are"
+                    " untrue in `false`, one sentence each: they feed routing and the 'confidently wrong'"
+                    " record, so a note alone is not enough.",
+     "inputSchema": {"type": "object", "required": ["call", "useful"], "additionalProperties": False,
+                     "properties": {
+                         "call": {"type": "string", "description": "The seat's call id, e.g. p20261022-101500-ab12.member1."},
+                         "useful": {"type": "boolean"},
+                         "note": {"type": "string", "description": "One or two sentences on why."},
+                         "real": {"type": "array", "items": {"type": "string"},
+                                  "description": "Claims it made that are true, one sentence each."},
+                         "false": {"type": "array", "items": {"type": "string"},
+                                   "description": "Claims it made that are untrue, one sentence each."}}},
+     "annotations": {"readOnlyHint": False, "destructiveHint": False, "idempotentHint": True,
+                     "openWorldHint": False}},
+]
+
+
+class ToolError(Exception):
+    """A tool call that failed: its message goes back as the tool's result, isError."""
+
+
+def base_url() -> str:
+    url = (os.environ.get("BENCHWARMERS_URL") or "").strip().rstrip("/") or "http://127.0.0.1:8122"
+    return url.removesuffix("/v1")
+
+
+def http(method: str, path: str, body: dict | None = None) -> dict:
+    url = base_url() + path
+    headers = {"Accept": "application/json"}
+    if key := (os.environ.get("BENCHWARMERS_KEY") or "").strip():
+        headers["Authorization"] = f"Bearer {key}"
+    data = None
+    if body is not None:
+        data = json.dumps(body).encode()
+        headers["Content-Type"] = "application/json"
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+    timeout = float(os.environ.get("BENCHWARMERS_TIMEOUT") or 900)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return json.load(r)
+    except urllib.error.HTTPError as e:
+        raw = e.read().decode("utf-8", "replace")
+        try:
+            message = json.loads(raw)["error"]["message"]
+        except (ValueError, KeyError, TypeError):
+            message = raw.strip()[:500] or e.reason
+        raise ToolError(f"Benchwarmers refused {method} {path} ({e.code}): {message}") from None
+    except (urllib.error.URLError, ConnectionError) as e:
+        reason = getattr(e, "reason", e)
+        if isinstance(reason, TimeoutError):
+            raise ToolError(f"no answer from Benchwarmers at {base_url()} in {timeout:.0f} s;"
+                            " set BENCHWARMERS_TIMEOUT for longer panels") from None
+        raise ToolError(f"Benchwarmers is not running at {base_url()} ({reason}). Start it (`benchwarmers`,"
+                        " or its service), or point BENCHWARMERS_URL at the server you use.") from None
+    except TimeoutError:
+        raise ToolError(f"no answer from Benchwarmers at {base_url()} in {timeout:.0f} s;"
+                        " set BENCHWARMERS_TIMEOUT for longer panels") from None
+    except ValueError:
+        raise ToolError(f"{base_url()} answered {method} {path} with something that is not JSON;"
+                        " is BENCHWARMERS_URL a Benchwarmers server?") from None
+
+
+def list_collections(_args: dict) -> list[str]:
+    rows = []
+    for m in http("GET", "/v1/models").get("data") or []:
+        if ":" in m.get("id", ""):                     # the same collection in its other reply mode
+            continue
+        together = m.get("together") or {}
+        rows.append({"name": m["id"], **{k: m.get(k) for k in ("description", "tags", "members", "explorers",
+                                                              "shape", "reply", "merger")},
+                     **{k: together.get(k) for k in ("context_window", "capabilities", "limits")}})
+    if not rows:
+        return ["No collections on this server (or none this gateway key may call). Make one on the"
+                " dashboard's Collections page."]
+    return [json.dumps(rows, indent=1, ensure_ascii=False)]
+
+
+def ask(args: dict) -> list[str]:
+    collection, prompt = args.get("collection"), args.get("prompt")
+    if not (isinstance(collection, str) and collection.strip()):
+        raise ToolError("`collection` is required: a name from list_collections, e.g. core")
+    if not (isinstance(prompt, str) and prompt.strip()):
+        raise ToolError("`prompt` is required")
+    collection = collection.strip()
+    messages = [{"role": "user", "content": prompt}]
+    if isinstance(args.get("system"), str) and args["system"].strip():
+        messages.insert(0, {"role": "system", "content": args["system"]})
+    body = {"model": collection if collection.startswith(PREFIX) else PREFIX + collection, "messages": messages}
+    if isinstance(args.get("merged"), bool):
+        body["bench"] = {"reply": "merged" if args["merged"] else "all"}
+    if args.get("with"):
+        body["with"] = args["with"]
+    reply = http("POST", "/v1/chat/completions", body)
+    content = reply["choices"][0]["message"]["content"]
+    bench = reply.get("bench") or {}
+    merged = bench.get("reply") == "merged"
+    # Each answer is already in the content unless it is the merge, so the
+    # seat data stays short: who sat, how it went, the id to judge it by.
+    keep = ("call", "seat", "alias", "status", "seconds", "error") + (("answer",) if merged else ())
+    seats = [{k: s[k] for k in keep if s.get(k) not in (None, "")} for s in bench.get("seats") or []]
+    for s in seats:
+        if "error" in s:
+            s["error"] = s["error"][-300:]             # a traceback's last lines say what went wrong
+    data = {"panel": bench.get("panel"), "collection": bench.get("collection"), "shape": bench.get("shape"),
+            "seats": seats, **{k: bench[k] for k in ("merger", "skipped", "note") if bench.get(k)}}
+    return [content, "Seats (name each one; send a verdict per seat you judged, by call id):\n"
+            + json.dumps(data, indent=1, ensure_ascii=False)]
+
+
+def verdict(args: dict) -> list[str]:
+    if not (isinstance(args.get("call"), str) and args["call"].strip()):
+        raise ToolError("`call` is required: the seat's call id from ask")
+    if not isinstance(args.get("useful"), bool):
+        raise ToolError("`useful` must be true or false")
+    body = {"call": args["call"].strip(), "useful": args["useful"], "note": args.get("note") or "",
+            "real": args.get("real") or [], "false": args.get("false") or []}
+    http("POST", "/bench/verdict", body)
+    return [f"Recorded {body['call']}: {'useful' if body['useful'] else 'not useful'},"
+            f" {len(body['real'])} real, {len(body['false'])} false."]
+
+
+HANDLERS = {"list_collections": list_collections, "ask": ask, "verdict": verdict}
+OUT = threading.Lock()
+
+
+def send(message: dict) -> None:
+    line = json.dumps(message, ensure_ascii=False) + "\n"
+    with OUT:
+        sys.stdout.buffer.write(line.encode("utf-8"))
+        sys.stdout.buffer.flush()
+
+
+def call_tool(rid, params: dict) -> None:
+    name, args = params.get("name"), params.get("arguments") or {}
+    if name not in HANDLERS:
+        return send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"unknown tool {name!r}"}})
+    try:
+        if not isinstance(args, dict):
+            raise ToolError("arguments must be an object")
+        texts, failed = HANDLERS[name](args), False
+    except ToolError as e:
+        texts, failed = [str(e)], True
+    except Exception as e:                             # noqa: BLE001 -- a bad reply must not kill the server
+        texts, failed = [f"{type(e).__name__}: {e}"], True
+    send({"jsonrpc": "2.0", "id": rid,
+          "result": {"content": [{"type": "text", "text": t} for t in texts], "isError": failed}})
+
+
+def handle(msg) -> None:
+    if not isinstance(msg, dict) or msg.get("jsonrpc") != "2.0" or not isinstance(msg.get("method"), str):
+        rid = msg.get("id") if isinstance(msg, dict) else None
+        return send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32600, "message": "invalid request"}})
+    if "id" not in msg:                                # a notification (initialized, cancelled): nothing to answer
+        return
+    rid, method, params = msg["id"], msg["method"], msg.get("params") or {}
+    if method == "initialize":
+        asked = params.get("protocolVersion")
+        send({"jsonrpc": "2.0", "id": rid, "result": {
+            "protocolVersion": asked if asked in PROTOCOLS else PROTOCOLS[0],
+            "capabilities": {"tools": {"listChanged": False}},
+            "serverInfo": {"name": "benchwarmers", "title": "Benchwarmers", "version": VERSION},
+            "instructions": RULES}})
+    elif method == "ping":
+        send({"jsonrpc": "2.0", "id": rid, "result": {}})
+    elif method == "tools/list":
+        send({"jsonrpc": "2.0", "id": rid, "result": {"tools": TOOLS}})
+    elif method == "tools/call":
+        # A panel takes minutes: on its own thread, so a ping or a second call is answered meanwhile.
+        # Not a daemon: when stdin closes, a call already sent (a verdict) still finishes.
+        threading.Thread(target=call_tool, args=(rid, params)).start()
+    else:
+        send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"no method {method!r}"}})
+
+
+def main() -> None:
+    for raw in sys.stdin.buffer:
+        if not raw.strip():
+            continue
+        try:
+            msg = json.loads(raw)
+        except ValueError:
+            send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error"}})
+            continue
+        handle(msg)
+
+
+if __name__ == "__main__":
+    main()

package/metrics/alerts.py

@@ -1,116 +1,116 @@
-"""Things that went wrong and are still wrong, where they will be seen.
-
-    python metrics/alerts.py            # what is currently wrong
-
-The scheduled checks already find problems. Until now they announced them by
-exiting non-zero, which Windows records and nobody reads — a model could be
-retired for a fortnight and the only trace would be a number in Task
-Scheduler. Work that reports only to a log nobody opens has not reported.
-
-KEYED AND SELF-CLEARING, NOT AN APPEND LOG
-------------------------------------------
-An alert has a stable `key`, so a daily check that keeps finding the same
-retired model updates one row rather than writing thirty. The first sighting is
-kept, because "missing since Tuesday" is the useful part — a model absent for a
-day may be an outage and one absent for a week is gone.
-
-**A check that finds nothing CLEARS its alerts.** Otherwise the file only ever
-grows and the dashboard turns into a wall of stale red that everyone learns to
-ignore, which is worse than no alerting at all. `sweep()` exists so a caller
-can say "these are all the problems I found" and have the rest retired
-automatically.
-
-WHAT IS NOT AN ALERT
---------------------
-Anything transient. A rate limit, a timeout, one failed call — those are
-`capacity` and they resolve themselves. An alert is for a state that will not
-fix itself and that somebody has to decide about: a model retired upstream, a
-registration pointing at a name that does not exist, a subscription that no
-longer covers what it did.
-"""
-
-from __future__ import annotations
-
-import datetime as dt
-import pathlib
-
-import locked
-
-ROOT = pathlib.Path(__file__).resolve().parent.parent
-from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
-FILE = DATA / "metrics" / "alerts.json"
-
-
-def _load() -> dict:
-    return locked.read_json(FILE)
-
-
-def _update(change):
-    """Read-change-write under a lock: the gateway's seat threads, the
-    scheduled checks and the dashboard all raise and clear alerts."""
-    return locked.update(FILE, change, sort_keys=True)
-
-
-def raise_alert(key: str, kind: str, message: str, action: str = "",
-                **extra) -> None:
-    """Record a problem, keeping the date it was first seen.
-
-    `extra` carries what the dashboard needs to ACT on it rather than just
-    show it: `alias` for a one-model action, `aliases` for retire-all.
-    """
-    today = dt.date.today().isoformat()
-
-    def put(data: dict) -> None:
-        data[key] = {
-            "kind": kind,
-            "message": message,
-            "action": action,
-            "first_seen": data.get(key, {}).get("first_seen", today),
-            "last_seen": today,
-            **extra,
-        }
-    _update(put)
-
-
-def clear(key: str) -> bool:
-    return _update(lambda data: data.pop(key, None) is not None)
-
-
-def sweep(prefix: str, still_wrong: set[str]) -> list[str]:
-    """Retire every alert under `prefix` that the caller did not re-raise.
-
-    This is what stops the file growing for ever. A check reports the complete
-    set of problems it found, and anything it used to find but no longer does
-    is resolved — a model that came back should not leave a red row behind it.
-    """
-    def retire(data: dict) -> list[str]:
-        gone = [k for k in data
-                if k.startswith(prefix) and k not in still_wrong]
-        for key in gone:
-            del data[key]
-        return gone
-    return _update(retire)
-
-
-def active() -> dict:
-    return _load()
-
-
-def main() -> None:
-    data = active()
-    if not data:
-        print("Nothing wrong.")
-        return
-    today = dt.date.today()
-    print(f"{len(data)} open:\n")
-    for key, a in sorted(data.items(),
-                         key=lambda kv: kv[1]["first_seen"]):
-        days = (today - dt.date.fromisoformat(a["first_seen"])).days
-        age = "today" if days == 0 else f"{days}d"
-        print(f"  [{a['kind']}] {a['message']}")
-        print(f"      first seen {a['first_seen']} ({age})"
-              + (f" — {a['action']}" if a["action"] else ""))
-
-
-if __name__ == "__main__":
-    main()
+"""Things that went wrong and are still wrong, where they will be seen.
+
+    python metrics/alerts.py            # what is currently wrong
+
+The scheduled checks already find problems. Until now they announced them by
+exiting non-zero, which Windows records and nobody reads — a model could be
+retired for a fortnight and the only trace would be a number in Task
+Scheduler. Work that reports only to a log nobody opens has not reported.
+
+KEYED AND SELF-CLEARING, NOT AN APPEND LOG
+------------------------------------------
+An alert has a stable `key`, so a daily check that keeps finding the same
+retired model updates one row rather than writing thirty. The first sighting is
+kept, because "missing since Tuesday" is the useful part — a model absent for a
+day may be an outage and one absent for a week is gone.
+
+**A check that finds nothing CLEARS its alerts.** Otherwise the file only ever
+grows and the dashboard turns into a wall of stale red that everyone learns to
+ignore, which is worse than no alerting at all. `sweep()` exists so a caller
+can say "these are all the problems I found" and have the rest retired
+automatically.
+
+WHAT IS NOT AN ALERT
+--------------------
+Anything transient. A rate limit, a timeout, one failed call — those are
+`capacity` and they resolve themselves. An alert is for a state that will not
+fix itself and that somebody has to decide about: a model retired upstream, a
+registration pointing at a name that does not exist, a subscription that no
+longer covers what it did.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import pathlib
+
+import locked
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+from paths import DATA  # noqa: E402  -- the install's data folder (metrics/paths.py)
+FILE = DATA / "metrics" / "alerts.json"
+
+
+def _load() -> dict:
+    return locked.read_json(FILE)
+
+
+def _update(change):
+    """Read-change-write under a lock: the gateway's seat threads, the
+    scheduled checks and the dashboard all raise and clear alerts."""
+    return locked.update(FILE, change, sort_keys=True)
+
+
+def raise_alert(key: str, kind: str, message: str, action: str = "",
+                **extra) -> None:
+    """Record a problem, keeping the date it was first seen.
+
+    `extra` carries what the dashboard needs to ACT on it rather than just
+    show it: `alias` for a one-model action, `aliases` for retire-all.
+    """
+    today = dt.date.today().isoformat()
+
+    def put(data: dict) -> None:
+        data[key] = {
+            "kind": kind,
+            "message": message,
+            "action": action,
+            "first_seen": data.get(key, {}).get("first_seen", today),
+            "last_seen": today,
+            **extra,
+        }
+    _update(put)
+
+
+def clear(key: str) -> bool:
+    return _update(lambda data: data.pop(key, None) is not None)
+
+
+def sweep(prefix: str, still_wrong: set[str]) -> list[str]:
+    """Retire every alert under `prefix` that the caller did not re-raise.
+
+    This is what stops the file growing for ever. A check reports the complete
+    set of problems it found, and anything it used to find but no longer does
+    is resolved — a model that came back should not leave a red row behind it.
+    """
+    def retire(data: dict) -> list[str]:
+        gone = [k for k in data
+                if k.startswith(prefix) and k not in still_wrong]
+        for key in gone:
+            del data[key]
+        return gone
+    return _update(retire)
+
+
+def active() -> dict:
+    return _load()
+
+
+def main() -> None:
+    data = active()
+    if not data:
+        print("Nothing wrong.")
+        return
+    today = dt.date.today()
+    print(f"{len(data)} open:\n")
+    for key, a in sorted(data.items(),
+                         key=lambda kv: kv[1]["first_seen"]):
+        days = (today - dt.date.fromisoformat(a["first_seen"])).days
+        age = "today" if days == 0 else f"{days}d"
+        print(f"  [{a['kind']}] {a['message']}")
+        print(f"      first seen {a['first_seen']} ({age})"
+              + (f" — {a['action']}" if a["action"] else ""))
+
+
+if __name__ == "__main__":
+    main()