@wendellresearch-ai/benchwarmers 0.5.5 → 0.6.0

package/metrics/catalog.py

@@ -31,6 +31,12 @@ WHERE THE FACTS COME FROM
   the same weights, but a provider can cap context lower.
 - Open WebUI's /api/models, where a connection passes metadata through
   (Inception, Reflection do; most do not).
+- A local model server's own API, for what its /v1/models leaves out: LM
+  Studio's /api/v1/models (vision, tool use, reasoning; the context window a
+  loaded model runs with, else the most it can take) and Ollama's /api/show
+  (its capabilities, and `<arch>.context_length` from model_info, the trained
+  window: Ollama may run it with less). Like a connection's own listing, these
+  beat a same-id match from anywhere else.
 - Everything else, including rate limits, by hand: freellmapi.co/models is a
   good place to look them up. The bench never fetches it; its data licence is
   not stated.
@@ -64,8 +70,9 @@ LIMIT_LABELS = {"rpm": "requests per minute", "rpd": "requests per day", "tpm":
 
 
 def _read(path: pathlib.Path) -> dict:
+    import locked
     try:
-        data = json.loads(path.read_text(encoding="utf-8"))
+        data = json.loads(locked.read_text(path))      # inside a render, from its snapshot
     except (OSError, ValueError):
         return {}
     return data if isinstance(data, dict) else {}
@@ -151,6 +158,39 @@ def _fetch(url: str, headers: dict | None = None) -> dict:
         return json.load(r)
 
 
+def _native(name: str, served) -> tuple[str, dict]:
+    """What a local model server says of its own models that its /v1/models
+    leaves out: LM Studio's /api/v1/models, else Ollama's /api/show, one call
+    for each of `served`. (source, {served id: facts}); ("", {}) from any other."""
+    import connect
+    import endpoints
+    root, secret = endpoints.base(name).removesuffix("/v1"), endpoints.key(name)
+    auth = endpoints.auth_headers("chat", secret) if secret else {}
+    status, data = connect.look(root + "/api/v1/models", 5, headers=auth)
+    out: dict[str, dict] = {}
+    if status == 200 and isinstance(data, dict) and isinstance(data.get("models"), list):
+        for m in data["models"]:
+            caps, runs = m.get("capabilities") or {}, m.get("loaded_instances") or []
+            # Loaded: the window it runs with now; else the most it can take.
+            ctx = min(filter(None, ((i.get("config") or {}).get("context_length") for i in runs)),
+                      default=m.get("max_context_length"))
+            facts = {"context_window": ctx, "vision": caps.get("vision"),
+                     "tools": caps.get("trained_for_tool_use"), "reasoning": "reasoning" in caps}
+            # A request names the key, a variant (key@q4_k_m) or a loaded instance's id.
+            for i in (m.get("key"), *(m.get("variants") or ()), *(r.get("id") for r in runs)):
+                out[i] = facts
+        return "lmstudio /api/v1/models", out
+    for s in served:
+        status, d = connect.look(root + "/api/show", 5, {"model": s}, auth)
+        if status == 200 and isinstance(d, dict) and isinstance(d.get("capabilities"), list):
+            caps, mi = d["capabilities"], d.get("model_info") or {}
+            # ponytail: the trained window; Ollama runs a model with its own num_ctx
+            # (OLLAMA_CONTEXT_LENGTH, a Modelfile's) and cuts a longer prompt silently.
+            out[s] = {"context_window": mi.get(f"{mi.get('general.architecture')}.context_length"),
+                      "vision": "vision" in caps, "tools": "tools" in caps, "reasoning": "thinking" in caps}
+    return ("ollama /api/show" if out else ""), out
+
+
 def _watch_price(alias: str, served: str, conn: str, old: dict | None, new: dict | None) -> None:
     """A free model that now has a price: raise it, before calls start failing.
     models.dev can lag, so this is an early warning, not proof; the bench's own
@@ -173,14 +213,11 @@ def _watch_price(alias: str, served: str, conn: str, old: dict | None, new: dict
 
 def refresh() -> dict:
     """Rebuild config/model-catalog.json for every registered chat model."""
+    import discover
     import endpoints
     import sources
     from run import _registered_names
-    for name in sources.FEEDS:
-        try:
-            sources.fetch(name)                      # at most once a day; cached copy otherwise
-        except Exception:                            # noqa: BLE001
-            pass
+    discover.refresh()                               # the catalogs and providers' lists, at most once a day
     before = catalog()
     registered = _registered_names()                   # alias -> served name
     try:
@@ -191,11 +228,17 @@ def refresh() -> dict:
     # is registered on: the direct openrouter endpoint lists qwen/qwen3.8-27b
     # too, and pooling the listings filed OpenRouter's 1M window as groq's.
     listings: dict[str, dict] = {}
+    native: dict[str, tuple[str, dict]] = {}           # a local server's own API, what /v1/models leaves out
     for name in endpoints.load():
         try:
             listings[name] = endpoints.models(name)
         except Exception:                              # noqa: BLE001
             listings[name] = {}
+        try:
+            if endpoints.kind(name) == "local":
+                native[name] = _native(name, listings[name])
+        except Exception:                              # noqa: BLE001 -- an answer of another shape: nothing from it
+            pass
     home = {r["alias"]: r["endpoint"] for r in endpoints.registry()}
     models = {}
     for alias, served in registered.items():
@@ -272,9 +315,11 @@ def refresh() -> dict:
                        structured_output=bool(feats & {"json_mode", "structured_outputs"}))
         if entry.get("input_modalities") is not None:
             own["vision"] = "image" in entry["input_modalities"]
-        for k, v in own.items():
-            if v is not None:
-                rec[k], src[k] = v, f"{ep or 'endpoint'} listing"
+        said, facts = native.get(ep, ("", {}))
+        for mine, by in ((own, f"{ep or 'endpoint'} listing"), (facts.get(served, {}), said)):
+            for k, v in mine.items():
+                if v is not None:
+                    rec[k], src[k] = v, by
         rec["sources"] = src
         models[alias] = rec
         _watch_price(alias, served, conn, before.get(alias, {}).get("cost"), rec.get("cost"))

package/metrics/client.py

@@ -57,7 +57,7 @@ down sets.
 WHAT EVERY CALL LEAVES BEHIND (2b)
 ----------------------------------
 - a ledger line (metrics/ledger.py): tokens in and out, seconds, status,
-  attempts, caller;
+  attempts, caller, and a failed call's error (`clip`: its start and end);
 - quota records from the reply's rate-limit headers and any 429 body
   (metrics/quota.py);
 - a silent-swap note when the reply names another model than the one
@@ -119,6 +119,7 @@ import json
 import os
 import pathlib
 import random
+import re
 import sys
 import threading
 import time
@@ -852,6 +853,11 @@ def _record(alias: str, res: dict, ctx: dict, t0: float, purpose: str) -> None:
             row["swap"] = True
         if res.get("budget_retry"):
             row["budget_retry"] = res["budget_retry"]
+        if status == "error":
+            # What failed, so the uptime report can tell our quota from the
+            # provider being down for calls no other log keeps (the suite's).
+            # Redacted first: an upstream may echo the request, images and all.
+            row["error"] = clip(redact(res["error"] or ""))
         ledger.append(row)
     except Exception as e:                             # noqa: BLE001
         # The slot stays taken until it expires (the call's deadline and a
@@ -906,6 +912,45 @@ def output(res: dict, timeout: float) -> str:
     return res["error"] or (res["text"] + "\n" if res["text"] else "")
 
 
+DATA_URLS = re.compile(r"data:[\w/.+-]*;base64,[A-Za-z0-9+/=_\\-]*", re.I)
+URLS = re.compile(r"https?://[^\s\"'<>\\]+", re.I)
+BASE64 = re.compile(r"[A-Za-z0-9+/]{100,}={0,2}")
+
+
+def placeholder(url: str) -> str:
+    """How an image is written down (the call log, offloads, the dashboard,
+    error text, the merger's copy of the request), never its bytes: a data
+    URL by its size, an http URL without its query (a signed URL's signature
+    is a credential) or a user:password."""
+    if url[:5].lower() == "data:":
+        return f"[image: data URL, {(len(url) * 3 // 4 + 999) // 1000:,} KB]"
+    u = urllib.parse.urlsplit(url)
+    return f"[image: {u.scheme}://{u.netloc.rpartition('@')[2]}{u.path[:200]}{'?...' if u.query else ''}]"
+
+
+def redact(text: str) -> str:
+    """`text` with each inline image as its placeholder, every http(s) URL
+    without its query or user:password, and any other long base64 run as its
+    size: an upstream may echo the request in an error, in whatever form, and
+    errors are logged, shown and learned from."""
+    def bare(m):
+        try:
+            u = urllib.parse.urlsplit(m.group())
+        except ValueError:                             # an unparseable netloc, such as "[::1"
+            return "[url]"
+        return f"{u.scheme}://{u.netloc.rpartition('@')[2]}{u.path}{'?...' if u.query else ''}"
+    text = URLS.sub(bare, DATA_URLS.sub(lambda m: placeholder(m.group()), text))
+    return BASE64.sub(lambda m: f"[base64, {(len(m.group()) * 3 // 4 + 999) // 1000:,} KB]", text)
+
+
+def clip(text: str, head: int = 200, tail: int = 300) -> str:
+    """An error kept short for a log, its start AND its end: the start says the
+    HTTP status (`Error: Error code: 429 - ...`), the end what a long body
+    (headers, a traceback, an HTML page) finally said. The uptime report
+    (suite/run.uptime_class) classifies from it."""
+    return text if len(text) <= head + tail else f"{text[:head]} … {text[-tail:]}"
+
+
 LOG_LOCKED = "sqlite3.OperationalError: database is locked"
 
 
@@ -930,6 +975,11 @@ def _selftest() -> None:
     assert not llm_log_failed_only("", tb)                                  # locked before the call
     assert not llm_log_failed_only("half an ans", "openai.APIError: stream cut\n")
     assert not llm_log_failed_only("pong", tb + "Error: something after\n")
+    # A long error keeps its status (the start) and its last words (the end), and no image.
+    long = "Error: Error code: 429 - " + "x" * 900 + " retry later"
+    assert clip(long).startswith("Error: Error code: 429") and clip(long).endswith("retry later")
+    assert len(clip(long)) == 503 and clip("short") == "short"
+    assert "AAAA" not in clip(redact("Error code: 400 - data:image/png;base64," + "A" * 900))
     ok = (b'event: x\ndata: {"type":"response.output_text.delta","delta":"re"}\n\n'
           b'data: {"type":"response.completed","response":{"model":"m","usage":{"total_tokens":3},'
           b'"output":[{"type":"reasoning"},{"type":"message","content":[{"type":"output_text","text":"ready"}]}]}}\n')

package/metrics/connect.py

@@ -60,6 +60,20 @@ ADDING A PROVIDER
 3. Run `python metrics/connect.py --selftest`: it checks every entry's shape,
    and every flow against a fake provider.
 4. Add an endpoint on the Endpoints page, pick the provider and sign in once for real.
+
+FOUND ON THIS MACHINE
+---------------------
+`detect()` looks for model servers running here (LOCAL: Ollama, LM Studio,
+Jan, llama.cpp, LocalAI, vLLM, LiteLLM, CLI Proxy API, OmniRoute), so the
+Endpoints page can offer each one with its address and kind filled in. It
+asks their usual ports, and any port whose listening process is one of them
+by name or command line (`listeners()`), since a port can be changed.
+Loopback only, GET only, no key sent, every port at once. A port alone is not
+proof: each answers /v1/models (or refuses it without a key), and where
+something else could sit on its port, a `sign` from its docs or source says
+which it is (llama.cpp and LocalAI share 8080); without one, the process
+does. Adding one is the ordinary Add form, test before save and all; Ollama
+at its usual port keeps its own flow.
 """
 
 from __future__ import annotations
@@ -68,6 +82,7 @@ import base64
 import datetime as dt
 import hashlib
 import json
+import os
 import pathlib
 import re
 import secrets
@@ -165,6 +180,211 @@ PROVIDERS: dict[str, dict] = {
 }
 
 
+def _rows(data) -> list:
+    """An OpenAI /models answer's entries, [] for anything else."""
+    rows = data.get("data") if isinstance(data, dict) else None
+    return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []
+
+
+def _version(status: int, data) -> bool:
+    """{"version": ...}: Ollama's /api/version, vLLM's and LocalAI's /version."""
+    return status == 200 and isinstance(data, dict) and "version" in data
+
+
+# Model servers detect() looks for: their usual ports; `sign`, a key-free GET
+# and a test of its (status, answer) where the port alone could be another
+# server, none where the port is its own; the endpoint each becomes; where that
+# is said, read 2026-11-08. llama.cpp moved its default from 8080 to 9931
+# (README, --port), so both are asked. Order matters on a shared port: the
+# first whose sign holds. `procs`, how its process's name starts (".exe" and
+# case aside), and `cmd`, a word in its command line for one run by Python or
+# Node: from release binaries, installers and launch code, read 2026-11-09.
+LOCAL: dict[str, dict] = {
+    "ollama": {"label": "Ollama", "ports": (11434,), "kind": "local", "provider": "ollama",
+               "sign": ("/api/version", _version), "procs": ("ollama",),
+               "docs": "https://github.com/ollama/ollama/blob/main/docs/api.md#version"},
+    # The app (Scoop's manifest, LM Studio.exe) or its headless daemon (lmstudio.ai/install.sh, llmster).
+    "lmstudio": {"label": "LM Studio", "ports": (1234,), "kind": "local", "procs": ("lm studio", "llmster"),
+                 # Its own REST API: {"models": [{"key", "capabilities", ...}]}, or a refusal when it wants a token.
+                 "sign": ("/api/v1/models", lambda s, d: s in (401, 403) or (
+                     s == 200 and isinstance(d, dict) and isinstance(d.get("models"), list))),
+                 "docs": "https://lmstudio.ai/docs/developer/rest/list"},
+    # The server runs in the app itself: Jan, now Jan-Desktop (src-tauri/tauri.conf.json).
+    "jan": {"label": "Jan", "ports": (1337,), "kind": "local", "procs": ("jan",),
+            "docs": "https://docs.jan.ai/desktop/api-server"},
+    "llamacpp": {"label": "llama.cpp server", "ports": (9931, 8080), "kind": "local", "procs": ("llama-server",),
+                 # Its /v1/models is public even with --api-key, each model owned_by "llamacpp".
+                 "sign": ("/v1/models", lambda s, d: any(r.get("owned_by") == "llamacpp" for r in _rows(d))),
+                 "docs": "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md"},
+    "localai": {"label": "LocalAI", "ports": (8080,), "kind": "local",
+                # No owned_by in its /v1/models (endpoints/openai/list.go); /version has no middleware.
+                "sign": ("/version", _version), "procs": ("local-ai",),   # also local-ai-v4.12.1-linux-amd64
+                "docs": "https://github.com/mudler/LocalAI/blob/master/core/http/routes/localai.go"},
+    "vllm": {"label": "vLLM", "ports": (8000,), "kind": "local", "cmd": ("vllm",),
+             "sign": ("/version", _version),          # --api-key guards /v1 and a few others, not this
+             "docs": "https://docs.vllm.ai/en/latest/cli/serve.html"},
+    "litellm": {"label": "LiteLLM", "ports": (4000,), "kind": "hub", "hub": "litellm", "cmd": ("litellm",),
+                "sign": ("/health/liveliness", lambda s, d: s == 200 and "alive" in str(d)),
+                "docs": "https://docs.litellm.ai/docs/proxy/health"},
+    # Its releases, Homebrew's formula, and its Docker image (CLIProxyAPI).
+    "cliproxy": {"label": "CLI Proxy API", "ports": (8317,), "kind": "hub", "hub": "cliproxy",
+                 "sign": ("/", lambda s, d: isinstance(d, dict) and d.get("message") == "CLI Proxy API Server"),
+                 "procs": ("cli-proxy-api", "cliproxyapi"),
+                 "docs": "https://github.com/router-for-me/CLIProxyAPI/blob/main/internal/api/server_routes.go"},
+    # Node running node_modules/omniroute/dist/server.js, or 9router/app/custom-server.js.
+    "omniroute": {"label": "OmniRoute / 9router", "ports": (20128,), "kind": "aggregator",
+                  "cmd": ("omniroute", "9router"), "docs": "https://github.com/decolua/9router"},
+}
+
+
+def look(url: str, timeout: float = 0.5, body: dict | None = None, headers: dict | None = None) -> tuple[int, object]:
+    """(status, the answer as JSON or text); (0, None) when nothing usable
+    answers. A GET, or a POST of `body` as JSON. Never through a proxy or a
+    redirect: it reaches `url`'s own host or nothing, a key in `headers` too."""
+    import client                                # its no-redirect handler
+    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), client._NoRedirect)
+    sent = {**UA, **({"Content-Type": "application/json"} if body else {}), **(headers or {})}
+    try:
+        try:
+            with opener.open(urllib.request.Request(url, json.dumps(body).encode() if body else None, sent),
+                             timeout=timeout) as r:
+                status, raw = r.status, r.read(1 << 20)
+        except urllib.error.HTTPError as e:
+            status, raw = e.code, e.read(1 << 16)
+    except Exception:                            # noqa: BLE001 -- refused, timed out, or not HTTP at all
+        return 0, None
+    try:
+        return status, json.loads(raw)
+    except ValueError:
+        return status, raw.decode("utf-8", "replace")
+
+
+def listeners() -> list[tuple[int, int, str, str]]:
+    """(pid, port, process name, command line) of each TCP listener on loopback
+    or all addresses, from the OS's own tools: netstat and PowerShell's process
+    list on Windows, ss on Linux, lsof on macOS, ps for a command line there.
+    [] when a tool is missing or fails (a container, no rights to see), which
+    leaves detect() the usual ports."""
+    import subprocess
+
+    def run(*cmd: str) -> str:
+        return subprocess.run(cmd, capture_output=True, text=True, errors="replace", timeout=5,
+                              creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)).stdout
+    try:
+        if sys.platform == "win32":
+            # A listener's far end is port 0; its state's name is translated, so not asked.
+            socks = [(int(p[4]), p[1]) for p in map(str.split, run("netstat", "-ano").splitlines())
+                     if len(p) == 5 and p[0] == "TCP" and p[2].endswith(":0")]
+            # Names and command lines in one look, sooner than tasklist gives names alone.
+            info = {p["ProcessId"]: (p["Name"] or "", p["CommandLine"] or "") for p in json.loads(run(
+                "powershell", "-NoProfile", "-NonInteractive", "-Command",
+                "Get-CimInstance Win32_Process | Select ProcessId,Name,CommandLine | ConvertTo-Json -Compress"))}
+        else:                                    # (pid, address, name), then each one's command line
+            if sys.platform == "darwin":
+                found, pid, name = [], 0, ""
+                for line in run("lsof", "-nP", "-iTCP", "-sTCP:LISTEN", "-Fpcn", "+c", "0").splitlines():
+                    if line[:1] == "p":
+                        pid = int(line[1:])
+                    elif line[:1] == "c":
+                        name = line[1:]
+                    elif line[:1] == "n":
+                        found.append((pid, line[1:], name))
+            else:                                # ... users:(("ollama",pid=812,fd=3))
+                found = [(int(pid), p[3], name) for p in map(str.split, run("ss", "-ltnp").splitlines())
+                         if p[:1] == ["LISTEN"] for name, pid in re.findall(r'"([^"]*)",pid=(\d+)', " ".join(p[5:]))]
+            pids = ",".join({str(f[0]) for f in found})
+            args = dict(line.strip().partition(" ")[::2] for line in run("ps", "-o", "pid=,args=", "-p", pids).splitlines())
+            socks, info = [f[:2] for f in found], {f[0]: (f[2], args.get(str(f[0]), "")) for f in found}
+    except Exception:                            # noqa: BLE001 -- missing, refused, timed out, or not what was expected
+        return []
+    rows = []
+    for pid, addr in socks:
+        host, _, port = addr.rpartition(":")
+        if host.strip("[]") in ("0.0.0.0", "127.0.0.1", "::", "::1", "*") and port.isdigit():
+            rows.append((pid, int(port), *info.get(pid, ("", ""))))
+    return rows
+
+
+def detect(skip=(), timeout: float = 0.5) -> list[dict]:
+    """The LOCAL servers answering on this machine, but none on a port in
+    `skip` (this install's own) or already an endpoint. Each: id, label, port,
+    base, kind, hub, provider (Ollama's Connect flow), models (None when the
+    list needs a key) and key. Asked: each one's usual ports, and the ports of
+    processes that are one of them (`procs`, `cmd`), or Ollama's own
+    OLLAMA_HOST; a port no such process holds is asked only if it is usual.
+    Every port at once, so a look takes the listing (about a second) and
+    `timeout`, a server that hangs included. Loopback only, so a GET here
+    never leaves this machine."""
+    from concurrent.futures import ThreadPoolExecutor
+    taken = set(skip) | _endpoint_ports()
+    by_port: dict[int, list[str]] = {}
+    host = re.search(r":(\d+)/?$", os.environ.get("OLLAMA_HOST", "").strip())   # as 0.0.0.0:11435
+    # ponytail: the listing (about 0.8 s on Windows), then the probes; overlap them if a look must be quicker
+    running = listeners() + ([(0, int(host[1]), "ollama", "")] if host else [])
+    matched = [(port, sid) for _, port, name, cmd in running for sid, s in LOCAL.items()
+               if name.lower().startswith(s.get("procs", ())) or any(c in cmd.lower() for c in s.get("cmd", ()))]
+    for port, sid in matched + [(port, sid) for sid, s in LOCAL.items() for port in s["ports"]]:
+        if port not in taken and sid not in by_port.setdefault(port, []):
+            by_port[port].append(sid)
+
+    def identify(port: int, ids: list[str]) -> dict | None:
+        got = {"/v1/models": look(f"http://127.0.0.1:{port}/v1/models", timeout)}
+        status, data = got["/v1/models"]
+        keyed = status in (401, 403)
+        if not keyed and not (status == 200 and isinstance(data, dict) and isinstance(data.get("data"), list)):
+            return None                          # not an OpenAI-compatible API, or nothing there
+        for sid in ids:
+            s = LOCAL[sid]
+            path, test = s.get("sign") or ("/v1/models", None)
+            if path not in got:
+                got[path] = look(f"http://127.0.0.1:{port}{path}", timeout)
+            if test is None or test(*got[path]):
+                own = s.get("provider", "") if port in s["ports"] else ""   # Ollama's flow knows only its usual port
+                return {"id": sid, "label": s["label"], "port": port, "kind": s["kind"], "hub": s.get("hub", ""),
+                        "provider": own, "docs": s["docs"],
+                        "base": PROVIDERS[own]["base"] if own else f"http://127.0.0.1:{port}/v1",
+                        "models": None if keyed else len(_rows(data)), "key": keyed}
+        return None
+
+    if not by_port:
+        return []
+    with ThreadPoolExecutor(len(by_port)) as pool:
+        return [f for f in pool.map(lambda kv: identify(*kv), by_port.items()) if f]
+
+
+def _endpoint_ports() -> set[int]:
+    """The ports of endpoints on this machine, under any of its names or
+    addresses: an endpoint added by a LAN address is the same server. No DNS:
+    resolving this machine's own name took seconds on some (macOS CI); an
+    address binds only if it is this machine's."""
+    import ipaddress
+    import socket
+    name = socket.gethostname().lower()
+
+    def here(host: str | None) -> bool:
+        if host in ("localhost", "127.0.0.1", "::1", name):
+            return True
+        try:
+            ip = ipaddress.ip_address(host or "")
+        except ValueError:
+            return False                         # another name: not resolved here
+        try:
+            with socket.socket(socket.AF_INET6 if ip.version == 6 else socket.AF_INET) as s:
+                s.bind((str(ip), 0))
+            return True
+        except OSError:
+            return False
+    ports = set()
+    for e in endpoints.load().values():
+        url = urllib.parse.urlsplit(str(e.get("base", "")))
+        try:
+            if here(url.hostname):
+                ports.add(url.port or (443 if url.scheme == "https" else 80))
+        except ValueError:                       # a port that is not a number: not one of these
+            pass
+    return ports
+
+
 class ConnectError(Exception):
     """A sign-in that cannot go on; the text is for the person signing in."""
 
@@ -515,6 +735,11 @@ def check() -> list[str]:
                 problems.append(f"{pid}: {k} must be an object")
         if (p.get("entry") or {}).get("kind") not in set(endpoints.KINDS) - {"hub"}:
             problems.append(f"{pid}: entry.kind is provider, aggregator or local (a hub is added by hand)")
+    for sid, s in LOCAL.items():                 # the id is the name an endpoint is offered under
+        if not endpoints.NAME.match(sid) or not s.get("docs", "").startswith("https://"):
+            problems.append(f"local {sid}: the id must match endpoints.NAME, and docs be https")
+        if s.get("kind") not in endpoints.KINDS or (s["kind"] == "hub") != (s.get("hub") in endpoints.HUBS):
+            problems.append(f"local {sid}: kind is one of endpoints.KINDS, and a hub names one of endpoints.HUBS")
     return problems