@wendellresearch-ai/benchwarmers 0.2.1 → 0.3.0

package/gateway.py

@@ -73,10 +73,40 @@ the seats' answers: its job is to reconcile what the seats said, not to look
 again, so any model can merge and a disagreement about the picture is
 reported rather than settled.
 
+OTHER FORMATS, AND STREAMING
+----------------------------
+The same panel answers Anthropic Messages (/v1/messages), OpenAI Responses
+(/v1/responses) and legacy completions (/v1/completions), Ollama (/api/chat,
+/api/generate, /api/tags, /api/version, /api/show) and Gemini
+(/v1beta/models/<collection>:generateContent and :streamGenerateContent):
+metrics/formats.py turns each request into OpenAI messages and the answer
+back into the format's reply. A panel refuses a request carrying tools,
+tool calls or tool results (400): its text is not a tool call.
+
+ROUTE MODE
+----------
+For agent clients (Claude Code, Codex, opencode, Cline...): `:route` on the
+model, `"bench": {"reply": "route"}`, or a collection's `reply`. No panel:
+one member gets the request as sent, tools, tool_choice, response_format,
+images and all, and its answer is relayed in the request's format, streamed
+or not (route_plan, route_turn, Handler._route). The member is picked by its
+record with some rotation, kept for the whole conversation (fingerprint),
+and replaced by the next if it fails before its first piece. Each turn is
+in the ledger and runs/gateway/routes.jsonl; each conversation is one
+unjudged offload per model that answered it, a verdict optional. GATEWAY.md
+"Route mode" has the decisions and the limits.
+
+Asked to stream, the reply's 200 goes as soon as the panel is seated (a
+refusal before that keeps its status), keep-alives go while the seats
+think, each seat's section goes as that seat finishes (a merged reply when
+the merge is done), and an error after the 200 goes into the stream in the
+format's own shape (Handler._stream).
+
 WHAT IT IS NOT
 --------------
-A single-model proxy. Tools such as Roo Code call Open WebUI directly; this
-exists only for the multi-model work. Seats are called with `llm -m`, exactly
+A proxy to a model of the caller's choosing: route mode picks a collection's
+member by its record, and a tool that wants one model by name calls that
+model's endpoint (or Open WebUI) directly. Seats are called with `llm -m`, exactly
 as offloads always were, or the built-in client (BENCH_CLIENT), so endpoints,
 headers and per-model options all come from the same registry. Full prompts
 and answers go to `runs/gateway/`, gitignored, with the name of the gateway
@@ -89,16 +119,20 @@ CORS). The dashboard answers these same routes on its own port.
 from __future__ import annotations
 
 import datetime as dt
+import hashlib
 import json
 import os
 import pathlib
+import queue
+import random
 import re
 import secrets
 import subprocess
 import sys
+import threading
 import time
 import urllib.parse
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 ROOT = pathlib.Path(__file__).resolve().parent
@@ -114,7 +148,9 @@ sys.path.insert(0, str(ROOT / "suite"))
 import bench_collections  # noqa: E402
 import client  # noqa: E402
 import decide  # noqa: E402
+import formats  # noqa: E402
 import guard  # noqa: E402
+import model_roles  # noqa: E402
 import offload  # noqa: E402
 import route  # noqa: E402
 
@@ -145,6 +181,13 @@ URLS = re.compile(r"https?://[^\s\"'<>\\]+", re.I)
 BASE64 = re.compile(r"[A-Za-z0-9+/]{100,}={0,2}")
 HEADER_NAMES = re.compile(r"[A-Za-z0-9-]{1,64}(\s*,\s*[A-Za-z0-9-]{1,64}){0,31}")
 SEAT_KEYS = ("call", "seat", "alias", "status", "seconds", "answer", "error")
+QUERY_KEY = re.compile(r"([?&](?:key|api_key|access_token)=)[^&\s'\"]*", re.I)
+# A streaming reply: how long a request may take to be seated before its 200
+# goes anyway (a refusal before then keeps its status), and how often a
+# keep-alive goes while the seats think. Idle timeouts in clients and proxies
+# start around 30 seconds.
+STREAM_GRACE = 1.0
+KEEPALIVE = 10.0
 # A merge that reads as consensus hides the dissent the bench runs on, so the
 # merger is told to keep who said what.
 MERGE_SYSTEM = ("Several models answered the same request. Write one answer to it from theirs: say where"
@@ -370,7 +413,18 @@ def estimate_tokens(text: str) -> int:
     size, which errs high too: OpenAI's, Anthropic's and Gemini's models spend
     up to about 1,600 on one at their usual detail. Qwen and Llama vision
     models can spend more on a very large image; a learned error corrects that."""
-    return int(len(text) / 4 * 1.1)
+    return formats.estimate(text)
+
+
+def prompt_tokens(messages: list[dict]) -> int:
+    """The estimate for a whole request, images counted: what a reply's usage
+    and a count_tokens route say."""
+    system, prompt = flatten(messages)
+    try:
+        pictures = len(images(messages))
+    except ValueError:                                 # refused by run_collection, which says why
+        pictures = 0
+    return estimate_tokens(f"{system}\n\n{prompt}".strip()) + IMAGE_TOKENS * pictures
 
 
 def too_small(needed: int, aliases) -> dict[str, int]:
@@ -440,11 +494,25 @@ def merge(collection: dict, members: list[str], shape: str, whole: str, seats: l
     return seat, ""
 
 
+def section(seat: dict) -> str:
+    """One seat's part of an every-answer reply: its heading, call id, status and answer."""
+    body = seat["answer"] or (f"_error: {seat['error'][:300]}_" if seat["error"] else
+                              "_empty reply — usually a token budget, not incapacity_")
+    return f"## {seat['seat']} — {seat['alias']}\n`{seat['call']}` · {seat['status']} · {seat['seconds']}s\n\n{body}\n"
+
+
 def run_collection(name: str, messages: list[dict], nominated: list[str] | None = None,
-                   reply: str = "", who: str = "local") -> tuple[str, dict]:
+                   reply: str = "", who: str = "local", on_part=None) -> tuple[str, dict]:
     """The panel's reply as markdown, and the same seats as data for the
     response's `bench` field. `reply`: the request's mode, "" for the
-    collection's own. `who`: the gateway key that asked, for the call log."""
+    collection's own. `who`: the gateway key that asked, for the call log.
+
+    `on_part`, for a streaming caller: called with each piece of an
+    every-answer reply as it is ready, the heading once the panel is seated,
+    each seat's section as that seat finishes, then the closing lines. The
+    pieces add up to the content, with the seats in the order they finished
+    rather than seat order. Not called for a merged reply: the merge is the
+    reply, and it comes at the end (or every answer, if it fails)."""
     pictures = images(messages)
     system, prompt = flatten(messages)
     if not prompt.strip():
@@ -477,6 +545,13 @@ def run_collection(name: str, messages: list[dict], nominated: list[str] | None
     # answer anyway, about a picture it never got. Skipped, and named.
     skipped += [{"seat": seat, "alias": a, "why": why} for seat, group in (("member", members), ("with", nominated))
                 for a in group if pictures and (why := cannot_see(a))]
+    # A VISION SPECIALIST SITS ONLY FOR PICTURES: its role says it reads
+    # images, not that it answers text, so on a text request its seat goes to
+    # the next member or an explorer. Named in `with`, it sits anyway.
+    if not pictures:
+        roles = model_roles.load()["models"]
+        skipped += [{"seat": "member", "alias": a, "why": "a vision specialist: sits only when the request"
+                     " carries images"} for a in members if roles.get(a) == "vision"]
     unseen = {k["alias"] for k in skipped}
     sitting = [a for a in members if a not in unseen]
     nominated = [a for a in nominated if a not in unseen]
@@ -522,27 +597,39 @@ def run_collection(name: str, messages: list[dict], nominated: list[str] | None
                           + "".join(f"; {k['alias']} not asked: {k['why']}" for k in skipped))
     panel_id = f"p{dt.datetime.now():%Y%m%d-%H%M%S}-{secrets.token_hex(2)}"
     model = bench_collections.PREFIX + name
-    with ThreadPoolExecutor(max_workers=len(seats)) as pool:
-        results = list(pool.map(lambda s: ask_seat(s[2], system, prompt, messages if pictures else None), seats))
-
     task = " ".join(prompt.split())[:90] + f" ({model}, {shape})"
-    LOG_DIR.mkdir(parents=True, exist_ok=True)
-    record = {"panel": panel_id, "at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
-              "model": model, "shape": shape, "shape_by": how, "key": who, "system": system,
-              "prompt": prompt, "seats": [], **({"skipped": skipped} if skipped else {})}
     out = [f"Bench {model} {panel_id} · {len(sitting)} members + {len(seats) - len(sitting)}"
            f" explorers · explorer shape {shape} ({how})\n"]
     if skipped:
         out.append("Not asked: " + "; ".join(f"{k['alias']} ({k['why']})" for k in skipped) + "\n")
-    for (key, label, alias), r in zip(seats, results):
-        call = f"{panel_id}.{key}"
-        record["seats"].append(record_seat(task, call, label, alias, r))
-        body = r["answer"] or (f"_error: {r['error'][:300]}_" if r["error"] else
-                               "_empty reply — usually a token budget, not incapacity_")
-        out.append(f"## {label} — {alias}\n`{call}` · {r['status']} · {r['seconds']}s\n\n{body}\n")
-    out.append("---\nRecord a verdict per seat once checked against the code:\n"
-               "`POST /bench/verdict` on this same server (see `bench.verdict`), or"
-               " `python metrics/offload.py verdict <call id> --useful|--not-useful --note \"...\"`")
+    emit = None if merged else on_part
+    if emit:
+        emit("\n".join(out))
+    done: dict[int, dict] = {}
+    with ThreadPoolExecutor(max_workers=len(seats)) as pool:
+        asked = {pool.submit(ask_seat, alias, system, prompt, messages if pictures else None): i
+                 for i, (_key, _label, alias) in enumerate(seats)}
+        # Each seat is logged the moment it answers, so explorer rotation
+        # counts it even if a slower seat (or the caller) never finishes.
+        for future in as_completed(asked):
+            i = asked[future]
+            key, label, alias = seats[i]
+            done[i] = record_seat(task, f"{panel_id}.{key}", label, alias, future.result())
+            if emit:
+                emit("\n" + section(done[i]))
+
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    record = {"panel": panel_id, "at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
+              "model": model, "shape": shape, "shape_by": how, "key": who, "system": system,
+              "prompt": prompt, "seats": [done[i] for i in range(len(seats))],
+              **({"skipped": skipped} if skipped else {})}
+    out += [section(s) for s in record["seats"]]
+    footer = ("---\nRecord a verdict per seat once checked against the code:\n"
+              "`POST /bench/verdict` on this same server (see `bench.verdict`), or"
+              " `python metrics/offload.py verdict <call id> --useful|--not-useful --note \"...\"`")
+    out.append(footer)
+    if emit:
+        emit("\n" + footer)
     content = "\n".join(out)
     # Beyond the OpenAI fields, so a caller reads seats without parsing the
     # markdown. Clients that do not know it ignore it.
@@ -577,6 +664,297 @@ def models() -> list[str]:
     return [bench_collections.PREFIX + n for n in bench_collections.load()]
 
 
+def listed(who: str) -> list[tuple[str, dict]]:
+    """(model name, what it is for) for every collection key `who` may call,
+    each three times: as set, and in each other reply mode, so a client that
+    offers only listed models (an agent's model picker above all, for
+    `:route`) can still pick any. What GET /v1/models, /api/tags and
+    /v1beta/models list, each in its own shape."""
+    s, rows = bench_collections.settings(), []
+    for n, c in bench_collections.load().items():
+        if not guard.may_call(who, n):                 # a key limited to some collections sees those
+            continue
+        d = bench_collections.described(n, c, s)
+        rows.append((bench_collections.PREFIX + n, d))
+        rows += [(f"{bench_collections.PREFIX}{n}:{other}", {**d, "reply": other})
+                 for other in bench_collections.REPLIES if other != d["reply"]]
+    return rows
+
+
+def bench_headers(bench: dict) -> dict:
+    """The panel's call ids as headers, for a format with no room for the
+    bench field in its body (Gemini): X-Bench-Panel, and X-Bench-Seats as
+    JSON, without the answers (they are in the reply)."""
+    seats = bench.get("seats", []) + ([bench["merger"]] if bench.get("merger") else [])
+    return {"X-Bench-Panel": str(bench.get("panel", "")),
+            "X-Bench-Seats": json.dumps([{k: s.get(k) for k in ("call", "seat", "alias", "status")} for s in seats])}
+
+
+def refusal(exc: Exception) -> tuple[int, str]:
+    """The status and message for a panel that could not run."""
+    if isinstance(exc, TooLarge):
+        return 413, str(exc)
+    if isinstance(exc, ValueError):
+        return 400, str(exc)
+    if isinstance(exc, LookupError):
+        return 503, str(exc)
+    return 500, f"the panel failed ({type(exc).__name__}: {exc})"
+
+
+# ---- route mode: one member, tools and all (GATEWAY.md "Route mode") --------
+
+# A whole agent turn; a file written as one tool call can stream for minutes.
+ROUTE_TIMEOUT = 600
+# How long a member may take to its first piece (text, reasoning or a tool
+# call) before the next is tried. Reasoning streams count, so only a model
+# that sends nothing at all waits this long.
+FIRST_TOKEN = 90
+ROUTE_TRIES = 3                # members tried for one turn before it fails
+# A streamed turn's 200 waits this long for the first piece, so a turn that
+# fails everywhere quickly keeps its status (a client retries a 429 or 503 it
+# can see, not an error inside a 200); after it, keep-alives.
+ROUTE_GRACE = 15.0
+STICKY_TTL = 3600              # seconds idle before a conversation is picked afresh
+ROTATE = 0.25                  # share of new conversations given to the member with the oldest record
+_STICKY: dict[str, dict] = {}  # ponytail: in memory, so a restart re-picks; a file if that ever matters
+_STICKY_LOCK = threading.Lock()
+
+
+def cannot_call_tools(alias: str) -> str:
+    """Why a request with tools cannot be routed to `alias`, or "" when it
+    can. As cannot_see, the catalog decides (the model's listed parameters,
+    your Tool calling override in its model popup, a probe that saw a call),
+    and a model it knows nothing about is not routed to: one that cannot call
+    tools answers in prose, and the agent waits for a call that never comes."""
+    import catalog
+    rec = catalog.info(alias)
+    if rec.get("tools"):
+        return ""
+    if rec.get("tools") is None:
+        return "the catalog does not say whether it calls tools; tick Tool calling in its model popup if it does"
+    return f"does not call tools (says {rec['sources'].get('tools', 'the catalog')})"
+
+
+def route_tokens(ask: dict) -> int:
+    """The size check's estimate for a routed request: the messages as a
+    panel counts them, plus the tool calls in them and the tool definitions,
+    which an agent's requests are full of."""
+    extra = [[m["tool_calls"] for m in ask["messages"] if m.get("tool_calls")], ask["upstream"].get("tools") or []]
+    return prompt_tokens(ask["messages"]) + (estimate_tokens(json.dumps(extra)) if any(extra) else 0)
+
+
+def fingerprint(name: str, who: str, messages: list[dict]) -> str:
+    """What keeps an agent's conversation on one model from turn to turn:
+    the collection, the gateway key, the system text and the first user
+    message, which every turn sends again unchanged. A side task with its
+    own system prompt (titling the chat, a subagent) is its own conversation."""
+    system = "\n".join(formats.text_of(m.get("content")) for m in messages if m.get("role") in ("system", "developer"))
+    first = next((formats.text_of(m.get("content")) for m in messages if m.get("role") == "user"), "")
+    return hashlib.sha256("\0".join((name, who, system, first)).encode()).hexdigest()[:24]
+
+
+def route_order(aliases: list[str]) -> list[str]:
+    """Best record first: useful verdicts counted, as the merger is chosen
+    (one lucky verdict must not outrank a long good record), then verified
+    suite findings, then a connection already paid for. Members cooling down
+    after a rate limit or failing their probes go last. Now and then
+    (ROTATE) the member whose record is oldest goes first instead, so one
+    that stops being picked still gets turns, and with them a record."""
+    try:
+        used = offload.summary()
+    except Exception:                                  # noqa: BLE001
+        used = {}
+    try:
+        found = route.suite_recall()
+    except Exception:                                  # noqa: BLE001
+        found = {}
+    try:
+        paid = route.preferred_connections()
+        tags = {a: route.full_tag_of(a) for a in aliases}
+    except Exception:                                  # noqa: BLE001
+        paid, tags = set(), {}
+    ranked = sorted(aliases, reverse=True, key=lambda a: (used.get(a, {}).get("useful", 0),
+                                                          found.get(a, (0, 0, 0))[0], tags.get(a) in paid))
+    cooling = retiring()
+    fresh = [a for a in ranked if a not in cooling]
+    if len(fresh) > 1 and random.random() < ROTATE:
+        stalest = min(fresh, key=lambda a: used.get(a, {}).get("last_date") or "")
+        fresh.remove(stalest)
+        fresh.insert(0, stalest)
+    return fresh + [a for a in ranked if a in cooling]
+
+
+def route_plan(name: str, ask: dict, who: str) -> dict:
+    """The members that may take this turn, in the order to try them, and
+    the conversation it belongs to. A conversation under way keeps its model
+    first; a new one is ranked by route_order. Excluded, and named with why,
+    as a panel excludes them: retired and ad hoc members, vision specialists
+    on a request without images, members reached
+    through another API than chat completions, and those that cannot call
+    tools (when the request has tools), see (when it has images) or read it.
+    Raises as run_collection does: TooLarge, ValueError, LookupError."""
+    import endpoints
+    from run import ad_hoc_models
+    members = list(dict.fromkeys(m for m in bench_collections.load()[name].get("members") or []
+                                 if isinstance(m, str) and m))
+    if not members:
+        raise LookupError(f"collection {name!r} has no members to route to")
+    pictures, tools = images(ask["messages"]), bool(ask["upstream"].get("tools"))
+    needed = route_tokens(ask) + IMAGE_TOKENS * len(pictures)
+    gone, held, small = retired(), ad_hoc_models(), too_small(needed, members)
+    roles = model_roles.load()["models"]
+    skipped, fit = [], []
+    for a in members:
+        api = endpoints.entry_of(a).get("api") or "chat"
+        why = (retired_why(gone[a]) if a in gone else
+               "ad hoc (expensive or capped), so never routed to automatically" if a in held else
+               "a vision specialist: routed to only when the request carries images"
+               if roles.get(a) == "vision" and not pictures else
+               f"reached through the {api} API; route mode passes requests on to chat completions only" if api != "chat"
+               else (tools and cannot_call_tools(a)) or (pictures and cannot_see(a)) or
+               (f"its context window ({small[a]:,} tokens) is under this request (about {needed:,})" if a in small else ""))
+        (skipped.append({"alias": a, "why": why}) if why else fit.append(a))
+    if not fit:
+        detail = "; ".join(f"{k['alias']} {k['why']}" for k in skipped)
+        if all(k["alias"] in small for k in skipped):
+            raise TooLarge(f"no member of bench/{name} can read this request: {detail}. Trim it, or choose a"
+                           " collection whose members have larger windows (GET /v1/models lists each one's).")
+        raise ValueError(f"no member of bench/{name} can take this request: {detail}")
+    key, now = fingerprint(name, who, ask["messages"]), time.monotonic()
+    with _STICKY_LOCK:
+        for k in [k for k, c in _STICKY.items() if now - c["at"] > STICKY_TTL]:
+            del _STICKY[k]
+        conv = _STICKY.get(key)
+    if conv is None or conv["alias"] not in fit:      # new, or its model cannot take this turn (an image, say)
+        ranked = route_order(fit)
+        first = next((formats.text_of(m.get("content")) for m in ask["messages"] if m.get("role") == "user"), "")
+        fresh = {"conv": f"r{dt.datetime.now():%Y%m%d-%H%M%S}-{secrets.token_hex(2)}", "turns": 0, "models": [],
+                 "task": " ".join(first.split())[:90], "alias": ranked[0], "order": ranked, "at": now}
+        with _STICKY_LOCK:
+            # Changed in place, or the first of two simultaneous new turns
+            # kept: every turn of one conversation shares one record.
+            if conv is None:
+                conv = _STICKY.setdefault(key, fresh)
+            else:
+                conv.update(alias=ranked[0], order=ranked, at=now)
+    order = [conv["alias"]] + [a for a in conv["order"] if a != conv["alias"] and a in fit]
+    return {"conv": conv, "order": order + [a for a in fit if a not in order], "skipped": skipped,
+            "images": bool(pictures)}
+
+
+def _fit(upstream: dict, alias: str) -> dict:
+    """The request's fields for `alias`: an output budget over the model's
+    known maximum is cut to it rather than refused (Claude Code asks every
+    model for 32,000)."""
+    import catalog
+    cap = catalog.info(alias).get("max_output")
+    return {k: min(v, cap) if k in ("max_tokens", "max_completion_tokens") and isinstance(v, int)
+            and isinstance(cap, int) and cap > 0 else v for k, v in upstream.items()}
+
+
+def route_turn(name: str, plan: dict, ask: dict, who: str, emit) -> None:
+    """One agent turn: the plan's members in order until one answers.
+
+    The answering member's chunks go to emit("chunk", chunk), held back
+    until the first piece of its answer (text or a tool call; reasoning
+    before it is held with it), so a member that fails before then (a 429, a
+    5xx, a pacing refusal, nothing in FIRST_TOKEN seconds, an empty reply
+    even after reasoning) is replaced by the next without the caller seeing
+    a thing; keep-alives cover the wait. emit("live",
+    alias) says who it is. Then emit("done", bench), or emit("error",
+    (status, message)) when every member tried failed, or the one under way
+    failed part way, which no other member can take up.
+
+    Every call reaches the usage ledger (purpose "route"). Every turn is a
+    line in runs/gateway/routes.jsonl, without the prompt (an agent resends
+    its whole conversation each turn). A conversation is one unjudged
+    offload per model that answered it, not one per turn (GATEWAY.md says why)."""
+    conv, order, tried = plan["conv"], plan["order"][:ROUTE_TRIES], []
+    for n, alias in enumerate(order):
+        held, live = [], [False]
+        last = n == len(order) - 1
+
+        def on_chunk(chunk, held=held, live=live, turn=formats.Turn(), alias=alias):
+            _, anything = turn.feed(chunk)
+            if live[0]:
+                return emit("chunk", chunk)
+            held.append(chunk)
+            if anything:
+                live[0] = True
+                emit("live", alias)
+                for c in held:
+                    emit("chunk", c)
+        res = client.chat(alias, "", timeout=ROUTE_TIMEOUT, messages=ask["messages"], body=_fit(ask["upstream"], alias),
+                          on_chunk=on_chunk, first_token=FIRST_TOKEN, retries=None if last else 0, purpose="route")
+        error = res["error"] or ("" if not res["timed_out"] else "timed out part way" if live[0]
+                                 else f"nothing in {FIRST_TOKEN}s")
+        if plan["images"]:
+            error = redact(error)                      # an upstream may echo an image back
+        if error:
+            try:
+                import learn
+                learn.record(alias, error, "gateway")  # a rate limit cools the model for the next pick
+            except Exception:                          # noqa: BLE001
+                pass
+        if not error and not live[0] and not last:
+            error = "an empty reply"                   # nothing was sent: another member may do better
+        if error and not live[0]:
+            tried.append({"alias": alias, "error": error[-300:]})
+            continue
+        if not live[0]:                                # the last member's empty reply is its answer
+            emit("live", alias)
+            for c in held:
+                emit("chunk", c)
+        with _STICKY_LOCK:
+            fresh = alias not in conv["models"]
+            if fresh:
+                conv["models"].append(alias)
+            conv.update(alias=alias, at=time.monotonic(), turns=conv["turns"] + 1)
+            call, turn_no = f"{conv['conv']}.{conv['models'].index(alias) + 1}", conv["turns"]
+        status = "error" if error else "ok" if res["text"] or res["tool_calls"] else "empty"
+        model = bench_collections.PREFIX + name
+        if fresh:
+            offload.add(alias, f"{conv['task']} ({model}:route)", None,
+                        f"route conversation; first turn {status} in {res['seconds']}s", call)
+        _log_route(conv, turn_no, call, model, who, alias, status, res, tried, error)
+        if error:
+            return emit("error", (res["status"] if res["status"] >= 400 else 502,
+                                  f"{alias} failed part way through the turn: {error[:300]}"))
+        # SHADOWS GO HERE, when they come: an opt-in that sends this same turn
+        # (ask, minus the caller's stream) to other members in the background,
+        # for evaluation only, after the caller has its answer. Each would be
+        # its own client.chat (purpose "shadow"), recorded as an unjudged
+        # offload on this conversation's call id with a suffix; nothing the
+        # caller sees waits for them or changes.
+        return emit("done", {"collection": model, "reply": "route", "conversation": conv["conv"], "call": call,
+                             "alias": alias, "turn": turn_no, **({"failed_over": tried} if tried else {}),
+                             **({"skipped": plan["skipped"]} if plan["skipped"] else {}),
+                             "verdict": 'optional, once per conversation: POST /bench/verdict {"call", "useful", "note"}'})
+    _log_route(conv, conv["turns"], "", bench_collections.PREFIX + name, who, "", "error", None, tried,
+               "every member failed")
+    # Every member rate limited (or held back by pacing): 429, so the caller
+    # backs off and tries again, as it would for one provider.
+    limited = all("Error code: 429" in t["error"] or "rate limit" in t["error"].lower() for t in tried)
+    emit("error", (429 if tried and limited else 502,
+                   f"no member of bench/{name} answered: " + "; ".join(f"{t['alias']}: {t['error'][:200]}" for t in tried)))
+
+
+def _log_route(conv: dict, turn: int, call: str, model: str, who: str, alias: str, status: str, res: dict | None,
+               tried: list[dict], error: str) -> None:
+    """One turn's line in runs/gateway/routes.jsonl, which the dashboard's
+    Gateway card reads for its route conversations."""
+    import ledger
+    res = res or {}
+    line = {"conversation": conv["conv"], "call": call, "at": dt.datetime.now().astimezone().isoformat(timespec="seconds"),
+            "model": model, "key": who, "alias": alias, "turn": turn, "task": conv["task"], "status": status,
+            "seconds": res.get("seconds"), "first_token_s": res.get("first_token_s"),
+            "tool_calls": res.get("tool_calls", 0), "finish": res.get("finish", ""), **ledger.tokens(res.get("usage") or {}),
+            **({"tried": tried} if tried else {}), **({"error": error[-300:]} if error else {})}
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    with (LOG_DIR / "routes.jsonl").open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(line, ensure_ascii=False) + "\n")
+
+
 class Handler(BaseHTTPRequestHandler):
     """The gateway's routes. serve.py's Handler subclasses this to answer them
     on the dashboard's port too, so the checks use the port actually served."""
@@ -592,8 +970,16 @@ class Handler(BaseHTTPRequestHandler):
         self.end_headers()
         self.wfile.write(body)
 
-    def _error(self, code: int, message: str, headers: dict | None = None) -> None:
-        self._json(code, {"error": {"message": message, "type": "bench_gateway", "code": code}}, headers=headers)
+    def _error(self, code: int, message: str, headers: dict | None = None, fmt: str = "chat") -> None:
+        """A refusal, in the shape the route's clients read (metrics/formats.py)."""
+        self._json(code, formats.INBOUND[fmt].error(code, message), headers=headers)
+
+    def _query_key(self, fmt: str) -> str | None:
+        """A Gemini route's ?key=, as Google's REST examples send it; no other
+        route takes a key in its URL, where proxies and histories keep it."""
+        if fmt != "gemini":
+            return None
+        return urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get("key", [None])[0]
 
     def end_headers(self):
         # CORS for a browser app on an allowed origin, on the gateway's routes
@@ -603,6 +989,8 @@ class Handler(BaseHTTPRequestHandler):
         if headers is not None and guard.gateway_route((getattr(self, "path", "") or "").split("?", 1)[0]) \
                 and (origin := guard.cors_origin(headers, self.server.server_port)):
             self.send_header("Access-Control-Allow-Origin", origin)
+            self.send_header("Access-Control-Expose-Headers",
+                             "X-Bench-Panel, X-Bench-Seats, X-Bench-Conversation, X-Bench-Model")
             self.send_header("Vary", "Origin")
         super().end_headers()
 
@@ -630,45 +1018,47 @@ class Handler(BaseHTTPRequestHandler):
         self.end_headers()
 
     def do_GET(self):                                  # noqa: N802
+        path = self.path.split("?", 1)[0].rstrip("/")
+        fmt = formats.format_of(path)
         try:
             guard.check(self.headers, self.server.server_port, post=False, max_body=0)
-            self.who = guard.gateway_caller(self.headers)
+            self.who = guard.gateway_caller(self.headers, self._query_key(fmt))
         except guard.Refused as r:
-            return self._error(r.status, str(r), r.headers)
-        if self.path.split("?", 1)[0].rstrip("/") in ("/v1/models", "/models"):
+            return self._error(r.status, str(r), r.headers, fmt)
+        if path in ("/v1/models", "/models"):
             # Beyond the OpenAI fields: what each collection is for, so an
             # orchestrator can choose one by task. Clients ignore extra keys.
-            # Each collection twice: as set, and in the other reply mode, so a
-            # client that offers only listed models can still pick either.
-            s, data = bench_collections.settings(), bench_collections.load()
-            rows = []
-            for n, c in data.items():
-                if not guard.may_call(self.who, n):    # a key limited to some collections sees those
-                    continue
-                d = bench_collections.described(n, c, s)
-                other = "all" if d["reply"] == "merged" else "merged"
-                rows += [{"id": bench_collections.PREFIX + n, "object": "model", "owned_by": "benchwarmers", **d},
-                         {"id": f"{bench_collections.PREFIX}{n}:{other}", "object": "model",
-                          "owned_by": "benchwarmers", **d, "reply": other}]
-            self._json(200, {"object": "list", "data": rows})
-        else:
-            self._error(404, "GET /v1/models is the only GET here")
+            return self._json(200, {"object": "list", "data": [
+                {"id": n, "object": "model", "owned_by": "benchwarmers", **d} for n, d in listed(self.who)]})
+        if path == "/api/tags":
+            return self._json(200, formats.ollama_tags(listed(self.who)))
+        if path == "/api/version":
+            return self._json(200, {"version": formats.OLLAMA_VERSION})
+        if path == "/v1beta/models":
+            return self._json(200, formats.gemini_models(listed(self.who)))
+        if path.startswith("/v1beta/models/"):
+            name = urllib.parse.unquote(path[len("/v1beta/models/"):])
+            row = dict(listed(self.who)).get(name)
+            return self._json(200, formats.gemini_model(name, row)) if row else \
+                self._error(404, f"{name!r} is not a collection here; GET /v1beta/models lists them", fmt=fmt)
+        self._error(404, "GET /v1/models, /api/tags, /api/version or /v1beta/models", fmt=fmt)
 
     def do_POST(self):                                 # noqa: N802
         # Without this any web page could spend model calls, seat ad hoc
         # models through `with`, or write verdicts (metrics/guard.py).
+        path = self.path.split("?", 1)[0].rstrip("/")
+        fmt = formats.format_of(path)
         try:
             length = guard.check(self.headers, self.server.server_port, post=True, max_body=MAX_BODY, cors=True)
-            self.who = guard.gateway_caller(self.headers)
+            self.who = guard.gateway_caller(self.headers, self._query_key(fmt))
         except guard.Refused as r:
-            return self._error(r.status, str(r) + (TOO_BIG if r.status == 413 else ""), r.headers)
+            return self._error(r.status, str(r) + (TOO_BIG if r.status == 413 else ""), r.headers, fmt)
         try:
             body = json.loads(self.rfile.read(length) or b"{}")
         except ValueError:
-            return self._error(400, "body is not JSON")
+            return self._error(400, "body is not JSON", fmt=fmt)
         if not isinstance(body, dict):
-            return self._error(400, "body must be a JSON object")
-        path = self.path.split("?", 1)[0].rstrip("/")
+            return self._error(400, "body must be a JSON object", fmt=fmt)
         if path == "/bench/verdict":
             if not isinstance(body.get("useful"), bool):
                 return self._error(400, '"useful" must be true or false')
@@ -681,66 +1071,202 @@ class Handler(BaseHTTPRequestHandler):
             except KeyError as exc:
                 return self._error(404, exc.args[0])
             return self._json(200, {"ok": True, "recorded": e})
-        if path not in ("/v1/chat/completions", "/chat/completions"):
-            return self._error(404, "POST /v1/chat/completions or /bench/verdict")
-        model = body.get("model") or ""
-        if not isinstance(model, str):
-            return self._error(400, '"model" must be a collection name such as bench/core')
-        model, suffix = bench_collections.split_reply(model)
+        # Each format's request becomes one Ask (metrics/formats.py); a part
+        # that would be lost on the way, tools above all, is refused with why.
+        gemini = formats.gemini_route(path)
+        try:
+            if gemini and gemini[1] == "countTokens":
+                ask = formats.gemini_ask(body.get("generateContentRequest") or body, gemini[0])
+                return self._json(200, {"totalTokens": prompt_tokens(ask["messages"])})
+            if gemini:
+                if gemini[1] not in ("generateContent", "streamGenerateContent"):
+                    return self._error(404, f"POST models/<collection>:generateContent or :streamGenerateContent,"
+                                            f" not :{gemini[1]}", fmt=fmt)
+                ask = formats.gemini_ask(body, gemini[0], gemini[1] == "streamGenerateContent")
+            elif path == "/v1/messages/count_tokens":
+                # An estimate, as the size check makes, without calling a model.
+                return self._json(200, {"input_tokens": prompt_tokens(formats.anthropic_ask(body)["messages"])})
+            elif path == "/api/show":
+                return self._show(body)
+            elif path in formats.POST_ROUTES:
+                ask = formats.INBOUND[fmt].ask(body)
+            else:
+                return self._error(404, "POST /v1/chat/completions, /v1/messages, /v1/responses, /v1/completions,"
+                                        " /api/chat, /api/generate, /v1beta/models/<collection>:generateContent"
+                                        " or /bench/verdict", fmt=fmt)
+        except formats.Unsupported as exc:
+            return self._error(400, str(exc), fmt=fmt)
+        alt = urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get("alt", [""])[0]
+        self._panel(fmt, ask, sse=alt == "sse")
+
+    def _show(self, body: dict) -> None:
+        """POST /api/show: Ollama's model card, for a collection the key may call."""
+        name = formats.ollama_model(body.get("model") or body.get("name") or "")
+        rows = dict(listed(self.who))
+        row = rows.get(name) or rows.get(bench_collections.PREFIX + str(name))
+        if not row:
+            return self._error(404, f"model {name!r} not found; GET /api/tags lists the collections",
+                               fmt="ollama_chat")
+        self._json(200, formats.ollama_show(name, row))
+
+    def _panel(self, fmt: str, ask: dict, sse: bool = True) -> None:
+        """One panel, for a request in any format: the collection it names,
+        the key's scope and cap, then the panel, answered in that format."""
+        model, suffix = bench_collections.split_reply(ask["model"])
         name = bench_collections.resolve(model)
         if name is None:
-            return self._error(400 if not model else 404, bench_collections.choose_message(model))
+            return self._error(400 if not model else 404, bench_collections.choose_message(model), fmt=fmt)
         model = bench_collections.PREFIX + name
-        messages, nominated, opts = body.get("messages"), body.get("with") or [], body.get("bench") or {}
-        if not (isinstance(messages, list) and all(isinstance(m, dict) for m in messages)):
-            return self._error(400, '"messages" must be a list of message objects')
-        if not (isinstance(nominated, list) and all(isinstance(a, str) for a in nominated)):
-            return self._error(400, '"with" must be a list of model aliases')
-        if not isinstance(opts, dict) or opts.get("reply", "all") not in bench_collections.REPLIES:
-            return self._error(400, '"bench" must be an object whose "reply" is "all" or "merged"')
         if not guard.may_call(self.who, name):
             allowed = ", ".join(bench_collections.PREFIX + c for c in guard.scope(self.who) or []) or "none"
-            return self._error(403, f"the gateway key {self.who} may not call {model}; it may call {allowed}")
+            return self._error(403, f"the gateway key {self.who} may not call {model}; it may call {allowed}",
+                               fmt=fmt)
         try:
             guard.spend(self.who)                      # counted once it names a collection it may call
         except guard.Refused as r:
-            return self._error(r.status, str(r), r.headers)
+            return self._error(r.status, str(r), r.headers, fmt)
+        reply = ask["reply"] or suffix                 # the body is the most specific, then the suffix, then the collection
+        # Chat completions have always answered with the collection's name;
+        # the other formats echo the name as asked, which their clients match.
+        shown = model if fmt == "chat" else ask["model"]
+        if (reply or bench_collections.load()[name].get("reply")) == "route":
+            return self._route(fmt, name, ask, shown, sse)
+        if ask["agentic"]:
+            return self._error(400, formats.TOOLS, fmt=fmt)
+        if ask["stream"]:
+            out = formats.INBOUND[fmt].stream(shown, prompt_tokens(ask["messages"]), sse=sse)
+            return self._stream(fmt, name, ask, reply, out)
+        started = time.monotonic()
         try:
-            # The body is the most specific, then the suffix, then the collection.
-            content, bench = run_collection(name, messages, nominated, opts.get("reply") or suffix, self.who)
-        except TooLarge as exc:
-            return self._error(413, str(exc))
-        except ValueError as exc:
-            return self._error(400, str(exc))
-        except LookupError as exc:
-            return self._error(503, str(exc))
-        reply_id, created = f"chatcmpl-{secrets.token_hex(8)}", int(time.time())
-        if not body.get("stream"):
-            return self._json(200, {
-                "id": reply_id, "object": "chat.completion", "created": created, "model": model,
-                "choices": [{"index": 0, "finish_reason": "stop",
-                             "message": {"role": "assistant", "content": content}}], "bench": bench})
-        # A panel finishes all at once, so a streaming client gets it as one
-        # chunk, merged or not. ponytail: nothing is sent until the panel and
-        # the merge are done, so a client or proxy with a shorter idle timeout
-        # drops it. Keep-alives mean sending the 200 before the panel runs, so
-        # the 413/503 refusals move into the stream; streaming the merge as it
-        # is written needs a streaming client.chat (RELEASE-PLAN acceptance 11).
-        self.send_response(200)
-        self.send_header("Content-Type", "text/event-stream")
-        self.end_headers()
-        for delta, finish in (({"role": "assistant", "content": content}, None), ({}, "stop")):
-            chunk = {"id": reply_id, "object": "chat.completion.chunk", "created": created,
-                     "model": model, "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
-                     **({"bench": bench} if finish is None else {})}
-            self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
-        self.wfile.write(b"data: [DONE]\n\n")
+            content, bench = run_collection(name, ask["messages"], ask["with"], reply, self.who)
+        except (ValueError, LookupError) as exc:       # TooLarge is a ValueError
+            return self._error(*refusal(exc), fmt=fmt)
+        usage = (prompt_tokens(ask["messages"]), estimate_tokens(content))
+        self._json(200, formats.INBOUND[fmt].reply(content, shown, bench, usage, time.monotonic() - started),
+                   bench_headers(bench) if fmt == "gemini" else None)
+
+    def _stream(self, fmt: str, name: str, ask: dict, reply: str, out: formats.Stream) -> None:
+        """The panel as it comes, in the format's stream shape: every answer
+        seat by seat as each finishes, or a merged reply when the merge is
+        done. The panel runs in its own thread and this one writes, so a
+        keep-alive goes out every KEEPALIVE seconds while the seats think. A
+        refusal raised before anything is ready (the size check, an image no
+        seat can see: milliseconds) still gets its status; the 200 goes once
+        the panel is seated or after STREAM_GRACE, whichever is first, and
+        from then on an error goes into the stream in the format's shape."""
+        events: queue.Queue = queue.Queue()
+
+        def work():
+            try:
+                events.put(("done", run_collection(name, ask["messages"], ask["with"], reply, self.who,
+                                                   on_part=lambda text: events.put(("part", text)))))
+            except Exception as exc:                   # noqa: BLE001 -- answered below, as a status or in the stream
+                events.put(("error", exc))
+        threading.Thread(target=work, daemon=True).start()
+        started = streamed = False
+        while True:
+            try:
+                kind, data = events.get(timeout=KEEPALIVE if started else STREAM_GRACE)
+            except queue.Empty:
+                kind, data = "idle", None
+            if kind == "error" and not started:
+                return self._error(*refusal(data), fmt=fmt)
+            try:
+                if not started:
+                    self.send_response(200)
+                    self.send_header("Content-Type", out.content_type)
+                    self.send_header("Cache-Control", "no-cache")
+                    self.end_headers()
+                    self.wfile.write(out.start())
+                    started = True
+                if kind == "idle":
+                    self.wfile.write(out.keepalive())
+                elif kind == "part":
+                    self.wfile.write(out.text(data))
+                    streamed = True
+                elif kind == "done":
+                    content, bench = data
+                    self.wfile.write((b"" if streamed else out.text(content)) + out.end(bench))
+                    return
+                else:
+                    self.wfile.write(out.error(*refusal(data)))
+                    return
+            except OSError:
+                return                                 # the caller left; the panel still finishes, every seat recorded
+
+    def _route(self, fmt: str, name: str, ask: dict, shown: str, sse: bool = True) -> None:
+        """Route mode: one member's answer, in the request's format. The
+        turn runs in its own thread (route_turn) and this one writes: a
+        whole reply once it is done, or a stream whose 200 waits for the
+        member's first piece, or ROUTE_GRACE, whichever is first, then
+        relays it, with a keep-alive whenever nothing went for KEEPALIVE."""
+        try:
+            plan = route_plan(name, ask, self.who)
+        except (ValueError, LookupError) as exc:       # TooLarge is a ValueError
+            return self._error(*refusal(exc), fmt=fmt)
+        events: queue.Queue = queue.Queue()
+
+        def work():
+            try:
+                route_turn(name, plan, ask, self.who, lambda kind, data=None: events.put((kind, data)))
+            except Exception as exc:                   # noqa: BLE001 -- answered below, as a status or in the stream
+                events.put(("error", (500, f"route mode failed ({type(exc).__name__}: {exc})")))
+        threading.Thread(target=work, daemon=True).start()
+        tokens, began = route_tokens(ask), time.monotonic()
+        headers = {"X-Bench-Conversation": plan["conv"]["conv"]}
+        if not ask["stream"]:
+            turn = formats.Turn()
+            while True:
+                kind, data = events.get()
+                if kind == "chunk":
+                    turn.feed(data)
+                elif kind == "done":
+                    return self._json(200, formats.route_reply(fmt, turn, shown, data, tokens, time.monotonic() - began,
+                                                               ask.get("custom")), {**headers, "X-Bench-Model": data["alias"]})
+                elif kind == "error":
+                    return self._error(*data, fmt=fmt)
+        out = formats.ROUTE[fmt](shown, tokens, sse=sse, custom=ask.get("custom"))
+        started, wrote = False, time.monotonic()
+        while True:
+            try:
+                kind, data = events.get(timeout=KEEPALIVE if started else max(0.0, ROUTE_GRACE - (time.monotonic() - began)))
+            except queue.Empty:
+                kind, data = "idle", None
+            if kind == "error" and not started:
+                return self._error(*data, fmt=fmt)
+            if kind == "live":
+                headers["X-Bench-Model"] = data
+                continue
+            try:
+                if not started:
+                    self.send_response(200)
+                    for k, v in {"Content-Type": out.content_type, "Cache-Control": "no-cache", **headers}.items():
+                        self.send_header(k, v)
+                    self.end_headers()
+                    self.wfile.write(out.start())
+                    started = True
+                piece = (out.chunk(data) if kind == "chunk" else out.end(data) if kind == "done" else
+                         out.error(*data) if kind == "error" else b"")
+                if not piece and time.monotonic() - wrote >= KEEPALIVE:
+                    piece = out.keepalive()
+                if piece:
+                    self.wfile.write(piece)
+                    wrote = time.monotonic()
+                if kind in ("done", "error"):
+                    return
+            except OSError:
+                # The caller left. ponytail: the member's answer still runs to
+                # its end and is recorded; stopping it needs a per-call STOP.
+                return
 
     def log_request(self, code="-", size="-"):         # route, status, who; never a query or a key
         guard.access(self, code)
 
     def log_message(self, fmt, *args):                 # quieter than the default
-        sys.stderr.write(f"{self.log_date_time_string()} {fmt % args}\n")
+        # http.server's own errors quote the request line, ?key= and all
+        # ("Bad request syntax ('GET /v1beta/models?key=...')").
+        line = QUERY_KEY.sub(r"\1...", fmt % args)
+        sys.stderr.write(f"{self.log_date_time_string()} {line}\n")
 
 
 def main() -> None: