@wendellresearch-ai/benchwarmers 0.2.1 → 0.3.0

package/metrics/cost.py

@@ -1,10 +1,13 @@
-"""What each model actually costs us, from the `llm` CLI's own log.
+"""What each model actually costs us, from the bench's usage ledger and the `llm` CLI's log.
 
     python metrics/cost.py
 
-**No new instrumentation.** The `llm` CLI has been logging every call all
-along — 382 turns with token counts were already on disk before this file
-existed. This reads them.
+TWO SOURCES, NO CALL IN BOTH. The built-in client writes a line for every
+call to the usage ledger (metrics/ledger.py, RELEASE-PLAN Phase 2b); calls
+made through the `llm` backend, and every call before the switch, are in
+llm's own logs.db, which it has kept all along (382 turns with token counts
+were on disk before this file existed). The built-in client never writes
+logs.db and `llm` never writes the ledger, so the two are added up.
 
 WHY REASONING TOKENS MATTER SEPARATELY
 --------------------------------------
@@ -77,33 +80,49 @@ def benchmark_calls() -> dict[str, int]:
         return {}
 
 
-def collect() -> dict:
-    """Per-model totals, with old aliases folded into their current name."""
+def _llm_turns():
+    """(model, input, output, reasoning or None, duration_ms) per turn in llm's log."""
     if not LOGS.exists():
-        return {}
-    rename = alias_history()
+        return
     db = sqlite3.connect(f"file:{LOGS}?mode=ro", uri=True)
     db.row_factory = sqlite3.Row
-
-    agg: dict[str, dict] = collections.defaultdict(
-        lambda: {"calls": 0, "input": 0, "output": 0, "reasoning": 0,
-                 "duration_ms": 0, "with_reasoning_data": 0})
     for row in db.execute(
             "SELECT model, input_tokens, output_tokens, token_details,"
             " duration_ms FROM turns"):
-        # Turns predating the versioned rename carry the old alias.
-        model = rename.get(row["model"], row["model"])
-        a = agg[model]
-        a["calls"] += 1
-        a["input"] += row["input_tokens"] or 0
-        a["output"] += row["output_tokens"] or 0
-        a["duration_ms"] += row["duration_ms"] or 0
         try:
             details = json.loads(row["token_details"] or "{}") or {}
         except ValueError:
             details = {}
         reasoning = ((details.get("completion_tokens_details") or {})
                      .get("reasoning_tokens"))
+        yield (row["model"], row["input_tokens"], row["output_tokens"], reasoning,
+               row["duration_ms"])
+
+
+def _ledger_calls():
+    """The same, per answered call in the usage ledger. Refused and failed
+    calls spent nothing, as llm's log (which keeps only answers) counts."""
+    import ledger
+    for row in ledger.rows():
+        if row.get("status") in ("ok", "empty"):
+            yield (row.get("alias") or "?", row.get("tokens_in"), row.get("tokens_out"),
+                   row.get("reasoning"), round((row.get("seconds") or 0) * 1000))
+
+
+def collect() -> dict:
+    """Per-model totals, with old aliases folded into their current name."""
+    rename = alias_history()
+    agg: dict[str, dict] = collections.defaultdict(
+        lambda: {"calls": 0, "input": 0, "output": 0, "reasoning": 0,
+                 "duration_ms": 0, "with_reasoning_data": 0})
+    for name, tokens_in, tokens_out, reasoning, ms in (*_llm_turns(), *_ledger_calls()):
+        # Turns predating the versioned rename carry the old alias.
+        model = rename.get(name, name)
+        a = agg[model]
+        a["calls"] += 1
+        a["input"] += tokens_in or 0
+        a["output"] += tokens_out or 0
+        a["duration_ms"] += ms or 0
         if reasoning is not None:
             a["reasoning"] += reasoning
             a["with_reasoning_data"] += 1

package/metrics/endpoints.py

@@ -88,6 +88,18 @@ HUBS = {"openwebui": "Open WebUI", "litellm": "LiteLLM", "cliproxy": "CLI Proxy
 # endpoint(), plan(connection, model) and add(connection, model), and the two
 # whose connections only the hub knows also upstreams(), for Discover.
 ADAPTERS = {"openwebui": "owui", "litellm": "litellm_hub", "cliproxy": "cliproxy"}
+# `api`: what an endpoint speaks, chat completions unless it says otherwise.
+# The built-in client builds the others' native requests (metrics/formats.py);
+# `responses` is set by Connect for ChatGPT's plan, the native two on the Add
+# form, for an API key whose provider has no chat completions route.
+APIS = {"chat": "OpenAI chat completions", "responses": "OpenAI Responses",
+        "anthropic": "Anthropic Messages (native)", "gemini": "Gemini generateContent (native)"}
+
+
+def auth_headers(api: str, secret: str) -> dict:
+    """How a key is sent: Bearer, except to Anthropic's and Gemini's native APIs."""
+    return {"anthropic": {"x-api-key": secret}, "gemini": {"x-goog-api-key": secret}}.get(
+        api, {"Authorization": f"Bearer {secret}"})
 
 
 def adapter(which: str):
@@ -271,13 +283,24 @@ def list_models(entry: dict, secret: str = "", timeout: int = 30) -> dict[str, d
     before saving it). `models_url` when the list lives elsewhere (GitHub
     Models: its catalog, a bare JSON list); entries marked `visibility` other
     than "list" are left out (ChatGPT's plan lists ones it will not serve)."""
+    api = entry.get("api") or "chat"
     headers = dict(entry.get("headers") or {})
     if secret:
-        headers["Authorization"] = f"Bearer {secret}"
+        headers.update(auth_headers(api, secret))
+    if api == "anthropic":
+        headers["anthropic-version"] = "2023-06-01"
     url = entry.get("models_url") or entry["base"].rstrip("/") + "/models"
+    if api == "gemini" and not entry.get("models_url"):
+        url += "?pageSize=1000"                        # 50 a page otherwise
     with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=timeout) as response:
         data = json.load(response)
-    rows = data.get("data", []) if isinstance(data, dict) else data
+    if api == "gemini" and isinstance(data, dict):
+        # Gemini's list: {"models": [{"name": "models/gemini-x", ...}]}; only
+        # what generateContent serves, by its bare name.
+        rows = [{**m, "id": str(m.get("name", "")).removeprefix("models/")} for m in data.get("models", [])
+                if isinstance(m, dict) and "generateContent" in (m.get("supportedGenerationMethods") or ())]
+    else:
+        rows = data.get("data", []) if isinstance(data, dict) else data
     return {e["id"]: e for e in rows
             if isinstance(e, dict) and e.get("id") and e.get("visibility", "list") == "list"}