@blockrun/clawrouter 0.12.44 → 0.12.45

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blockrun/clawrouter",
-  "version": "0.12.44",
+  "version": "0.12.45",
   "description": "Smart LLM router — save 92% on inference costs. 41+ models, one wallet, x402 micropayments.",
   "type": "module",
   "main": "dist/index.js",

package/scripts/benchmark.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+"""
+BlockRun Model Performance Benchmark v2
+Measures TTFT, tokens/s, and total latency for all BlockRun models.
+Uses varied prompts to bypass response dedup cache.
+"""
+
+import time
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from openai import OpenAI
+
+MODELS = [
+    # OpenAI
+    "openai/gpt-5.4",
+    "openai/gpt-5.4-pro",
+    "openai/gpt-5.3",
+    "openai/gpt-5.3-codex",
+    "openai/gpt-5.2",
+    "openai/gpt-5.2-pro",
+    "openai/gpt-5-mini",
+    "openai/gpt-5-nano",
+    "openai/gpt-4.1",
+    "openai/gpt-4.1-mini",
+    "openai/gpt-4.1-nano",
+    "openai/gpt-4o",
+    "openai/gpt-4o-mini",
+    "openai/o3",
+    "openai/o3-mini",
+    "openai/o4-mini",
+    "openai/o1",
+    "openai/o1-mini",
+    # Anthropic
+    "anthropic/claude-sonnet-4.6",
+    "anthropic/claude-opus-4.6",
+    "anthropic/claude-haiku-4.5",
+    # Google
+    "google/gemini-3.1-pro",
+    "google/gemini-3-pro-preview",
+    "google/gemini-3-flash-preview",
+    "google/gemini-2.5-pro",
+    "google/gemini-2.5-flash",
+    "google/gemini-2.5-flash-lite",
+    # DeepSeek
+    "deepseek/deepseek-chat",
+    "deepseek/deepseek-reasoner",
+    # Moonshot
+    "moonshot/kimi-k2.5",
+    # xAI
+    "xai/grok-3",
+    "xai/grok-3-mini",
+    "xai/grok-4-fast-reasoning",
+    "xai/grok-4-fast-non-reasoning",
+    "xai/grok-4-1-fast-reasoning",
+    "xai/grok-4-1-fast-non-reasoning",
+    "xai/grok-4-0709",
+    # MiniMax
+    "minimax/minimax-m2.5",
+    # NVIDIA
+    "nvidia/gpt-oss-120b",
+]
+
+# Varied prompts to bypass dedup cache — each run uses a different prompt
+PROMPTS = [
+    "Write a Python function that checks if a string is a valid IPv4 address. Include edge cases and a docstring.",
+    "Write a Python function that finds the longest common subsequence of two strings. Include type hints and examples.",
+    "Write a Python function that implements a simple LRU cache using OrderedDict. Include usage examples.",
+    "Write a Python function that converts a Roman numeral string to an integer. Handle all standard cases.",
+    "Write a Python function that flattens a nested list of arbitrary depth. Include type hints.",
+]
+
+NUM_REQUESTS = 2
+MAX_TOKENS = 256
+
+
+def estimate_tokens(text: str) -> int:
+    """Rough token estimate: ~4 chars per token for English/code."""
+    return max(1, len(text) // 4)
+
+
+def benchmark_model(client: OpenAI, model: str) -> list:
+    """Benchmark a single model with streaming."""
+    results = []
+
+    for i in range(NUM_REQUESTS):
+        prompt = PROMPTS[i % len(PROMPTS)]
+
+        try:
+            start = time.perf_counter()
+            ttft = None
+            full_response = ""
+            first_chunk_time = None
+            last_chunk_time = None
+            chunk_count = 0
+
+            stream = client.chat.completions.create(
+                model=model,
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=MAX_TOKENS,
+                stream=True,
+                temperature=0.7,
+            )
+
+            for chunk in stream:
+                now = time.perf_counter()
+                if chunk.choices and chunk.choices[0].delta.content:
+                    content = chunk.choices[0].delta.content
+                    if ttft is None:
+                        ttft = (now - start) * 1000  # ms
+                        first_chunk_time = now
+                    full_response += content
+                    last_chunk_time = now
+                    chunk_count += 1
+
+            end = time.perf_counter()
+            total_time = (end - start) * 1000  # ms
+
+            # Estimate output tokens
+            output_tokens = estimate_tokens(full_response)
+
+            # Tokens per second (generation phase: from first token to last token)
+            if first_chunk_time and last_chunk_time and last_chunk_time > first_chunk_time:
+                gen_time_s = last_chunk_time - first_chunk_time
+                tps = output_tokens / gen_time_s if gen_time_s > 0.001 else 0
+            elif output_tokens > 0 and ttft is not None:
+                # Single chunk — use total time minus TTFT
+                gen_time_s = (end - start) - (ttft / 1000)
+                tps = output_tokens / gen_time_s if gen_time_s > 0.001 else 0
+            else:
+                tps = 0
+
+            # Inter-token latency (approximate from chunks)
+            if first_chunk_time and last_chunk_time and chunk_count > 1:
+                avg_itl = ((last_chunk_time - first_chunk_time) / (chunk_count - 1)) * 1000
+            else:
+                avg_itl = None
+
+            results.append({
+                "request": i + 1,
+                "ttft_ms": round(ttft, 1) if ttft is not None else None,
+                "total_ms": round(total_time, 1),
+                "output_tokens": output_tokens,
+                "output_chars": len(full_response),
+                "chunks": chunk_count,
+                "tokens_per_sec": round(tps, 1),
+                "avg_itl_ms": round(avg_itl, 1) if avg_itl is not None else None,
+                "status": "success",
+            })
+
+            ttft_s = f"{ttft:.0f}" if ttft is not None else "N/A"
+            print(f"  Run {i+1}/{NUM_REQUESTS}: TTFT={ttft_s}ms, {tps:.0f} tok/s, ~{output_tokens} tokens ({len(full_response)} chars), {total_time:.0f}ms")
+
+        except Exception as e:
+            error_msg = str(e)[:300]
+            results.append({
+                "request": i + 1,
+                "status": "error",
+                "error": error_msg,
+            })
+            print(f"  Run {i+1}/{NUM_REQUESTS}: ERROR - {error_msg[:150]}")
+            # If rate limited or out of funds, wait longer
+            if "429" in error_msg or "rate" in error_msg.lower() or "402" in error_msg:
+                print("  >> Rate limited or payment issue, waiting 30s...")
+                time.sleep(30)
+
+        time.sleep(3)
+
+    return results
+
+
+def aggregate(results: list) -> dict:
+    """Aggregate results across runs."""
+    successes = [r for r in results if r["status"] == "success"]
+    if not successes:
+        return {"error_rate": 1.0, "runs": len(results)}
+
+    ttft_vals = [r["ttft_ms"] for r in successes if r.get("ttft_ms") is not None]
+    itl_vals = [r["avg_itl_ms"] for r in successes if r.get("avg_itl_ms") is not None]
+    tps_vals = [r["tokens_per_sec"] for r in successes if r["tokens_per_sec"] > 0]
+
+    agg = {
+        "runs": len(results),
+        "successes": len(successes),
+        "error_rate": round(1 - len(successes) / len(results), 2),
+        "avg_output_tokens": round(sum(r["output_tokens"] for r in successes) / len(successes), 0),
+        "avg_total_ms": round(sum(r["total_ms"] for r in successes) / len(successes), 0),
+    }
+    if tps_vals:
+        agg["avg_tokens_per_sec"] = round(sum(tps_vals) / len(tps_vals), 1)
+        agg["max_tokens_per_sec"] = round(max(tps_vals), 1)
+    if ttft_vals:
+        agg["avg_ttft_ms"] = round(sum(ttft_vals) / len(ttft_vals), 0)
+        agg["p50_ttft_ms"] = round(sorted(ttft_vals)[len(ttft_vals) // 2], 0)
+    if itl_vals:
+        agg["avg_itl_ms"] = round(sum(itl_vals) / len(itl_vals), 1)
+
+    return agg
+
+
+def main():
+    client = OpenAI(
+        api_key="x402",
+        base_url="http://localhost:18789/v1",
+    )
+
+    print("Testing connection to ClawRouter...")
+    try:
+        models = client.models.list()
+        print(f"Connected. {len(models.data)} models available.")
+    except Exception as e:
+        print(f"Failed to connect: {e}")
+        sys.exit(1)
+
+    print(f"Benchmark: {NUM_REQUESTS} requests per model, {MAX_TOKENS} max tokens")
+    print(f"Prompts varied per request to bypass dedup cache\n")
+
+    all_results = {}
+    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    models_to_test = MODELS
+    if len(sys.argv) > 1:
+        models_to_test = sys.argv[1:]
+
+    total = len(models_to_test)
+    for idx, model in enumerate(models_to_test, 1):
+        print(f"\n[{idx}/{total}] {model}")
+        print("-" * 50)
+
+        results = benchmark_model(client, model)
+        agg = aggregate(results)
+        all_results[model] = {"raw": results, "summary": agg}
+
+        if agg.get("successes", 0) > 0:
+            ttft_str = f"TTFT={agg.get('avg_ttft_ms', 'N/A')}ms, " if "avg_ttft_ms" in agg else ""
+            tps_str = f"{agg.get('avg_tokens_per_sec', 0)} tok/s, " if "avg_tokens_per_sec" in agg else ""
+            print(f"  >> {ttft_str}{tps_str}{agg['avg_total_ms']:.0f}ms avg total")
+        else:
+            print(f"  >> ALL FAILED")
+
+        # Save incrementally
+        output = {
+            "benchmark": "BlockRun Model Performance",
+            "version": "1.0",
+            "timestamp": timestamp,
+            "config": {
+                "num_requests": NUM_REQUESTS,
+                "max_tokens": MAX_TOKENS,
+                "prompts": PROMPTS,
+                "token_estimation": "~4 chars per token",
+                "endpoint": "localhost:18789 (ClawRouter → blockrun.ai x402)",
+            },
+            "results": all_results,
+        }
+        outfile = os.path.join(os.path.dirname(__file__), "..", "benchmark-results.json")
+        with open(outfile, "w") as f:
+            json.dump(output, f, indent=2)
+
+    # Leaderboard
+    print("\n" + "=" * 90)
+    print("BLOCKRUN MODEL PERFORMANCE LEADERBOARD")
+    print(f"Tested: {timestamp}")
+    print("=" * 90)
+
+    ranked = []
+    for model, data in all_results.items():
+        s = data["summary"]
+        if s.get("avg_tokens_per_sec", 0) > 0:
+            ranked.append((model, s))
+
+    ranked.sort(key=lambda x: x[1]["avg_tokens_per_sec"], reverse=True)
+
+    print(f"\n{'#':<4} {'Model':<40} {'TTFT':<9} {'Tok/s':<9} {'ITL':<9} {'Total':<9} {'~Tokens':<8}")
+    print("-" * 88)
+    for i, (model, s) in enumerate(ranked, 1):
+        ttft = f"{s['avg_ttft_ms']:.0f}ms" if "avg_ttft_ms" in s else "N/A"
+        itl = f"{s['avg_itl_ms']:.1f}ms" if "avg_itl_ms" in s else "N/A"
+        print(f"{i:<4} {model:<40} {ttft:<9} {s['avg_tokens_per_sec']:<9.1f} {itl:<9} {s['avg_total_ms']:<9.0f} {s['avg_output_tokens']:<8.0f}")
+
+    # Models with issues
+    zero_tps = [(m, d["summary"]) for m, d in all_results.items()
+                if d["summary"].get("avg_tokens_per_sec", 0) == 0 and d["summary"].get("successes", 0) > 0]
+    if zero_tps:
+        print(f"\nModels with 0 tok/s (single-chunk or non-streaming response):")
+        for model, s in zero_tps:
+            ttft = f"{s.get('avg_ttft_ms', 'N/A')}ms"
+            print(f"  {model}: TTFT={ttft}, ~{s.get('avg_output_tokens', 0)} tokens, {s['avg_total_ms']:.0f}ms total")
+
+    errors = [(m, d["summary"]) for m, d in all_results.items() if d["summary"].get("error_rate", 0) > 0]
+    if errors:
+        print(f"\nModels with errors:")
+        for model, s in errors:
+            print(f"  {model}: {s.get('error_rate', 1.0)*100:.0f}% error rate")
+
+    print(f"\nResults saved to benchmark-results.json")
+
+
+if __name__ == "__main__":
+    main()