adaptive-memory-multi-model-router 2.14.49 → 2.14.52

package/data/benchmark-results.json

@@ -0,0 +1,47 @@
+{
+  "timestamp": "2026-06-07T22:37:49.653Z",
+  "version": "2.14.44",
+  "routing": {
+    "total": 81,
+    "correct": 29,
+    "accuracy": 35.8,
+    "offByOne": 66,
+    "offByOneAccuracy": 81.5,
+    "totalCost": 53.7775,
+    "avgCost": 0.66392,
+    "perTier": {
+      "free": {
+        "total": 20,
+        "correct": 20
+      },
+      "cheap": {
+        "total": 21,
+        "correct": 0
+      },
+      "mid": {
+        "total": 20,
+        "correct": 9
+      },
+      "premium": {
+        "total": 20,
+        "correct": 0
+      }
+    }
+  },
+  "memory": {
+    "total": 4,
+    "passed": 4,
+    "accuracy": 100
+  },
+  "robustness": {
+    "total": 6,
+    "passed": 6,
+    "accuracy": 100
+  },
+  "cost": {
+    "avgSavingsPct": -6587,
+    "totalA3m": 4.34,
+    "totalPremium": 0.107
+  },
+  "overallScore": -1925
+}

package/data/labeled-benchmark.json

@@ -0,0 +1,88 @@
+{
+  "queries": [
+    {"query": "What is 2+2?", "actualTier": "free"},
+    {"query": "What is the capital of France?", "actualTier": "free"},
+    {"query": "How do you say hello in Spanish?", "actualTier": "free"},
+    {"query": "Convert 100 Celsius to Fahrenheit", "actualTier": "free"},
+    {"query": "What is the largest planet in the solar system?", "actualTier": "free"},
+    {"query": "How many ounces in a pound?", "actualTier": "free"},
+    {"query": "What is the speed of light?", "actualTier": "free"},
+    {"query": "Who wrote Romeo and Juliet?", "actualTier": "free"},
+    {"query": "What is photosynthesis?", "actualTier": "free"},
+    {"query": "What is the square root of 144?", "actualTier": "free"},
+    {"query": "Name three primary colors", "actualTier": "free"},
+    {"query": "What is the chemical symbol for gold?", "actualTier": "free"},
+    {"query": "How many continents are there?", "actualTier": "free"},
+    {"query": "What is gravity?", "actualTier": "free"},
+    {"query": "Define 'ubiquitous'", "actualTier": "free"},
+    {"query": "What does 'benevolent' mean?", "actualTier": "free"},
+    {"query": "How many bytes in a kilobyte?", "actualTier": "free"},
+    {"query": "What is H2O commonly known as?", "actualTier": "free"},
+    {"query": "Translate 'good morning' to French", "actualTier": "free"},
+    {"query": "What is the time in Tokyo?", "actualTier": "free"},
+
+    {"query": "Write a Python function to sort a list", "actualTier": "cheap"},
+    {"query": "Explain the difference between TCP and UDP", "actualTier": "cheap"},
+    {"query": "Write a SQL query to find duplicate records", "actualTier": "cheap"},
+    {"query": "Create a REST API endpoint in Express.js", "actualTier": "cheap"},
+    {"query": "Explain Docker containers vs virtual machines", "actualTier": "cheap"},
+    {"query": "Write a regex to validate email addresses", "actualTier": "cheap"},
+    {"query": "How do I reverse a string in JavaScript?", "actualTier": "cheap"},
+    {"query": "Write a for loop in Python", "actualTier": "cheap"},
+    {"query": "What is the difference between let and const in JavaScript?", "actualTier": "cheap"},
+    {"query": "Explain what an API is to a non-technical person", "actualTier": "cheap"},
+    {"query": "How do I center a div in CSS?", "actualTier": "cheap"},
+    {"query": "Write a basic HTML form", "actualTier": "cheap"},
+    {"query": "Explain git merge vs rebase", "actualTier": "cheap"},
+    {"query": "What is a RESTful API?", "actualTier": "cheap"},
+    {"query": "How do I read a file in Node.js?", "actualTier": "cheap"},
+    {"query": "Write a simple unit test in Jest", "actualTier": "cheap"},
+    {"query": "Explain what a database index does", "actualTier": "cheap"},
+    {"query": "How do I parse JSON in Python?", "actualTier": "cheap"},
+    {"query": "Write a Python script to send an email", "actualTier": "cheap"},
+    {"query": "Create a simple login form in HTML/CSS", "actualTier": "cheap"},
+    {"query": "How do I make an HTTP request in JavaScript?", "actualTier": "cheap"},
+
+    {"query": "Design a REST API for a blog with users, posts, and comments", "actualTier": "mid"},
+    {"query": "Explain how OAuth 2.0 authentication works", "actualTier": "mid"},
+    {"query": "Design a rate limiting system for a web API", "actualTier": "mid"},
+    {"query": "How would you implement a caching layer for a web application?", "actualTier": "mid"},
+    {"query": "Compare and contrast SQL and NoSQL databases", "actualTier": "mid"},
+    {"query": "Design a user authentication system with JWT tokens", "actualTier": "mid"},
+    {"query": "Explain the CAP theorem and its implications", "actualTier": "mid"},
+    {"query": "How do you design a URL shortening service like bit.ly?", "actualTier": "mid"},
+    {"query": "Design a real-time notification system", "actualTier": "mid"},
+    {"query": "Explain the pros and cons of microservices vs monolith", "actualTier": "mid"},
+    {"query": "How would you implement search functionality in a web app?", "actualTier": "mid"},
+    {"query": "Design a payment processing system", "actualTier": "mid"},
+    {"query": "Explain how blockchain achieves consensus", "actualTier": "mid"},
+    {"query": "Design a multi-tenant SaaS architecture", "actualTier": "mid"},
+    {"query": "How do you handle database migrations in production?", "actualTier": "mid"},
+    {"query": "Design a logging and monitoring system", "actualTier": "mid"},
+    {"query": "Explain load balancing algorithms", "actualTier": "mid"},
+    {"query": "How would you design a file upload system?", "actualTier": "mid"},
+    {"query": "Design a chat application with WebSockets", "actualTier": "mid"},
+    {"query": "Explain event-driven architecture patterns", "actualTier": "mid"},
+
+    {"query": "Design a distributed system that handles 10 million requests per second", "actualTier": "premium"},
+    {"query": "Architect a real-time collaboration system like Google Docs", "actualTier": "premium"},
+    {"query": "Design a machine learning pipeline from data ingestion to model serving", "actualTier": "premium"},
+    {"query": "Create a comprehensive technical design for a social media platform at scale", "actualTier": "premium"},
+    {"query": "Design a zero-downtime deployment strategy for a distributed system", "actualTier": "premium"},
+    {"query": "Architect a system that processes streaming data with sub-second latency", "actualTier": "premium"},
+    {"query": "Design a fraud detection system using machine learning", "actualTier": "premium"},
+    {"query": "Create an architecture for a global CDN with edge computing", "actualTier": "premium"},
+    {"query": "Design a multi-region active-active database architecture", "actualTier": "premium"},
+    {"query": "Architect a system for real-time video transcoding at scale", "actualTier": "premium"},
+    {"query": "Design a privacy-preserving data analytics system", "actualTier": "premium"},
+    {"query": "Create a comprehensive security architecture for a fintech application", "actualTier": "premium"},
+    {"query": "Design a system that handles 1 billion events per day", "actualTier": "premium"},
+    {"query": "Architect a serverless system with consistent performance", "actualTier": "premium"},
+    {"query": "Design a data warehouse architecture for petabyte-scale analytics", "actualTier": "premium"},
+    {"query": "Create a multi-cloud hybrid architecture strategy", "actualTier": "premium"},
+    {"query": "Design a system for real-time anomaly detection in financial transactions", "actualTier": "premium"},
+    {"query": "Architect a mesh network for IoT at scale", "actualTier": "premium"},
+    {"query": "Design a comprehensive disaster recovery strategy", "actualTier": "premium"},
+    {"query": "Create an architecture for autonomous vehicle sensor fusion", "actualTier": "premium"}
+  ]
+}

package/demo/3blue1brown_video.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""
+A3M Router — 3Blue1Brown Style Explainer Video Generator
+
+Creates animated frames showing:
+1. The problem: expensive AI routing
+2. The solution: intelligent routing
+3. How it works: 12 signals
+4. Results: 76.43 score, 213x cheaper
+
+Usage:
+    python3 demo/3blue1brown_video.py
+"""
+
+import subprocess
+import json
+import sys
+import math
+import os
+
+# Check dependencies
+def check_deps():
+    try:
+        import matplotlib
+        import numpy
+        print("✅ matplotlib, numpy available")
+        return True
+    except ImportError as e:
+        print(f"❌ Missing: {e}")
+        print("Install: pip install matplotlib numpy pillow")
+        return False
+
+def run_command(cmd, cwd=None):
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True, cwd=cwd)
+    return result.returncode == 0, result.stdout, result.stderr
+
+# Frame definitions - each scene
+SCENES = [
+    {
+        "id": 1,
+        "title": "The AI Cost Problem",
+        "content": "Every AI query goes to GPT-4\n$10.02 per 1,000 queries",
+        "visual": "gpt4_cost_visual",
+        "duration": 3.0
+    },
+    {
+        "id": 2,
+        "title": "The Solution",
+        "content": "Route queries intelligently\nto the cheapest capable model",
+        "visual": "routing_diagram",
+        "duration": 3.0
+    },
+    {
+        "id": 3,
+        "title": "How A3M Routes",
+        "content": "12 signals → instant decision\n<1ms, no GPU needed",
+        "visual": "signals_animation",
+        "duration": 4.0
+    },
+    {
+        "id": 4,
+        "title": "The Result",
+        "content": "#1 on RouterArena\n76.43 accuracy at $0.047/1K",
+        "visual": "benchmark_reveal",
+        "duration": 3.0
+    },
+    {
+        "id": 5,
+        "title": "213x Cheaper",
+        "content": "vs GPT-5: $10.02 → $0.047\nSame quality, 1/213th the cost",
+        "visual": "cost_comparison",
+        "duration": 3.0
+    }
+]
+
+def generate_frame_matplotlib(scene_num, frame_num, total_frames, output_path):
+    """Generate a single frame using matplotlib"""
+    import matplotlib.pyplot as plt
+    import matplotlib.patches as patches
+    import numpy as np
+    
+    # 3Blue1Brown color scheme
+    bg_color = '#1a1a2e'
+    text_color = '#ffffff'
+    accent1 = '#00a8ff'  # Blue
+    accent2 = '#ff6b6b'  # Red
+    accent3 = '#4ecdc4'  # Teal
+    accent4 = '#ffe66d'  # Yellow
+    
+    fig, ax = plt.subplots(figsize=(16, 9), facecolor=bg_color)
+    ax.set_facecolor(bg_color)
+    ax.set_xlim(0, 16)
+    ax.set_ylim(0, 9)
+    ax.axis('off')
+    
+    if scene_num == 1:
+        # Scene 1: GPT-4 cost visualization
+        # Big "$10.02" in center
+        ax.text(8, 5.5, '$10.02', fontsize=120, color=accent2, 
+                ha='center', va='center', fontweight='bold', family='monospace')
+        ax.text(8, 3.5, 'per 1,000 queries', fontsize=36, color=text_color,
+                ha='center', va='center')
+        ax.text(8, 1.5, 'GPT-4o', fontsize=28, color=accent1,
+                ha='center', va='center', style='italic')
+        
+        # Animated circles representing money
+        progress = frame_num / total_frames
+        for i in range(int(progress * 20)):
+            x = 2 + (i % 5) * 3
+            y = 7.5 - (i // 5) * 0.8
+            circle = patches.Circle((x, y), 0.3, color=accent2, alpha=0.6)
+            ax.add_patch(circle)
+            
+    elif scene_num == 2:
+        # Scene 2: Routing diagram
+        ax.text(8, 7.5, 'A3M Router', fontsize=48, color=accent1,
+                ha='center', va='center', fontweight='bold')
+        
+        # Query box
+        rect = patches.FancyBboxPatch((1, 3.5), 3, 2, boxstyle="round,pad=0.1",
+                                       facecolor=accent3, edgecolor='white', linewidth=2)
+        ax.add_patch(rect)
+        ax.text(2.5, 4.5, 'Query', fontsize=24, color=bg_color,
+                ha='center', va='center', fontweight='bold')
+        
+        # Arrow
+        ax.annotate('', xy=(5.5, 4.5), xytext=(4, 4.5),
+                    arrowprops=dict(arrowstyle='->', color=accent1, lw=3))
+        
+        # Router box
+        rect = patches.FancyBboxPatch((5.5, 3), 4, 3, boxstyle="round,pad=0.1",
+                                       facecolor=accent1, edgecolor='white', linewidth=3)
+        ax.add_patch(rect)
+        ax.text(7.5, 5, 'A3M', fontsize=28, color=text_color,
+                ha='center', va='center', fontweight='bold')
+        ax.text(7.5, 4, '12 signals', fontsize=18, color='white',
+                ha='center', va='center')
+        ax.text(7.5, 3.3, '<1ms routing', fontsize=14, color='white',
+                ha='center', va='center')
+        
+        # Arrows to providers
+        providers = [('Groq\nFree', 12, 6.5, accent3),
+                    ('GPT-4o\nPremium', 12, 4.5, accent2),
+                    ('Gemini\nMid', 12, 2.5, accent4)]
+        
+        progress = frame_num / total_frames
+        for i, (name, x, y, color) in enumerate(providers):
+            if progress > (i * 0.3):
+                ax.annotate('', xy=(x, y), xytext=(9.5, 4.5),
+                            arrowprops=dict(arrowstyle='->', color=color, lw=2))
+                rect = patches.FancyBboxPatch((x-0.8, y-0.5), 1.6, 1,
+                                               boxstyle="round,pad=0.05",
+                                               facecolor=color, edgecolor='white', 
+                                               linewidth=1, alpha=min(1, (progress - i*0.3)*3))
+                ax.add_patch(rect)
+                ax.text(x, y, name, fontsize=12, color=bg_color,
+                        ha='center', va='center', fontweight='bold', alpha=min(1, (progress - i*0.3)*3))
+                
+    elif scene_num == 3:
+        # Scene 3: 12 signals animation
+        ax.text(8, 8, '12 Keyword Signals', fontsize=44, color=accent1,
+                ha='center', va='center', fontweight='bold')
+        ax.text(8, 7, 'Multi-dimensional query analysis', fontsize=24, color=text_color,
+                ha='center', va='center', alpha=0.7)
+        
+        # Grid of signals
+        signals = ['coding', 'math', 'creative', 'factual', 
+                   'simple', 'complex', 'debug', 'explain',
+                   'short', 'long', 'structured', 'open']
+        
+        progress = frame_num / total_frames
+        cols = 4
+        for idx, sig in enumerate(signals):
+            row = idx // cols
+            col = idx % cols
+            x = 3 + col * 2.5
+            y = 5.5 - row * 1.8
+            
+            alpha = min(1, max(0, (progress - idx * 0.08) * 3))
+            if alpha > 0:
+                color = [accent1, accent3, accent4, accent2][idx % 4]
+                circle = patches.Circle((x, y), 0.5, color=color, alpha=alpha * 0.8)
+                ax.add_patch(circle)
+                ax.text(x, y, sig, fontsize=14, color=bg_color,
+                        ha='center', va='center', fontweight='bold', alpha=alpha)
+                
+    elif scene_num == 4:
+        # Scene 4: Benchmark reveal
+        ax.text(8, 7, 'RouterArena Benchmark', fontsize=44, color=text_color,
+                ha='center', va='center', fontweight='bold')
+        
+        progress = frame_num / total_frames
+        
+        # Score reveal
+        if progress > 0.3:
+            score_alpha = min(1, (progress - 0.3) * 3)
+            ax.text(8, 5, '76.43', fontsize=120, color=accent1,
+                    ha='center', va='center', fontweight='bold', alpha=score_alpha,
+                    family='monospace')
+            ax.text(8, 3, '#1 OF 19 ROUTERS', fontsize=28, color=accent3,
+                    ha='center', va='center', alpha=score_alpha)
+            ax.text(8, 2, 'arXiv:2510.00202', fontsize=18, color=text_color,
+                    ha='center', va='center', alpha=score_alpha * 0.7)
+            
+    elif scene_num == 5:
+        # Scene 5: Cost comparison
+        ax.text(8, 7.5, '213x Cheaper', fontsize=56, color=accent3,
+                ha='center', va='center', fontweight='bold')
+        
+        progress = frame_num / total_frames
+        
+        # GPT-5 cost
+        if progress > 0.2:
+            alpha = min(1, (progress - 0.2) * 3)
+            ax.text(4, 5, '$10.02', fontsize=64, color=accent2,
+                    ha='center', va='center', fontweight='bold', alpha=alpha)
+            ax.text(4, 3.5, 'GPT-5', fontsize=24, color=text_color,
+                    ha='center', va='center', alpha=alpha)
+            
+        # A3M cost
+        if progress > 0.5:
+            alpha = min(1, (progress - 0.5) * 3)
+            ax.text(12, 5, '$0.047', fontsize=64, color=accent3,
+                    ha='center', va='center', fontweight='bold', alpha=alpha)
+            ax.text(12, 3.5, 'A3M', fontsize=24, color=text_color,
+                    ha='center', va='center', alpha=alpha)
+            
+        # VS text
+        if progress > 0.7:
+            alpha = min(1, (progress - 0.7) * 3)
+            ax.text(8, 5, 'vs', fontsize=36, color=text_color,
+                    ha='center', va='center', alpha=alpha * 0.5)
+            ax.text(8, 2, 'Same quality, 1/213th the cost',
+                    fontsize=22, color=text_color, ha='center', va='center', alpha=alpha)
+    
+    plt.tight_layout()
+    plt.savefig(output_path, facecolor=bg_color, dpi=100)
+    plt.close()
+
+def main():
+    if not check_deps():
+        sys.exit(1)
+    
+    print("🎬 A3M Router — 3Blue1Brown Style Video Generator")
+    print("=" * 50)
+    
+    # Create output directory
+    output_dir = '/Users/Subho/adaptive-memory-multi-model-router/demo/frames_3b1b'
+    os.makedirs(output_dir, exist_ok=True)
+    
+    total_scenes = len(SCENES)
+    frames_per_scene = 30  # 30 frames per scene = ~1s at 30fps
+    
+    print(f"Generating {total_scenes} scenes, {frames_per_scene} frames each...")
+    print(f"Output: {output_dir}")
+    
+    for scene in SCENES:
+        scene_id = scene['id']
+        scene_dir = os.path.join(output_dir, f'scene_{scene_id}')
+        os.makedirs(scene_dir, exist_ok=True)
+        
+        print(f"\n📽️ Scene {scene_id}: {scene['title']}")
+        
+        for frame_num in range(1, frames_per_scene + 1):
+            output_path = os.path.join(scene_dir, f'frame_{frame_num:04d}.png')
+            generate_frame_matplotlib(scene_id, frame_num, frames_per_scene, output_path)
+            
+            if frame_num % 10 == 0:
+                print(f"  Frame {frame_num}/{frames_per_scene}")
+    
+    print(f"\n✅ Generated {total_scenes * frames_per_scene} frames")
+    print(f"\nNext steps:")
+    print(f"1. Install ffmpeg: brew install ffmpeg")
+    print(f"2. Run: ./demo/render_3b1b_video.sh")
+    
+    # Save scene config
+    with open(os.path.join(output_dir, 'scenes.json'), 'w') as f:
+        json.dump(SCENES, f, indent=2)
+    
+    print(f"\n3. Or use Python to create video:")
+    print(f"   python3 -c \"from PIL import Image; import subprocess\"")
+    print(f"   # Then run ffmpeg to compile")
+
+if __name__ == '__main__':
+    main()