adaptive-memory-multi-model-router 2.13.18 → 2.13.22

package/demo/server.js

@@ -0,0 +1,405 @@
+/**
+ * A3M Router Demo Server
+ *
+ * Interactive web demo showing parallel multi-LLM routing.
+ * Works without API keys — falls back to mock results.
+ */
+
+import express from "express";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// ============================================================
+// Lazy-load A3M Router (fail gracefully if not installed locally)
+// ============================================================
+let a3m;
+try {
+  a3m = await import("adaptive-memory-multi-model-router");
+} catch {
+  console.warn("[demo] A3M Router not installed locally — using file:// import");
+  // Try relative path for development
+  try {
+    a3m = await import("../dist/index.js");
+  } catch {
+    a3m = null;
+  }
+}
+
+// ============================================================
+// Fallback responses for when no real API keys are configured
+// ============================================================
+const MOCK_RESPONSES = {
+  "Write a haiku about Kubernetes": {
+    content: `Pods drift through the cloud\nOrchestrator hums softly\nContainers at rest`,
+    model: "groq/llama-3.3-70b-versatile",
+    provider: "Groq",
+  },
+  "Explain quantum computing to a 5-year-old": {
+    content: `Imagine you have a magical coin. Normal coins are either heads OR tails. But a magical quantum coin can be heads AND tails at the same time! That's what quantum computers use — they have special "magic coins" called qubits (say "kyoo-bits"). 
+
+While a regular computer checks one answer at a time (like trying every key on a keychain), a quantum computer can check ALL the answers at once. That makes it super fast at solving certain puzzles, like finding the right key instantly!`,
+    model: "anthropic/claude-sonnet-4-20250514",
+    provider: "Anthropic",
+  },
+  "Write a React component for a search bar": {
+    content: `\`\`\`jsx
+import React, { useState, useCallback } from 'react';
+
+function SearchBar({ onSearch, placeholder = "Search..." }) {
+  const [query, setQuery] = useState("");
+  const [isFocused, setIsFocused] = useState(false);
+
+  const handleChange = useCallback((e) => {
+    setQuery(e.target.value);
+  }, []);
+
+  const handleSubmit = useCallback((e) => {
+    e.preventDefault();
+    if (query.trim()) onSearch(query.trim());
+  }, [query, onSearch]);
+
+  const handleClear = useCallback(() => {
+    setQuery("");
+  }, []);
+
+  return (
+    <form
+      onSubmit={handleSubmit}
+      style={{
+
+        display: "flex",
+        alignItems: "center",
+        background: isFocused ? "#fff" : "#f5f5f5",
+        border: \`2px solid \${isFocused ? "#6366f1" : "#e5e7eb"}\`,
+        borderRadius: "12px",
+        padding: "8px 16px",
+        transition: "all 0.2s ease",
+        maxWidth: "500px",
+      }}
+    >
+      <svg width="20" height="20" viewBox="0 0 20 20" fill="none" style={{ marginRight: 8 }}>
+        <path d="M8 4a4 4 0 100 8 4 4 0 000-8zM2 8a6 6 0 1110.89 3.476l4.817 4.817a1 1 0 01-1.414 1.414l-4.816-4.816A6 6 0 012 8z" fill="#9ca3af"/>
+      </svg>
+      <input
+        type="text"
+        value={query}
+        onChange={handleChange}
+        onFocus={() => setIsFocused(true)}
+        onBlur={() => setIsFocused(false)}
+        placeholder={placeholder}
+        style={{
+          flex: 1,
+          border: "none",
+          outline: "none",
+          background: "transparent",
+          fontSize: "16px",
+          padding: "4px 0",
+          color: "#1f2937",
+        }}
+      />
+      {query && (
+        <button
+          type="button"
+          onClick={handleClear}
+          style={{
+            background: "none",
+            border: "none",
+            cursor: "pointer",
+            padding: 4,
+            color: "#9ca3af",
+          }}
+          aria-label="Clear search"
+        >
+          ✕
+        </button>
+      )}
+      <button
+        type="submit"
+        style={{
+          background: "#6366f1",
+          color: "#fff",
+          border: "none",
+          borderRadius: "8px",
+          padding: "8px 16px",
+          marginLeft: 8,
+          cursor: "pointer",
+          fontWeight: 600,
+          fontSize: "14px",
+        }}
+      >
+        Search
+      </button>
+    </form>
+  );
+}
+
+export default SearchBar;
+\`\`\``,
+    model: "openai/gpt-4o",
+    provider: "OpenAI",
+  },
+};
+
+const DEFAULT_MOCK = {
+  content:
+    "A3M Router selected the optimal model for your query. Configure API keys to get real LLM responses — the routing engine analyzed your query's complexity, domain, and requirements to find the best provider.",
+  model: "a3m-routing/auto",
+  provider: "A3M Routing Engine",
+};
+
+// ============================================================
+// Cost calculator (based on A3M Router token costs)
+// ============================================================
+const MODEL_COSTS = {
+  "groq/llama-3.3-70b-versatile": { input: 0.59, output: 0.79 },
+  "anthropic/claude-sonnet-4-20250514": { input: 3.0, output: 15.0 },
+  "openai/gpt-4o": { input: 2.5, output: 10.0 },
+};
+
+function estimateCost(model, content) {
+  const tokens = Math.ceil(content.length / 4); // rough estimate
+  const costs = MODEL_COSTS[model] || { input: 0.5, output: 0.5 };
+  return ((tokens / 1_000_000) * (costs.input + costs.output)).toFixed(6);
+}
+
+// ============================================================
+// Express App
+// ============================================================
+const app = express();
+app.use(express.json());
+app.use(express.static(path.join(__dirname, "public")));
+
+// ── GET /api/providers ───────────────────────────────────
+app.get("/api/providers", (req, res) => {
+  let providers = {};
+  let hasRealKeys = false;
+
+  if (a3m) {
+    try {
+      providers = a3m.getAvailableProviders();
+      hasRealKeys = Object.keys(providers).length > 0;
+    } catch {
+      providers = {};
+    }
+  }
+
+  // Build a clean list of what's available
+  const list = Object.entries(providers).map(([id, p]) => ({
+    id,
+    name: p.name,
+    tier: p.tier,
+    models: p.models?.length || 0,
+    type: p.type,
+    costPerM: p.costPerK
+      ? { input: p.costPerK.input, output: p.costPerK.output }
+      : { input: 0, output: 0 },
+  }));
+
+  res.json({
+    count: list.length,
+    hasApiKeys: hasRealKeys,
+    providers: list,
+    note: hasRealKeys
+      ? "API keys detected — real LLM responses enabled"
+      : "No API keys configured — showing simulated responses",
+  });
+});
+
+// ── POST /api/analyze ───────────────────────────────────
+app.post("/api/analyze", (req, res) => {
+  const { prompt } = req.body;
+  if (!prompt || typeof prompt !== "string") {
+    return res.status(400).json({ error: "prompt is required" });
+  }
+
+  let features = null;
+  let routing = null;
+
+  if (a3m) {
+    try {
+      features = a3m.extractQueryFeatures(prompt);
+      routing = a3m.routeQuery(prompt);
+    } catch {
+      // fallback below
+    }
+  }
+
+  if (!features) {
+    const complexity = Math.min(1, 0.15 + prompt.length / 500);
+    features = {
+      complexity,
+      length: prompt.split(/\s+/).length,
+      has_code: /function|class |def |import|<[a-z]+|const |let |var /i.test(prompt),
+      has_math: /\d+[\+\-\*\/=]|calculate|sum|equation/i.test(prompt),
+      requires_reasoning: /why|explain|analyze|compare|reason/i.test(prompt),
+      is_creative: /write|story|poem|create|imagine/i.test(prompt),
+      detected_domain: "general",
+    };
+  }
+
+  if (!routing) {
+    routing = {
+      primary_model: "a3m/auto-routed",
+      confidence: 0.85,
+      reasoning: "Intelligent routing based on query complexity and domain",
+      estimated_cost: "0.000001",
+      estimated_latency_ms: 420,
+    };
+  }
+
+  res.json({ features, routing });
+});
+
+// ── POST /api/route ────────────────────────────────────
+app.post("/api/route", async (req, res) => {
+  const { prompt, mode } = req.body;
+  if (!prompt || typeof prompt !== "string") {
+    return res.status(400).json({ error: "prompt is required" });
+  }
+
+  const startTime = Date.now();
+  const isEnsemble = mode === "ensemble";
+
+  try {
+    // 1) Route the query using A3M Router
+    let routing = null;
+    let features = null;
+
+    if (a3m) {
+      try {
+        routing = a3m.routeQuery(prompt);
+        features = a3m.extractQueryFeatures(prompt);
+      } catch {
+        // fallback
+      }
+    }
+
+    // 2) Try real LLM execution via proxy server
+    let content = null;
+    let provider = null;
+    let model = null;
+    let usedRealProvider = false;
+
+    // Check if we have any API keys configured
+    let availableProviders = {};
+    if (a3m) {
+      try {
+        availableProviders = a3m.getAvailableProviders();
+      } catch {}
+    }
+
+    const hasRealKeys = Object.keys(availableProviders).length > 0;
+
+    if (hasRealKeys && a3m) {
+      try {
+        // Try routing via proxy
+        const modelName = "auto";
+        const messages = [{ role: "user", content: prompt }];
+
+        const modelMapper = await import("../dist/server/modelMapper.js");
+        const mapping = modelMapper.resolveModel(modelName, prompt);
+        if (mapping && mapping.apiKey) {
+          const baseUrl = mapping.baseUrl;
+          const body = {
+            model: mapping.model,
+            messages,
+            max_tokens: 512,
+          };
+          const headers = { "Content-Type": "application/json" };
+          if (mapping.apiKey) headers["Authorization"] = `Bearer ${mapping.apiKey}`;
+
+          const resp = await fetch(baseUrl, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+          });
+          const data = await resp.json();
+          if (data.choices?.[0]?.message?.content) {
+            content = data.choices[0].message.content;
+            provider = mapping.providerId;
+            model = mapping.model;
+            usedRealProvider = true;
+          }
+        }
+      } catch (e) {
+        console.warn("[demo] Real provider call failed:", e.message);
+      }
+    }
+
+    // 3) Fallback to mock if no real response
+    if (!content) {
+      const mockKey = Object.keys(MOCK_RESPONSES).find((k) => prompt.includes(k));
+      const mock = mockKey ? MOCK_RESPONSES[mockKey] : DEFAULT_MOCK;
+      content = mock.content;
+      provider = mock.provider;
+      model = mock.model;
+    }
+
+    // 4) Ensemble mode: return multiple responses
+    let responses = null;
+    if (isEnsemble) {
+      const providers = ["Groq", "Anthropic", "OpenAI"];
+      responses = providers.map((p) => ({
+        provider: p,
+        model: p === "Groq"
+          ? "llama-3.3-70b-versatile"
+          : p === "Anthropic"
+            ? "claude-sonnet-4-20250514"
+            : "gpt-4o",
+        content: `[Simulated response from ${p}] A3M Router selected ${p} for optimal performance on your query. Configure API keys to get real responses from all providers simultaneously.`,
+        latency: Math.floor(Math.random() * 400 + 200),
+        cost: (Math.random() * 0.002).toFixed(6),
+      }));
+    }
+
+    const latency = Date.now() - startTime;
+    const cost = usedRealProvider
+      ? estimateCost(model, content)
+      : "0.000001";
+
+    res.json({
+      result: content,
+      provider,
+      model,
+      mode: isEnsemble ? "ensemble" : "single",
+      latency,
+      cost,
+      usedMock: !usedRealProvider,
+      features,
+      routing,
+      responses,
+      hasRealKeys,
+    });
+  } catch (err) {
+    console.error("[demo] Route error:", err);
+    res.status(500).json({
+      error: err.message,
+      result: "A3M Router encountered an error processing your query.",
+      provider: "A3M Router",
+      model: "error",
+      latency: Date.now() - startTime,
+      cost: "0",
+      usedMock: true,
+    });
+  }
+});
+
+// ── Serve index.html for all other routes ──────────────
+app.get("*", (req, res) => {
+  res.sendFile(path.join(__dirname, "public", "index.html"));
+});
+
+// ── Start ──────────────────────────────────────────────
+const PORT = process.env.PORT || 3000;
+app.listen(PORT, () => {
+  console.log(`
+  A3M Router Demo Server
+  ─────────────────────────
+  Running:  http://localhost:${PORT}
+  API Key:  ${process.env.OPENAI_API_KEY ? "Yes" : "No (mock mode)"}
+  Providers: ${a3m ? "A3M SDK loaded" : "A3M SDK not available"}
+  Mode:     ${process.env.OPENAI_API_KEY || process.env.GROQ_API_KEY ? "Live" : "Demo (mock responses)"}
+  `);
+});

package/dist/cli.js

@@ -154,6 +154,10 @@ async function callProvider(providerId, model, prompt, maxTokens) {
 // ============================================================
 
 async function main() {
+  // Show banner
+  const banner = require('./scripts/banner.js');
+  process.stdout.write(banner);
+
   const router = createA3MRouter({ memory: { maxSize: 1000 } });
 
   switch (command) {

package/docker-compose.yml

@@ -0,0 +1,74 @@
+# =============================================================================
+# A3M Router — Docker Compose
+# Runs the main router service + OpenAI-compatible proxy side-by-side
+# =============================================================================
+
+version: "3.9"
+
+services:
+
+  # ---------------------------------------------------------------------------
+  # A3M Router — Main API & health endpoint
+  # ---------------------------------------------------------------------------
+  a3m-router:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: a3m-router
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    environment:
+      - NODE_ENV=production
+      - PORT=3000
+      # Pass through any .env variables needed at runtime
+      - NVIDIA_API_KEY=${NVIDIA_API_KEY}
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - GEMINI_API_KEY=${GEMINI_API_KEY}
+      - GROQ_API_KEY=${GROQ_API_KEY}
+      - CEREBRAS_API_KEY=${CEREBRAS_API_KEY}
+      # Add additional provider keys as needed
+    env_file:
+      - .env
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3000/health"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+  # ---------------------------------------------------------------------------
+  # A3M Proxy — OpenAI-compatible proxy server (port 8787)
+  # ---------------------------------------------------------------------------
+  a3m-proxy:
+    build:
+      context: .
+      dockerfile: Dockerfile.proxy
+    container_name: a3m-proxy
+    restart: unless-stopped
+    ports:
+      - "8787:8787"
+    environment:
+      - NODE_ENV=production
+      - PORT=8787
+      - ROUTER_BASE_URL=http://a3m-router:3000
+    depends_on:
+      a3m-router:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8787/health"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"

package/docs/BENCHMARK.md

@@ -1,5 +1,16 @@
 # A3M Router — Independent Benchmark
 
+A3M Router is benchmarked on two independent dimensions:
+
+1. **Latency** — How much overhead does the gateway add? (real API calls)
+2. **Routing Accuracy** — How well does the complexity classifier sort queries into tiers? (offline, 200 queries)
+
+Both benchmarks are reproducible — scripts live in `scripts/`.
+
+---
+
+## 1. Latency Benchmark
+
 **The question everyone asks:** *"How much latency does a gateway add?"*
 
 **The answer:** +96ms for passthrough, +236ms for full intelligent routing — on a 138ms baseline.
@@ -8,9 +19,7 @@
 
 *Left: latency comparison. Right: cost savings projection. Dark theme.*
 
----
-
-## The TL;DR
+### The TL;DR
 
 ```
 Direct call to Groq:          ──▸ 138ms  (baseline)
@@ -25,9 +34,7 @@ Through A3M auto (routed):    ──▸ 374ms  (+140ms = routing decision)
 
 **Total overhead: 236ms.** Less than the time it takes to blink.
 
----
-
-## The Details
+### The Details
 
 | Scenario | Time | What's happening |
 |:---------|:----:|:-----------------|
@@ -35,11 +42,9 @@ Through A3M auto (routed):    ──▸ 374ms  (+140ms = routing decision)
 | **Through A3M (forced route)** | **234ms** | Request hits A3M proxy. Guardrails scan for prompt injection (17 patterns) and PII. Cache checks for semantic duplicates. Cost tracker logs the call. Request forwarded to Groq. Response logged. |
 | **Through A3M (auto route)** | **374ms** | Everything above, plus: A3M's router extracts 12 signals from the query text — domain, task type, complexity, verb intensity, multi-step structure. Scores it. Assigns a tier. Selects the cheapest capable model. Forwards the request. |
 
-**The extra 140ms for auto-routing is the intelligence.** It's the difference between "throw every query at GPT-4o" and "route simple questions to free tier, code questions to DeepSeek, expert questions to premium."
-
----
+**The extra 140ms for auto-routing is the intelligence.**
 
-## The Trade-Off
+### The Trade-Off
 
 ```text
                         Without A3M                  With A3M
@@ -54,9 +59,7 @@ Cost visibility:        End-of-month surprise        Per-query tracking + budget
 
 **236ms of overhead saves you $2,604/year.** That's about $11 per millisecond.
 
----
-
-## Why Most Gateways Don't Publish This
+### Why Most Gateways Don't Publish This
 
 Every gateway adds latency. Most don't publish their numbers because they're either:
 
@@ -66,23 +69,15 @@ Every gateway adds latency. Most don't publish their numbers because they're eit
 
 A3M publishes this because the numbers are honest and the trade-off is clear: **pay 236ms, save 62%, get production-grade security.**
 
----
-
-## Reproduce This Yourself
+### Reproduce This
 
 ```bash
-# Install the benchmark tool
 pip install llm-gateway-bench
-
-# Start A3M proxy
 npx a3m-router serve
-
-# Run comparison
 python3 -m llm_gateway_bench.cli run groq \
   --model llama-3.3-70b-versatile \
   --prompt "What is the capital of France?" \
   --requests 10
-
 python3 -m llm_gateway_bench.cli run custom \
   --model auto \
   --base-url http://localhost:8787/v1 \
@@ -94,3 +89,82 @@ python3 -m llm_gateway_bench.cli run custom \
 **Run date:** 2026-05-26  
 **Provider:** Groq (llama-3.3-70b-versatile)  
 **Methodology:** 3 prompts × 5 requests = 15 calls per scenario, real API calls
+
+---
+
+## 2. Routing Accuracy Benchmark
+
+**The question everyone asks:** *"Does the complexity classifier actually pick the right tier?"*
+
+**The answer:** **99.5% ±1 tier accuracy** across 200 diverse queries — no ML training needed.
+
+Benchmark script: `scripts/routing-benchmark-v2.js`  
+Methodology: RouteLLM-inspired (arXiv:2404.06035), 4-tier classification
+
+### Results (2026-05-28)
+
+| Metric | Score | What It Means |
+|:-------|:-----:|:--------------|
+| **±1 Tier Accuracy** | **99.5%** | Only 1 in 200 queries is misrouted by >1 tier |
+| Exact Tier Match | 64.5% | ~2 in 3 queries hit the *exact* right tier |
+| Free Tier Recall | 92.0% | Simple queries correctly routed to $0 models |
+| Cheap Tier Recall | 78.3% | Standard code/translation routed to cheap |
+| Mid Tier Recall | 36.0% | Complex reasoning often routed cheaper (fallback-safe) |
+| Premium Tier Recall | 45.0% | Expert queries routed to premium |
+| Over-routing (waste) | 7.0% | Sent to a stronger but costlier model than needed |
+| Under-routing (risk) | 28.5% | Sent weak first; auto-fallback in <2s |
+| Cost Savings vs All-Premium | **61.6%** | At 100K queries/mo: **save $77.04/mo** |
+
+### Confusion Matrix
+
+```
+Expected \\ Routed   free    cheap    mid   premium
+──────────────────────────────────────────────────
+free                 46✓      4       0       0
+cheap                11      47✓      2       0
+mid                   0      24      18✓      8
+premium               0       1      21      18✓
+```
+
+### Complexity Score Distribution
+
+```
+free         avg=0.125  range=[0.100, 0.270]
+cheap        avg=0.275  range=[0.100, 0.575]
+mid          avg=0.477  range=[0.230, 0.710]
+premium      avg=0.690  range=[0.430, 1.000]
+```
+
+### Test Set
+
+- **50 simple** — trivia, basic math, yes/no (target: free)
+- **60 medium** — code snippets, summarization, translation (target: cheap)
+- **50 complex** — reasoning, analysis, system design (target: mid)
+- **40 expert** — legal, medical, security, finance (target: premium)
+
+### Third-Party Cross-Validation
+
+A3M's tier assignments align with **MMLU accuracy rankings**:
+
+```
+Provider          MMLU    A3M Tier    Source
+────────────────────────────────────────────────
+gpt-4o            88.7%   premium     MMLU Leaderboard
+claude-3.5-sonnet  88.4%   premium     MMLU Leaderboard
+gemini-1.5-pro     85.7%   premium     MMLU Leaderboard
+mistral-large      84.2%   mid         MMLU Leaderboard
+llama-3.3-70b      82.5%   mid         MMLU Leaderboard
+deepseek-v2        78.3%   mid         MMLU Leaderboard
+llama-3.1-8b       68.3%   cheap       MMLU Leaderboard
+```
+
+**References:** [MMLU Leaderboard](https://paperswithcode.com/sota/multi-task-language-understanding-on-mmlu), [RouteLLM arXiv:2404.06035](https://arxiv.org/abs/2404.06035)
+
+### Reproduce This
+
+```bash
+cd /path/to/a3m-router
+node scripts/routing-benchmark-v2.js
+```
+
+Outputs `benchmark-results.json` with full breakdown.