adaptive-memory-multi-model-router 2.14.46 → 2.14.48

package/scripts/local-router-benchmark.ts

@@ -1,154 +0,0 @@
-/**
- * Local Router Benchmark - Test routing decisions without API calls
- * 
- * Uses the test dataset to evaluate routing accuracy per tier.
- * Focuses on understanding WHERE the router fails.
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-// Load test data
-const TEST_DATA_PATH = './data/test/standup_word_level.json';
-const RESULTS_PATH = './benchmark-results.json';
-
-interface QueryFeature {
-  actualTier: 'free' | 'cheap' | 'mid' | 'premium';
-  query: string;
-  routingReason: string;
-}
-
-interface ConfusionEntry {
-  actual: string;
-  predicted: string;
-  query: string;
-  correct: boolean;
-}
-
-interface BenchmarkResult {
-  timestamp: string;
-  version: string;
-  total: number;
-  correct: number;
-  accuracy: number;
-  confusion: { [actual: string]: { [predicted: string]: number } };
-  failures: ConfusionEntry[];
-  tierAccuracy: { [tier: string]: { correct: number; total: number; accuracy: number } };
-  failureAnalysis: {
-    underRoutes: { [tier: string]: number };
-    overRoutes: { [tier: string]: number };
-  };
-}
-
-function loadTestData(): QueryFeature[] {
-  const data = JSON.parse(fs.readFileSync(TEST_DATA_PATH, 'utf-8'));
-  return data.queries || data;
-}
-
-function analyzeFailures(failures: ConfusionEntry[]): { [key: string]: number } {
-  const analysis: { [key: string]: number } = {};
-  
-  for (const f of failures) {
-    if (!f.correct) {
-      const key = `${f.actual}→${f.predicted}`;
-      analysis[key] = (analysis[key] || 0) + 1;
-    }
-  }
-  
-  return analysis;
-}
-
-async function runBenchmark(): Promise<BenchmarkResult> {
-  const queries = loadTestData();
-  const { routeQuery, createRouterContext } = await import('../src/index');
-  
-  // Create context once
-  const context = await createRouterContext();
-  
-  const confusion: { [key: string]: { [key: string]: number } } = {
-    'free': {}, 'cheap': {}, 'mid': {}, 'premium': {}
-  };
-  
-  const failures: ConfusionEntry[] = [];
-  let correct = 0;
-  
-  for (const item of queries) {
-    const query = item.query || item.text || item.input || String(item);
-    const actualTier = item.actualTier || item.tier || item.optimalTier || 'mid';
-    
-    try {
-      // Route the query
-      const result = await routeQuery(query, context);
-      const predictedTier = result.model.startsWith('deepseek') ? 'free' :
-                           result.model.includes('ministral') ? 'mid' :
-                           result.model.includes('gemini') ? 'premium' : 'cheap';
-      
-      const isCorrect = predictedTier === actualTier;
-      if (isCorrect) correct++;
-      
-      // Record confusion
-      if (!confusion[actualTier][predictedTier]) {
-        confusion[actualTier][predictedTier] = 0;
-      }
-      confusion[actualTier][predictedTier]++;
-      
-      if (!isCorrect) {
-        failures.push({
-          actual: actualTier,
-          predicted: predictedTier,
-          query: query.substring(0, 200),
-          correct: false
-        });
-      }
-    } catch (e) {
-      console.error(`Error routing query: ${e}`);
-    }
-  }
-  
-  // Calculate tier accuracy
-  const tierAccuracy: { [key: string]: { correct: number; total: number; accuracy: number } } = {};
-  for (const tier of Object.keys(confusion)) {
-    const total = Object.values(confusion[tier]).reduce((a, b) => a + b, 0);
-    const tierCorrect = confusion[tier][tier] || 0;
-    tierAccuracy[tier] = {
-      correct: tierCorrect,
-      total,
-      accuracy: total > 0 ? tierCorrect / total : 0
-    };
-  }
-  
-  // Analyze failures
-  const failureAnalysis = analyzeFailures(failures);
-  
-  const result: BenchmarkResult = {
-    timestamp: new Date().toISOString(),
-    version: 'local-benchmark',
-    total: queries.length,
-    correct,
-    accuracy: correct / queries.length,
-    confusion,
-    failures: failures.slice(0, 50), // Keep top 50 failures
-    tierAccuracy,
-    failureAnalysis
-  };
-  
-  // Save results
-  fs.writeFileSync(RESULTS_PATH, JSON.stringify(result, null, 2));
-  
-  console.log(`\n=== Benchmark Results ===`);
-  console.log(`Total: ${result.total}, Correct: ${result.correct}`);
-  console.log(`Accuracy: ${(result.accuracy * 100).toFixed(1)}%`);
-  console.log(`\nTier Accuracy:`);
-  for (const [tier, stats] of Object.entries(result.tierAccuracy)) {
-    console.log(`  ${tier}: ${stats.correct}/${stats.total} = ${(stats.accuracy * 100).toFixed(0)}%`);
-  }
-  console.log(`\nTop Failures:`);
-  const sortedFailures = Object.entries(failureAnalysis).sort((a, b) => b[1] - a[1]);
-  for (const [key, count] of sortedFailures.slice(0, 10)) {
-    console.log(`  ${key}: ${count}`);
-  }
-  
-  return result;
-}
-
-runBenchmark().catch(console.error);

package/scripts/post-all.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# 🚀 A3M Router — One-command posting script
-# Set these env vars first:
-#   DEV_TO_API_KEY=your_key_from_dev.to/settings
-#   HN_USERNAME=your_hn_username
-#   HN_PASSWORD=your_hn_password
-#
-# Or just run the ones you have keys for.
-
-set -e
-DIR="$(cd "$(dirname "$0")/.." && pwd)"
-
-echo "=== 📺 Posting to dev.to ==="
-if [ -n "$DEV_TO_API_KEY" ]; then
-  BODY=$(cat "$DIR/articles/FRESH_devto_2026_05.md")
-  curl -s -X POST https://dev.to/api/articles \
-    -H "Content-Type: application/json" \
-    -H "Authorization: Bearer $DEV_TO_API_KEY" \
-    -d "$(python3 -c "
-import json
-with open('$DIR/articles/FRESH_devto_2026_05.md') as f:
-    body = f.read()
-print(json.dumps({
-    'article': {
-        'title': 'Three LLM Infrastructure Problems That Shouldn't Exist in 2026',
-        'body_markdown': body,
-        'tags': ['llm', 'opensource', 'typescript', 'ai', 'devops'],
-        'published': true,
-        'main_image': 'https://raw.githubusercontent.com/Das-rebel/a3m-router/main/docs/benchmark-chart.png'
-    }
-}))")" | python3 -c "import json,sys;d=json.load(sys.stdin);print('  Published:', d.get('url','FAILED'))"
-else
-  echo "  Skipping (set DEV_TO_API_KEY)"
-fi
-
-echo ""
-echo "=== 📺 Dev.to draft posted as private. Approve at dev.to/dashboard."
-echo "=== 🐙 HN: Post manually at https://news.ycombinator.com/submit"
-echo "=== 🔴 Reddit: Post manually at each subreddit"
-echo ""
-echo "Posting kit: $DIR/articles/POSTING_KIT_2026_05.md"

package/scripts/publish_fcc.py

@@ -1,106 +0,0 @@
-"""
-FreeCodeCamp - Publish existing draft for A3M Router article
-"""
-import sys, json, time, os
-sys.path.insert(0, os.path.expanduser("~/omniclaw/skills/browser/sota-browser"))
-from playwright.sync_api import sync_playwright, TimeoutError as PwTimeout
-
-PROFILE_PATH = "/var/folders/l9/4016_h694kdbx1wvdg5881bh0000gp/T/chrome_profile_100llyqq"
-FCC_EMAIL = "subho.matteragent@gmail.com"
-FCC_PASSWORD = "YourChance2025!"
-
-def main():
-    with sync_playwright() as p:
-        browser = p.chromium.launch_persistent_context(
-            PROFILE_PATH,
-            headless=False,
-            args=["--no-sandbox", "--disable-blink-features=AutomationControlled"]
-        )
-        page = browser.new_page()
-        
-        try:
-            # Go to FCC publishing settings
-            print("[FCC] Navigating to publishing settings...")
-            page.goto("https://www.freecodecamp.org/news/settings/publishing", timeout=30000)
-            time.sleep(3)
-            page.screenshot(path="/tmp/fcc_1_landing.png")
-            
-            # Check if we need to login
-            current_url = page.url
-            print(f"[FCC] Current URL: {current_url}")
-            
-            if "login" in current_url or "signin" in current_url or "email" in page.content().lower():
-                print("[FCC] Need to login...")
-                # Try clicking email/password
-                try:
-                    # Look for email field
-                    email_input = page.wait_for_selector('input[type="email"], input[name="email"], input[placeholder*="email" i]', timeout=5000)
-                    email_input.fill(FCC_EMAIL)
-                    time.sleep(1)
-                    
-                    # Look for password field
-                    password_input = page.wait_for_selector('input[type="password"], input[name="password"]', timeout=5000)
-                    password_input.fill(FCC_PASSWORD)
-                    time.sleep(1)
-                    
-                    # Find and click submit
-                    submit_btn = page.query_selector('button[type="submit"], button:has-text("Sign in"), button:has-text("Login")')
-                    if submit_btn:
-                        submit_btn.click()
-                        time.sleep(5)
-                    
-                    page.screenshot(path="/tmp/fcc_2_after_login.png")
-                except Exception as e:
-                    print(f"[FCC] Login interaction failed: {e}")
-                    page.screenshot(path="/tmp/fcc_2_login_error.png")
-            
-            # Now try to access publishing page
-            page.goto("https://www.freecodecamp.org/news/settings/publishing", timeout=30000)
-            time.sleep(3)
-            page.screenshot(path="/tmp/fcc_3_publishing.png")
-            
-            # Check for drafts
-            content = page.content()
-            if "draft" in content.lower():
-                print("[FCC] Found drafts section!")
-                
-                # Try to find and click on a draft
-                draft_links = page.query_selector_all('a:has-text("A3M"), a:has-text("Router"), a:has-text("LLM Infrastructure")')
-                if draft_links:
-                    print(f"[FCC] Found {len(draft_links)} matching draft links")
-                    draft_links[0].click()
-                    time.sleep(3)
-                    page.screenshot(path="/tmp/fcc_4_draft_editor.png")
-                    
-                    # Look for publish button
-                    publish_btn = page.query_selector('button:has-text("Publish"), button:has-text("Submit"), button:has-text("Post")')
-                    if publish_btn:
-                        print("[FCC] Found publish button, clicking...")
-                        publish_btn.click()
-                        time.sleep(3)
-                        page.screenshot(path="/tmp/fcc_5_after_publish.png")
-                        print("[FCC] Publishing attempted!")
-                    else:
-                        print("[FCC] No publish button found on draft page")
-                else:
-                    print("[FCC] No matching draft links found")
-                    # List all links for debugging
-                    all_links = page.query_selector_all('a[href*="draft"], a[href*="edit"]')
-                    for link in all_links:
-                        print(f"  Link: {link.inner_text()} -> {link.get_attribute('href')}")
-            else:
-                print("[FCC] No drafts section visible")
-                # Check what's on the page
-                print(f"[FCC] Page title: {page.title()}")
-            
-            # Save the screenshot for debugging
-            page.screenshot(path="/tmp/fcc_final.png")
-            
-        except Exception as e:
-            print(f"[FCC] Error: {e}")
-            page.screenshot(path="/tmp/fcc_error.png")
-        finally:
-            browser.close()
-
-if __name__ == "__main__":
-    main()

package/scripts/push-to-gitee.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-# Push to Gitee mirror for Chinese SEO
-# Usage: bash scripts/push-to-gitee.sh
-
-set -e
-
-GITEE_REPO="https://gitee.com/das-rebel/a3m-router.git"
-
-echo "🇨🇳 Pushing to Gitee mirror..."
-
-# Add gitee remote if not already added
-if ! git remote | grep -q gitee; then
-    git remote add gitee "$GITEE_REPO"
-fi
-
-# Push main branch
-git push gitee main --force 2>&1 || {
-    echo "❌ Push failed. You may need to:"
-    echo "  1. Create the repo on gitee.com first"
-    echo "  2. Or authenticate with: git config credential.helper store"
-    exit 1
-}
-
-echo "✅ Pushed to Gitee!"
-echo "  https://gitee.com/das-rebel/a3m-router"

package/scripts/routerarena_ensemble.js

@@ -1,144 +0,0 @@
-#!/usr/bin/env node
-/**
- * A3M Router — RouterArena Full Ensemble Evaluation
- * Runs ALL configured models on ALL RouterArena queries in parallel
- * Applies A3M confidence-weighted voting
- */
-
-const https = require('https');
-const http = require('http');
-const fs = require('fs');
-const path = require('path');
-
-// ======== Model Configuration ========
-const MODELS = [
-  { name: 'deepseek-chat', url: 'https://api.deepseek.com/chat/completions', 
-    key: process.env.DEEPSEEK_API_KEY, weight: 1.0 },
-  { name: 'meta/llama-3.3-70b-instruct', url: 'https://integrate.api.nvidia.com/v1/chat/completions',
-    key: process.env.NVIDIA_API_KEY, weight: 0.85 },
-  { name: 'mistralai/ministral-3-8b-2512', url: 'https://api.mistral.ai/v1/chat/completions',
-    key: process.env.MISTRAL_API_KEY, apiModel: 'ministral-8b-2512', weight: 0.7 },
-  { name: 'mistralai/ministral-3-14b-2512', url: 'https://api.mistral.ai/v1/chat/completions',
-    key: process.env.MISTRAL_API_KEY, apiModel: 'ministral-14b-2512', weight: 0.8 },
-  { name: 'nvidia/nemotron-3-super-120b-a12b', url: 'https://openrouter.ai/api/v1/chat/completions',
-    key: process.env.OPENROUTER_API_KEY, apiModel: 'nvidia/nemotron-3-super-120b-a12b:free', weight: 0.75 },
-  { name: 'google/gemma-4-26b-a4b-it', url: 'https://openrouter.ai/api/v1/chat/completions',
-    key: process.env.OPENROUTER_API_KEY, apiModel: 'google/gemma-4-26b-a4b-it:free', weight: 0.65 },
-];
-
-function callModel(model, prompt) {
-  return new Promise((resolve) => {
-    const apiModel = model.apiModel || model.name;
-    const body = JSON.stringify({
-      model: apiModel,
-      messages: [{ role: 'user', content: prompt }],
-      max_tokens: 200,
-      temperature: 0
-    });
-    
-    const url = new URL(model.url);
-    const req = (url.protocol === 'https:' ? https : http).request({
-      hostname: url.hostname, path: url.pathname, method: 'POST',
-      headers: { 'Authorization': `Bearer ${model.key}`, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
-      timeout: 30000
-    }, (res) => {
-      let data = '';
-      res.on('data', c => data += c);
-      res.on('end', () => {
-        try {
-          const d = JSON.parse(data);
-          if (d.choices?.[0]?.message?.content) {
-            resolve({ model: model.name, answer: d.choices[0].message.content, tokens: d.usage, success: true });
-          } else {
-            resolve({ model: model.name, success: false, error: (d.error?.message || 'unknown').substring(0,60) });
-          }
-        } catch(e) {
-          resolve({ model: model.name, success: false, error: 'parse error' });
-        }
-      });
-    });
-    req.on('error', e => resolve({ model: model.name, success: false, error: e.message.substring(0,60) }));
-    req.on('timeout', () => { req.destroy(); resolve({ model: model.name, success: false, error: 'timeout' }); });
-    req.write(body);
-    req.end();
-  });
-}
-
-function ensembleVote(results) {
-  const successful = results.filter(r => r.success);
-  if (successful.length === 0) return null;
-  
-  // Simple majority: if >50% agree on same answer, use it
-  const answers = successful.map(r => r.answer.trim().toLowerCase());
-  const counts = {};
-  answers.forEach(a => { counts[a] = (counts[a]||0) + 1; });
-  const sorted = Object.entries(counts).sort((a,b) => b[1] - a[1]);
-  
-  if (sorted[0][1] >= Math.ceil(successful.length / 2)) {
-    // Majority exists
-    for (const r of successful) {
-      if (r.answer.trim().toLowerCase() === sorted[0][0]) return r;
-    }
-  }
-  // Fallback: highest-weight model's answer
-  return successful[0];
-}
-
-async function main() {
-  const args = process.argv.slice(2);
-  const limit = parseInt(args[0]) || 100;
-  const predFile = args[1] || '../RouterArena/router_inference/predictions/a3m-router.json';
-  
-  console.log(`A3M Ensemble: ${MODELS.length} models on ${limit} RouterArena queries`);
-  
-  const preds = JSON.parse(fs.readFileSync(predFile, 'utf8'));
-  const regular = preds.filter(p => !p.for_optimality).slice(0, limit);
-  
-  const results = [];
-  const start = Date.now();
-  
-  for (let i = 0; i < regular.length; i++) {
-    const p = regular[i];
-    
-    // Call ALL models in parallel
-    const modelResults = await Promise.all(MODELS.map(m => callModel(m, p.prompt)));
-    
-    // Vote
-    const winner = ensembleVote(modelResults);
-    
-    const successCount = modelResults.filter(r => r.success).length;
-    const totalTokens = modelResults.reduce((s,r) => s + (r.tokens?.total_tokens || 0), 0);
-    
-    results.push({
-      index: i,
-      query: p.prompt?.substring(0, 80),
-      models_called: MODELS.length,
-      models_success: successCount,
-      ensemble_answer: winner?.answer?.substring(0, 100),
-      answers: Object.fromEntries(modelResults.map(r => [r.model, r.success ? '✅' : '❌'])),
-      total_tokens: totalTokens
-    });
-    
-    if ((i+1) % 10 === 0) {
-      const elapsed = ((Date.now() - start) / 1000).toFixed(0);
-      console.log(`[${i+1}/${limit}] ${elapsed}s | ${successCount}/${MODELS.length} models | ${results.filter(r=>r.ensemble_answer).length} voted`);
-    }
-    
-    // Small delay between queries
-    await new Promise(r => setTimeout(r, 200));
-  }
-  
-  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-  const withAnswer = results.filter(r => r.ensemble_answer).length;
-  
-  console.log(`\n=== A3M Ensemble Summary ===`);
-  console.log(`Queries: ${results.length}`);
-  console.log(`Models/query: ${MODELS.length}`);
-  console.log(`Ensemble answers: ${withAnswer}/${results.length}`);
-  console.log(`Time: ${elapsed}s (${(elapsed/results.length).toFixed(1)}s/query)`);
-  
-  fs.writeFileSync('ensemble_results.json', JSON.stringify(results, null, 2));
-  console.log(`Saved: ensemble_results.json`);
-}
-
-main().catch(e => { console.error(e); process.exit(1); });