adaptive-memory-multi-model-router 2.11.0 → 2.12.0

package/eval/run_eval.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line, idx) => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {
+        throw new Error(`Invalid JSONL at line ${idx + 1}: ${error.message}`);
+      }
+    });
+}
+
+function safeGet(obj, key, fallback = false) {
+  return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(4));
+}
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function evaluateCase(item) {
+  const decision = routeQuery(item.prompt);
+  const expected = item.expected || {};
+  const checks = [];
+
+  if (expected.complexity) {
+    const c = decision.features?.complexity ?? 0;
+    const ok = c >= expected.complexity.min && c <= expected.complexity.max;
+    checks.push({ type: 'complexity', ok, actual: c, expected: expected.complexity });
+  }
+
+  if (expected.flags) {
+    for (const [flag, expectedValue] of Object.entries(expected.flags)) {
+      const actual = safeGet(decision.features || {}, flag, false);
+      checks.push({ type: 'flag', flag, ok: actual === expectedValue, actual, expected: expectedValue });
+    }
+  }
+
+  if (expected.domain) {
+    const actual = decision.features?.detected_domain || '';
+    checks.push({ type: 'domain', ok: actual === expected.domain, actual, expected: expected.domain });
+  }
+
+  if (expected.provider_type) {
+    const actual = decision.provider_type || '';
+    checks.push({ type: 'provider_type', ok: actual === expected.provider_type, actual, expected: expected.provider_type });
+  }
+
+  return {
+    id: item.id,
+    prompt: item.prompt,
+    decision: {
+      primary_model: decision.primary_model,
+      provider_type: decision.provider_type,
+      estimated_cost: decision.estimated_cost,
+      complexity: decision.features?.complexity,
+      detected_domain: decision.features?.detected_domain || ''
+    },
+    checks
+  };
+}
+
+function summarize(results) {
+  const allChecks = results.flatMap((r) => r.checks);
+  const byType = (type) => allChecks.filter((c) => c.type === type);
+  const rate = (arr) => (arr.length ? arr.filter((x) => x.ok).length / arr.length : 1);
+
+  const complexity = rate(byType('complexity'));
+  const flags = rate(byType('flag'));
+  const domain = rate(byType('domain'));
+  const providerType = rate(byType('provider_type'));
+
+  const weighted = [complexity, flags, domain, providerType];
+  const overall = weighted.reduce((a, b) => a + b, 0) / weighted.length;
+
+  return {
+    dataset_size: results.length,
+    checks_count: allChecks.length,
+    complexity_accuracy: toFixed(complexity),
+    flag_accuracy: toFixed(flags),
+    domain_accuracy: toFixed(domain),
+    provider_type_accuracy: toFixed(providerType),
+    overall_score: toFixed(overall)
+  };
+}
+
+function gate(summary, thresholds, baseline) {
+  const failures = [];
+
+  if (summary.dataset_size < thresholds.min_dataset_size) {
+    failures.push(`dataset_size ${summary.dataset_size} < min_dataset_size ${thresholds.min_dataset_size}`);
+  }
+  if (summary.complexity_accuracy < thresholds.min_complexity_accuracy) {
+    failures.push(`complexity_accuracy ${summary.complexity_accuracy} < ${thresholds.min_complexity_accuracy}`);
+  }
+  if (summary.flag_accuracy < thresholds.min_flag_accuracy) {
+    failures.push(`flag_accuracy ${summary.flag_accuracy} < ${thresholds.min_flag_accuracy}`);
+  }
+  if (summary.domain_accuracy < thresholds.min_domain_accuracy) {
+    failures.push(`domain_accuracy ${summary.domain_accuracy} < ${thresholds.min_domain_accuracy}`);
+  }
+  if (summary.provider_type_accuracy < thresholds.min_provider_type_accuracy) {
+    failures.push(`provider_type_accuracy ${summary.provider_type_accuracy} < ${thresholds.min_provider_type_accuracy}`);
+  }
+  if (summary.overall_score < thresholds.min_overall_score) {
+    failures.push(`overall_score ${summary.overall_score} < ${thresholds.min_overall_score}`);
+  }
+
+  if (baseline?.summary) {
+    const delta = baseline.summary.overall_score - summary.overall_score;
+    if (delta > thresholds.max_regression_delta) {
+      failures.push(
+        `overall_score regression ${toFixed(delta)} > max_regression_delta ${thresholds.max_regression_delta}`
+      );
+    }
+  }
+
+  return failures;
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const datasetPath = path.join(evalDir, 'benchmark_dataset.jsonl');
+  const thresholdsPath = path.join(evalDir, 'thresholds.json');
+  const baselinePath = path.join(evalDir, 'baselines', 'main.json');
+  const resultsPath = path.join(evalDir, 'results', 'latest.json');
+
+  const dataset = readJsonl(datasetPath);
+  const thresholds = readJson(thresholdsPath);
+  const results = dataset.map(evaluateCase);
+  const summary = summarize(results);
+  const baseline = fs.existsSync(baselinePath) ? readJson(baselinePath) : null;
+  const failures = gate(summary, thresholds, baseline);
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    commit: process.env.GITHUB_SHA || null,
+    summary,
+    failures,
+    baseline_used: Boolean(baseline),
+    results
+  };
+
+  ensureDir(path.dirname(resultsPath));
+  fs.writeFileSync(resultsPath, JSON.stringify(output, null, 2));
+
+  console.log('\nA3M Routing Eval Summary');
+  console.log('------------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Results file: ${resultsPath}`);
+
+  if (failures.length) {
+    appendExperimentRecord({
+      experiment_id: `routing_eval_${Date.now()}`,
+      dataset_version: 'core_regression_v1',
+      run_type: 'routing_eval',
+      metrics: summary,
+      decision: 'fail',
+      notes: failures
+    });
+    console.error('\nEval gate FAILED:');
+    for (const failure of failures) {
+      console.error(`- ${failure}`);
+    }
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `routing_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'routing_eval',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nEval gate PASSED');
+}
+
+main();

package/eval/run_fault_injection.js

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { ProviderRetryHandler } = require('../dist/routing/providerRetry.js');
+const { ProviderHealthManager } = require('../dist/routing/providerHealth.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+async function scenarioRetryTransientThenSuccess() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 3,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  const result = await handler.executeWithRetry('openai', async () => {
+    callCount += 1;
+    if (callCount < 3) {
+      const err = new Error('temporary timeout');
+      err.code = 'ETIMEDOUT';
+      throw err;
+    }
+    return 'ok';
+  });
+
+  return result === 'ok' && callCount === 3;
+}
+
+async function scenarioNoRetryOnBadRequest() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('openai', async () => {
+      callCount += 1;
+      const err = new Error('bad request');
+      err.status = 400;
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioNoRetryOnChineseQuotaAccountErrors() {
+  const handler = new ProviderRetryHandler({
+    moonshot: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['429', '503', 'ETIMEDOUT']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('moonshot', async () => {
+      callCount += 1;
+      const err = new Error('request reached organization TPD rate limit, current: 1501880, limit: 1500000');
+      err.status = 429;
+      err.code = 'rate_limit_reached_error';
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioCircuitBreakerOpens() {
+  const health = new ProviderHealthManager({
+    circuitBreakerThreshold: 3,
+    cooldownMs: 30000,
+    windowSize: 10
+  });
+
+  const provider = 'mock/provider';
+  health.recordFailure(provider, 'e1');
+  health.recordFailure(provider, 'e2');
+  health.recordFailure(provider, 'e3');
+
+  const state = health.getHealth(provider);
+  return state.isHealthy === false && state.consecutiveErrors >= 3 && state.cooldownUntil > Date.now();
+}
+
+async function scenarioFallbackOrdering() {
+  const health = new ProviderHealthManager({ windowSize: 10 });
+
+  health.recordSuccess('fast', 80);
+  health.recordSuccess('fast', 90);
+
+  health.recordSuccess('slow', 500);
+  health.recordSuccess('slow', 600);
+
+  health.recordFailure('broken', 'x1');
+  health.recordFailure('broken', 'x2');
+  health.recordFailure('broken', 'x3');
+
+  const chain = health.getFallbackChain(['broken', 'slow', 'fast']);
+  return chain[0] === 'fast' && chain[chain.length - 1] === 'broken';
+}
+
+async function main() {
+  const thresholds = readJson(path.join(__dirname, 'fault_injection_thresholds.json'));
+  const scenarios = [
+    ['retry_transient_then_success', scenarioRetryTransientThenSuccess],
+    ['no_retry_on_bad_request', scenarioNoRetryOnBadRequest],
+    ['no_retry_on_chinese_quota_account_errors', scenarioNoRetryOnChineseQuotaAccountErrors],
+    ['circuit_breaker_opens', scenarioCircuitBreakerOpens],
+    ['fallback_ordering', scenarioFallbackOrdering]
+  ];
+
+  const results = [];
+  for (const [name, fn] of scenarios) {
+    try {
+      const ok = await fn();
+      results.push({ name, ok, error: null });
+    } catch (error) {
+      results.push({ name, ok: false, error: String(error) });
+    }
+  }
+
+  const passed = results.filter((r) => r.ok).length;
+  const passRate = results.length ? passed / results.length : 0;
+  const summary = {
+    total: results.length,
+    passed,
+    failed: results.length - passed,
+    pass_rate: Number(passRate.toFixed(4))
+  };
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    thresholds,
+    results
+  };
+
+  const outDir = path.join(__dirname, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(path.join(outDir, 'fault_injection_latest.json'), JSON.stringify(output, null, 2));
+
+  console.log('\nFault Injection Summary');
+  console.log('-----------------------');
+  console.log(JSON.stringify(summary, null, 2));
+
+  if (passRate < thresholds.required_pass_rate) {
+    appendExperimentRecord({
+      experiment_id: `fault_injection_${Date.now()}`,
+      dataset_version: 'fault_scenarios_v1',
+      run_type: 'fault_injection',
+      metrics: summary,
+      decision: 'fail',
+      notes: results.filter((r) => !r.ok).map((r) => `${r.name}: ${r.error || 'failed'}`)
+    });
+    console.error(
+      `\nFault injection gate FAILED: pass_rate ${summary.pass_rate} < ${thresholds.required_pass_rate}`
+    );
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `fault_injection_${Date.now()}`,
+    dataset_version: 'fault_scenarios_v1',
+    run_type: 'fault_injection',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nFault injection gate PASSED');
+}
+
+main();

package/eval/run_shadow_eval.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(6));
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const dataset = readJsonl(path.join(evalDir, 'benchmark_dataset.jsonl'));
+  const candidateBudgetMultiplier = Number(process.env.A3M_SHADOW_BUDGET_MULTIPLIER || '0.85');
+
+  let divergence = 0;
+  let projectedCostDelta = 0;
+  const comparisons = [];
+
+  for (const row of dataset) {
+    const primary = routeQuery(row.prompt);
+    const shadow = routeQuery(row.prompt, undefined, candidateBudgetMultiplier);
+
+    const changed = primary.primary_model !== shadow.primary_model;
+    if (changed) divergence += 1;
+
+    const costDelta = Number((shadow.estimated_cost || 0) - (primary.estimated_cost || 0));
+    projectedCostDelta += costDelta;
+
+    comparisons.push({
+      id: row.id,
+      primary_model: primary.primary_model,
+      shadow_model: shadow.primary_model,
+      changed,
+      primary_cost: primary.estimated_cost,
+      shadow_cost: shadow.estimated_cost,
+      cost_delta: toFixed(costDelta)
+    });
+  }
+
+  const summary = {
+    dataset_size: dataset.length,
+    candidate_budget_multiplier: candidateBudgetMultiplier,
+    divergence_rate: toFixed(divergence / dataset.length),
+    changed_cases: divergence,
+    projected_total_cost_delta: toFixed(projectedCostDelta),
+    projected_avg_cost_delta: toFixed(projectedCostDelta / dataset.length)
+  };
+
+  const out = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    comparisons
+  };
+
+  const outDir = path.join(evalDir, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  const outPath = path.join(outDir, 'shadow_latest.json');
+  fs.writeFileSync(outPath, JSON.stringify(out, null, 2));
+
+  appendExperimentRecord({
+    experiment_id: `shadow_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'shadow_eval',
+    metrics: summary,
+    decision: 'informational',
+    notes: []
+  });
+
+  console.log('\nShadow Eval Summary');
+  console.log('-------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Shadow output: ${outPath}`);
+}
+
+main();

package/eval/thresholds.json

@@ -0,0 +1,9 @@
+{
+  "min_dataset_size": 12,
+  "min_complexity_accuracy": 0.85,
+  "min_flag_accuracy": 0.8,
+  "min_domain_accuracy": 0.8,
+  "min_provider_type_accuracy": 0.75,
+  "min_overall_score": 0.85,
+  "max_regression_delta": 0.03
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "adaptive-memory-multi-model-router",
-  "version": "2.11.0",
+  "version": "2.12.0",
   "shortName": "A3M Router",
   "displayName": "A3M Router - Adaptive Memory Multi-Model Router",
   "description": "LLM router & AI gateway — 99.5% routing accuracy, 47 providers (DeepSeek, Kimi/Moonshot, Qwen, Zhipu GLM, Yi + more). Semantic cache, guardrails, cost analytics. Built on 30+ arXiv papers (SGLang, Medusa, MemoRAG). Zero ML, 19.5KB. TypeScript + Python SDK. MIT.",
@@ -729,6 +729,14 @@
   "homepage": "https://das-rebel.github.io/adaptive-memory-multi-model-router/",
   "scripts": {
     "test": "node test.js && node test/provider-test.js",
+    "test:py": "python3 -m pytest -q",
+    "test:all": "npm test && npm run test:py",
+    "eval:routing": "node eval/run_eval.js",
+    "eval:golden": "node eval/check_golden_routes.js",
+    "eval:faults": "node eval/run_fault_injection.js",
+    "eval:shadow": "node eval/run_shadow_eval.js",
+    "eval:report": "node eval/generate_report.js",
+    "eval:all": "npm run eval:routing && npm run eval:golden && npm run eval:faults && npm run eval:shadow && npm run eval:report",
     "test:providers": "node test/provider-test.js",
     "benchmark": "node test/benchmark.js",
     "benchmark:verbose": "node test/benchmark.js --verbose",

package/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+asyncio_mode = auto