adaptive-memory-multi-model-router 2.11.0 → 2.12.1

package/eval/run_fault_injection.js

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { ProviderRetryHandler } = require('../dist/routing/providerRetry.js');
+const { ProviderHealthManager } = require('../dist/routing/providerHealth.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+async function scenarioRetryTransientThenSuccess() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 3,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  const result = await handler.executeWithRetry('openai', async () => {
+    callCount += 1;
+    if (callCount < 3) {
+      const err = new Error('temporary timeout');
+      err.code = 'ETIMEDOUT';
+      throw err;
+    }
+    return 'ok';
+  });
+
+  return result === 'ok' && callCount === 3;
+}
+
+async function scenarioNoRetryOnBadRequest() {
+  const handler = new ProviderRetryHandler({
+    openai: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['ETIMEDOUT', '503']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('openai', async () => {
+      callCount += 1;
+      const err = new Error('bad request');
+      err.status = 400;
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioNoRetryOnChineseQuotaAccountErrors() {
+  const handler = new ProviderRetryHandler({
+    moonshot: {
+      timeout: 3000,
+      retry: {
+        maxRetries: 5,
+        initialDelayMs: 10,
+        maxDelayMs: 100,
+        backoffMultiplier: 2,
+        retryableErrors: ['429', '503', 'ETIMEDOUT']
+      }
+    }
+  });
+
+  let callCount = 0;
+  try {
+    await handler.executeWithRetry('moonshot', async () => {
+      callCount += 1;
+      const err = new Error('request reached organization TPD rate limit, current: 1501880, limit: 1500000');
+      err.status = 429;
+      err.code = 'rate_limit_reached_error';
+      throw err;
+    });
+    return false;
+  } catch {
+    return callCount === 1;
+  }
+}
+
+async function scenarioCircuitBreakerOpens() {
+  const health = new ProviderHealthManager({
+    circuitBreakerThreshold: 3,
+    cooldownMs: 30000,
+    windowSize: 10
+  });
+
+  const provider = 'mock/provider';
+  health.recordFailure(provider, 'e1');
+  health.recordFailure(provider, 'e2');
+  health.recordFailure(provider, 'e3');
+
+  const state = health.getHealth(provider);
+  return state.isHealthy === false && state.consecutiveErrors >= 3 && state.cooldownUntil > Date.now();
+}
+
+async function scenarioFallbackOrdering() {
+  const health = new ProviderHealthManager({ windowSize: 10 });
+
+  health.recordSuccess('fast', 80);
+  health.recordSuccess('fast', 90);
+
+  health.recordSuccess('slow', 500);
+  health.recordSuccess('slow', 600);
+
+  health.recordFailure('broken', 'x1');
+  health.recordFailure('broken', 'x2');
+  health.recordFailure('broken', 'x3');
+
+  const chain = health.getFallbackChain(['broken', 'slow', 'fast']);
+  return chain[0] === 'fast' && chain[chain.length - 1] === 'broken';
+}
+
+async function main() {
+  const thresholds = readJson(path.join(__dirname, 'fault_injection_thresholds.json'));
+  const scenarios = [
+    ['retry_transient_then_success', scenarioRetryTransientThenSuccess],
+    ['no_retry_on_bad_request', scenarioNoRetryOnBadRequest],
+    ['no_retry_on_chinese_quota_account_errors', scenarioNoRetryOnChineseQuotaAccountErrors],
+    ['circuit_breaker_opens', scenarioCircuitBreakerOpens],
+    ['fallback_ordering', scenarioFallbackOrdering]
+  ];
+
+  const results = [];
+  for (const [name, fn] of scenarios) {
+    try {
+      const ok = await fn();
+      results.push({ name, ok, error: null });
+    } catch (error) {
+      results.push({ name, ok: false, error: String(error) });
+    }
+  }
+
+  const passed = results.filter((r) => r.ok).length;
+  const passRate = results.length ? passed / results.length : 0;
+  const summary = {
+    total: results.length,
+    passed,
+    failed: results.length - passed,
+    pass_rate: Number(passRate.toFixed(4))
+  };
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    thresholds,
+    results
+  };
+
+  const outDir = path.join(__dirname, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  fs.writeFileSync(path.join(outDir, 'fault_injection_latest.json'), JSON.stringify(output, null, 2));
+
+  console.log('\nFault Injection Summary');
+  console.log('-----------------------');
+  console.log(JSON.stringify(summary, null, 2));
+
+  if (passRate < thresholds.required_pass_rate) {
+    appendExperimentRecord({
+      experiment_id: `fault_injection_${Date.now()}`,
+      dataset_version: 'fault_scenarios_v1',
+      run_type: 'fault_injection',
+      metrics: summary,
+      decision: 'fail',
+      notes: results.filter((r) => !r.ok).map((r) => `${r.name}: ${r.error || 'failed'}`)
+    });
+    console.error(
+      `\nFault injection gate FAILED: pass_rate ${summary.pass_rate} < ${thresholds.required_pass_rate}`
+    );
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `fault_injection_${Date.now()}`,
+    dataset_version: 'fault_scenarios_v1',
+    run_type: 'fault_injection',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nFault injection gate PASSED');
+}
+
+main();

package/eval/run_shadow_eval.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(6));
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const dataset = readJsonl(path.join(evalDir, 'benchmark_dataset.jsonl'));
+  const candidateBudgetMultiplier = Number(process.env.A3M_SHADOW_BUDGET_MULTIPLIER || '0.85');
+
+  let divergence = 0;
+  let projectedCostDelta = 0;
+  const comparisons = [];
+
+  for (const row of dataset) {
+    const primary = routeQuery(row.prompt);
+    const shadow = routeQuery(row.prompt, undefined, candidateBudgetMultiplier);
+
+    const changed = primary.primary_model !== shadow.primary_model;
+    if (changed) divergence += 1;
+
+    const costDelta = Number((shadow.estimated_cost || 0) - (primary.estimated_cost || 0));
+    projectedCostDelta += costDelta;
+
+    comparisons.push({
+      id: row.id,
+      primary_model: primary.primary_model,
+      shadow_model: shadow.primary_model,
+      changed,
+      primary_cost: primary.estimated_cost,
+      shadow_cost: shadow.estimated_cost,
+      cost_delta: toFixed(costDelta)
+    });
+  }
+
+  const summary = {
+    dataset_size: dataset.length,
+    candidate_budget_multiplier: candidateBudgetMultiplier,
+    divergence_rate: toFixed(divergence / dataset.length),
+    changed_cases: divergence,
+    projected_total_cost_delta: toFixed(projectedCostDelta),
+    projected_avg_cost_delta: toFixed(projectedCostDelta / dataset.length)
+  };
+
+  const out = {
+    timestamp_utc: new Date().toISOString(),
+    summary,
+    comparisons
+  };
+
+  const outDir = path.join(evalDir, 'results');
+  fs.mkdirSync(outDir, { recursive: true });
+  const outPath = path.join(outDir, 'shadow_latest.json');
+  fs.writeFileSync(outPath, JSON.stringify(out, null, 2));
+
+  appendExperimentRecord({
+    experiment_id: `shadow_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'shadow_eval',
+    metrics: summary,
+    decision: 'informational',
+    notes: []
+  });
+
+  console.log('\nShadow Eval Summary');
+  console.log('-------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Shadow output: ${outPath}`);
+}
+
+main();

package/eval/thresholds.json

@@ -0,0 +1,9 @@
+{
+  "min_dataset_size": 12,
+  "min_complexity_accuracy": 0.85,
+  "min_flag_accuracy": 0.8,
+  "min_domain_accuracy": 0.8,
+  "min_provider_type_accuracy": 0.75,
+  "min_overall_score": 0.85,
+  "max_regression_delta": 0.03
+}