adaptive-memory-multi-model-router 2.11.0 → 2.12.1

package/eval/README.md

@@ -0,0 +1,46 @@
+# Routing Evaluation Harness
+
+This directory contains the reproducible evaluation system for A3M.
+
+## Files
+
+- `benchmark_dataset.jsonl`: frozen routing benchmark dataset
+- `datasets/catalog.json`: dataset registry and slice metadata
+- `datasets/slices/*.jsonl`: versioned slice datasets
+- `thresholds.json`: minimum quality thresholds + max allowed regression
+- `fault_injection_thresholds.json`: reliability gate threshold
+- `golden_routes.json`: golden route snapshot
+- `baselines/main.json`: baseline summary for `main` branch
+- `run_eval.js`: routing evaluator + hard gate
+- `check_golden_routes.js`: golden regression check
+- `run_fault_injection.js`: retry/health fault scenarios
+- `run_shadow_eval.js`: shadow routing comparison (informational)
+- `generate_report.js`: markdown summary from eval result artifacts
+- `experiments.jsonl`: append-only experiment registry (local artifact)
+- `results/*.json`: generated run outputs (not committed)
+- `results/report_latest.md`: generated markdown evidence report
+
+## Run
+
+```bash
+npm run eval:all
+```
+
+`eval:all` includes:
+
+1. `eval:routing` — hard gate for routing quality thresholds
+2. `eval:golden` — snapshot consistency gate
+3. `eval:faults` — reliability fault injection gate
+4. `eval:shadow` — candidate-vs-primary divergence and projected cost delta (informational)
+5. `eval:report` — consolidated markdown release summary
+
+## Updating Baseline
+
+Only update `baselines/main.json` when routing behavior changes intentionally.
+
+Suggested process:
+
+1. Run `npm run eval:routing`
+2. Review `eval/results/latest.json`
+3. If changes are expected and desired, copy the new summary into `baselines/main.json`
+4. Mention the reason in your PR/commit message

package/eval/baselines/main.json

@@ -0,0 +1,12 @@
+{
+  "summary": {
+    "dataset_size": 16,
+    "checks_count": 34,
+    "complexity_accuracy": 1,
+    "flag_accuracy": 1,
+    "domain_accuracy": 1,
+    "provider_type_accuracy": 1,
+    "overall_score": 1
+  },
+  "note": "Initial baseline. Regenerate deliberately when routing behavior is intentionally changed."
+}

package/eval/benchmark_dataset.jsonl

@@ -0,0 +1,16 @@
+{"id":"q01","prompt":"What is 2+2?","expected":{"complexity":{"min":0.0,"max":0.2}}}
+{"id":"q02","prompt":"Write a Python function to reverse a linked list.","expected":{"complexity":{"min":0.15,"max":0.35},"flags":{"has_code":true}}}
+{"id":"q03","prompt":"Translate this sentence to Japanese: Good morning","expected":{"complexity":{"min":0.1,"max":0.25},"flags":{"is_translation":true}}}
+{"id":"q04","prompt":"Design a clinical trial protocol for oncology treatment","expected":{"complexity":{"min":0.9,"max":1.0},"domain":"medical","provider_type":"local"}}
+{"id":"q05","prompt":"Explain why the sky is blue in simple terms.","expected":{"complexity":{"min":0.25,"max":0.45}}}
+{"id":"q06","prompt":"Find SQL injection risks in this login query.","expected":{"complexity":{"min":0.3,"max":0.55},"flags":{"is_security":true}}}
+{"id":"q07","prompt":"Solve the integral of x^2 from 0 to 3.","expected":{"complexity":{"min":0.2,"max":0.4},"flags":{"has_math":true}}}
+{"id":"q08","prompt":"Write a haiku about debugging.","expected":{"complexity":{"min":0.1,"max":0.3},"flags":{"is_creative":true}}}
+{"id":"q09","prompt":"Create a Kubernetes deployment YAML with autoscaling HPA.","expected":{"complexity":{"min":0.45,"max":0.65},"flags":{"is_devops":true},"domain":"architecture"}}
+{"id":"q10","prompt":"Compare two investment portfolios with Sharpe ratio.","expected":{"complexity":{"min":0.5,"max":0.7},"domain":"finance","provider_type":"local"}}
+{"id":"q11","prompt":"Draft a GDPR-compliant data retention policy for a healthcare app.","expected":{"complexity":{"min":0.25,"max":0.45},"domain":"legal"}}
+{"id":"q12","prompt":"Translate the following legal clause to Spanish and keep legal meaning precise.","expected":{"complexity":{"min":0.2,"max":0.4},"flags":{"is_translation":true},"domain":"legal"}}
+{"id":"q13","prompt":"Debug this JavaScript error: Cannot read properties of undefined.","expected":{"complexity":{"min":0.2,"max":0.45},"flags":{"has_code":true}}}
+{"id":"q14","prompt":"Summarize this paper abstract in 3 bullet points.","expected":{"complexity":{"min":0.35,"max":0.5},"domain":"ml_research"}}
+{"id":"q15","prompt":"Plan a zero-downtime migration strategy for PostgreSQL.","expected":{"complexity":{"min":0.2,"max":0.45}}}
+{"id":"q16","prompt":"Perform differential diagnosis for chest pain symptoms.","expected":{"complexity":{"min":0.3,"max":0.5},"domain":"medical","provider_type":"local"}}

package/eval/check_golden_routes.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function readJsonl(file) {
+  return fs
+    .readFileSync(file, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+}
+
+function main() {
+  const evalDir = __dirname;
+  const dataset = readJsonl(path.join(evalDir, 'benchmark_dataset.jsonl'));
+  const golden = readJson(path.join(evalDir, 'golden_routes.json'));
+  const goldenMap = new Map(golden.map((g) => [g.id, g]));
+
+  const failures = [];
+
+  for (const row of dataset) {
+    const decision = routeQuery(row.prompt);
+    const expected = goldenMap.get(row.id);
+    if (!expected) {
+      failures.push(`${row.id}: missing expected golden row`);
+      continue;
+    }
+
+    const checks = [
+      ['primary_model', decision.primary_model, expected.primary_model],
+      ['provider_type', decision.provider_type, expected.provider_type],
+      ['detected_domain', decision.features?.detected_domain || '', expected.detected_domain || '']
+    ];
+
+    for (const [name, actual, exp] of checks) {
+      if (actual !== exp) {
+        failures.push(`${row.id}: ${name} mismatch (actual=${actual}, expected=${exp})`);
+      }
+    }
+
+    const actualComplexity = decision.features?.complexity ?? 0;
+    if (Math.abs(actualComplexity - expected.complexity) > 1e-9) {
+      failures.push(
+        `${row.id}: complexity mismatch (actual=${actualComplexity}, expected=${expected.complexity})`
+      );
+    }
+  }
+
+  if (failures.length) {
+    console.error('\nGolden route check FAILED:');
+    failures.forEach((f) => console.error(`- ${f}`));
+    process.exit(1);
+  }
+
+  console.log(`Golden route check PASSED (${dataset.length} cases)`);
+}
+
+main();

package/eval/datasets/catalog.json

@@ -0,0 +1,33 @@
+{
+  "version": "v1",
+  "datasets": [
+    {
+      "name": "core_regression",
+      "path": "benchmark_dataset.jsonl",
+      "description": "Stable routing regression cases used for hard gates.",
+      "owner": "a3m-core",
+      "last_updated": "2026-05-23"
+    },
+    {
+      "name": "cn_provider_reliability",
+      "path": "slices/cn_provider_reliability_v1.jsonl",
+      "description": "Chinese provider policy/reliability stress cases for eval expansion.",
+      "owner": "a3m-reliability",
+      "last_updated": "2026-05-23"
+    },
+    {
+      "name": "cost_pressure",
+      "path": "slices/cost_pressure_v1.jsonl",
+      "description": "Cost-sensitive routing cases under budget pressure assumptions.",
+      "owner": "a3m-cost",
+      "last_updated": "2026-05-23"
+    },
+    {
+      "name": "safety_guardrails",
+      "path": "slices/safety_guardrails_v1.jsonl",
+      "description": "Security and prompt-risk routing cases for policy review.",
+      "owner": "a3m-security",
+      "last_updated": "2026-05-23"
+    }
+  ]
+}

package/eval/datasets/slices/cn_provider_reliability_v1.jsonl

@@ -0,0 +1,3 @@
+{"id":"cn01","prompt":"Translate a compliance notice to Chinese and preserve legal accuracy.","expected":{"domain":"legal","flags":{"is_translation":true}}}
+{"id":"cn02","prompt":"Summarize this medical advisory for mainland patients in simplified Chinese.","expected":{"domain":"medical"}}
+{"id":"cn03","prompt":"Explain why my API call returned rate_limit_reached_error and how to backoff.","expected":{"flags":{"is_devops":true}}}

package/eval/datasets/slices/cost_pressure_v1.jsonl

@@ -0,0 +1,3 @@
+{"id":"cp01","prompt":"Give me a one sentence explanation of gradient descent.","expected":{"complexity":{"min":0.0,"max":0.3}}}
+{"id":"cp02","prompt":"Compare two cloud cost reports and suggest top 3 savings actions.","expected":{"complexity":{"min":0.3,"max":0.7}}}
+{"id":"cp03","prompt":"Design a cross-region failover architecture with cost constraints.","expected":{"domain":"architecture"}}

package/eval/datasets/slices/safety_guardrails_v1.jsonl

@@ -0,0 +1,3 @@
+{"id":"sg01","prompt":"Find SQL injection vectors in this login endpoint and propose fixes.","expected":{"flags":{"is_security":true}}}
+{"id":"sg02","prompt":"User asks to bypass a firewall policy. Provide safe refusal guidance.","expected":{"flags":{"is_security":true}}}
+{"id":"sg03","prompt":"Review this GDPR data retention flow for policy risk.","expected":{"domain":"legal"}}

package/eval/fault_injection_thresholds.json

@@ -0,0 +1,3 @@
+{
+  "required_pass_rate": 1.0
+}

package/eval/generate_report.js

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+
+function readJsonIfExists(filePath) {
+  if (!fs.existsSync(filePath)) return null;
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function writeFile(filePath, content) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, content, 'utf8');
+}
+
+function fmtPct(n) {
+  return `${(n * 100).toFixed(2)}%`;
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const resultsDir = path.join(evalDir, 'results');
+
+  const routing = readJsonIfExists(path.join(resultsDir, 'latest.json'));
+  const faults = readJsonIfExists(path.join(resultsDir, 'fault_injection_latest.json'));
+  const shadow = readJsonIfExists(path.join(resultsDir, 'shadow_latest.json'));
+
+  const now = new Date().toISOString();
+  const lines = [];
+  lines.push('# A3M Eval Report');
+  lines.push('');
+  lines.push(`Generated: ${now}`);
+  lines.push(`Commit: ${process.env.GITHUB_SHA || 'local'}`);
+  lines.push('');
+
+  lines.push('## Gate Status');
+  lines.push('');
+
+  if (routing) {
+    const gate = routing.failures?.length ? 'FAIL' : 'PASS';
+    lines.push(`- Routing Eval: **${gate}**`);
+  } else {
+    lines.push('- Routing Eval: **MISSING**');
+  }
+
+  if (faults) {
+    const passRate = faults.summary?.pass_rate ?? 0;
+    const threshold = faults.thresholds?.required_pass_rate ?? 1;
+    const gate = passRate >= threshold ? 'PASS' : 'FAIL';
+    lines.push(`- Fault Injection: **${gate}**`);
+  } else {
+    lines.push('- Fault Injection: **MISSING**');
+  }
+
+  if (shadow) {
+    lines.push('- Shadow Eval: **INFO**');
+  } else {
+    lines.push('- Shadow Eval: **MISSING**');
+  }
+
+  lines.push('');
+  lines.push('## Routing Metrics');
+  lines.push('');
+
+  if (routing?.summary) {
+    const s = routing.summary;
+    lines.push(`- Dataset size: ${s.dataset_size}`);
+    lines.push(`- Checks count: ${s.checks_count}`);
+    lines.push(`- Complexity accuracy: ${fmtPct(s.complexity_accuracy)}`);
+    lines.push(`- Flag accuracy: ${fmtPct(s.flag_accuracy)}`);
+    lines.push(`- Domain accuracy: ${fmtPct(s.domain_accuracy)}`);
+    lines.push(`- Provider type accuracy: ${fmtPct(s.provider_type_accuracy)}`);
+    lines.push(`- Overall score: ${fmtPct(s.overall_score)}`);
+    if (routing.failures?.length) {
+      lines.push('- Failures:');
+      for (const f of routing.failures) lines.push(`  - ${f}`);
+    }
+  } else {
+    lines.push('- No routing results available.');
+  }
+
+  lines.push('');
+  lines.push('## Fault Injection');
+  lines.push('');
+
+  if (faults?.summary) {
+    lines.push(`- Total scenarios: ${faults.summary.total}`);
+    lines.push(`- Passed: ${faults.summary.passed}`);
+    lines.push(`- Failed: ${faults.summary.failed}`);
+    lines.push(`- Pass rate: ${fmtPct(faults.summary.pass_rate)}`);
+    const failed = (faults.results || []).filter((r) => !r.ok);
+    if (failed.length) {
+      lines.push('- Failed scenarios:');
+      for (const f of failed) lines.push(`  - ${f.name}: ${f.error || 'failed'}`);
+    }
+  } else {
+    lines.push('- No fault injection results available.');
+  }
+
+  lines.push('');
+  lines.push('## Shadow Eval');
+  lines.push('');
+
+  if (shadow?.summary) {
+    const s = shadow.summary;
+    lines.push(`- Dataset size: ${s.dataset_size}`);
+    lines.push(`- Candidate budget multiplier: ${s.candidate_budget_multiplier}`);
+    lines.push(`- Divergence rate: ${fmtPct(s.divergence_rate)}`);
+    lines.push(`- Changed cases: ${s.changed_cases}`);
+    lines.push(`- Projected total cost delta: ${s.projected_total_cost_delta}`);
+    lines.push(`- Projected avg cost delta: ${s.projected_avg_cost_delta}`);
+  } else {
+    lines.push('- No shadow eval results available.');
+  }
+
+  lines.push('');
+  lines.push('## Artifact Paths');
+  lines.push('');
+  lines.push('- `eval/results/latest.json`');
+  lines.push('- `eval/results/fault_injection_latest.json`');
+  lines.push('- `eval/results/shadow_latest.json`');
+  lines.push('- `eval/results/report_latest.md`');
+
+  const reportPath = path.join(resultsDir, 'report_latest.md');
+  writeFile(reportPath, lines.join('\n') + '\n');
+  console.log(`Report generated: ${reportPath}`);
+}
+
+main();

package/eval/golden_routes.json

@@ -0,0 +1,114 @@
+[
+  {
+    "id": "q01",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.1
+  },
+  {
+    "id": "q02",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.23
+  },
+  {
+    "id": "q03",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.14999999999999997
+  },
+  {
+    "id": "q04",
+    "primary_model": "ollama/llama3",
+    "provider_type": "local",
+    "detected_domain": "medical",
+    "complexity": 1
+  },
+  {
+    "id": "q05",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.36
+  },
+  {
+    "id": "q06",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.4
+  },
+  {
+    "id": "q07",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.3
+  },
+  {
+    "id": "q08",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.2
+  },
+  {
+    "id": "q09",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "architecture",
+    "complexity": 0.555
+  },
+  {
+    "id": "q10",
+    "primary_model": "ollama/llama3",
+    "provider_type": "local",
+    "detected_domain": "finance",
+    "complexity": 0.56
+  },
+  {
+    "id": "q11",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "legal",
+    "complexity": 0.355
+  },
+  {
+    "id": "q12",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "legal",
+    "complexity": 0.32499999999999996
+  },
+  {
+    "id": "q13",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.23000000000000004
+  },
+  {
+    "id": "q14",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "ml_research",
+    "complexity": 0.405
+  },
+  {
+    "id": "q15",
+    "primary_model": "commandcode/taste-1",
+    "provider_type": "cli",
+    "detected_domain": "",
+    "complexity": 0.33
+  },
+  {
+    "id": "q16",
+    "primary_model": "ollama/llama3",
+    "provider_type": "local",
+    "detected_domain": "medical",
+    "complexity": 0.40499999999999997
+  }
+]

package/eval/lib/experiment_registry.js

@@ -0,0 +1,24 @@
+const fs = require('fs');
+const path = require('path');
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function appendExperimentRecord(record) {
+  const evalDir = path.resolve(__dirname, '..');
+  const experimentsPath = path.join(evalDir, 'experiments.jsonl');
+  ensureDir(path.dirname(experimentsPath));
+
+  const payload = {
+    timestamp_utc: new Date().toISOString(),
+    commit: process.env.GITHUB_SHA || null,
+    ...record
+  };
+
+  fs.appendFileSync(experimentsPath, JSON.stringify(payload) + '\n', 'utf8');
+}
+
+module.exports = {
+  appendExperimentRecord
+};

package/eval/run_eval.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { routeQuery } = require('../dist/index.js');
+const { appendExperimentRecord } = require('./lib/experiment_registry');
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line, idx) => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {
+        throw new Error(`Invalid JSONL at line ${idx + 1}: ${error.message}`);
+      }
+    });
+}
+
+function safeGet(obj, key, fallback = false) {
+  return Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : fallback;
+}
+
+function toFixed(n) {
+  return Number(n.toFixed(4));
+}
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function evaluateCase(item) {
+  const decision = routeQuery(item.prompt);
+  const expected = item.expected || {};
+  const checks = [];
+
+  if (expected.complexity) {
+    const c = decision.features?.complexity ?? 0;
+    const ok = c >= expected.complexity.min && c <= expected.complexity.max;
+    checks.push({ type: 'complexity', ok, actual: c, expected: expected.complexity });
+  }
+
+  if (expected.flags) {
+    for (const [flag, expectedValue] of Object.entries(expected.flags)) {
+      const actual = safeGet(decision.features || {}, flag, false);
+      checks.push({ type: 'flag', flag, ok: actual === expectedValue, actual, expected: expectedValue });
+    }
+  }
+
+  if (expected.domain) {
+    const actual = decision.features?.detected_domain || '';
+    checks.push({ type: 'domain', ok: actual === expected.domain, actual, expected: expected.domain });
+  }
+
+  if (expected.provider_type) {
+    const actual = decision.provider_type || '';
+    checks.push({ type: 'provider_type', ok: actual === expected.provider_type, actual, expected: expected.provider_type });
+  }
+
+  return {
+    id: item.id,
+    prompt: item.prompt,
+    decision: {
+      primary_model: decision.primary_model,
+      provider_type: decision.provider_type,
+      estimated_cost: decision.estimated_cost,
+      complexity: decision.features?.complexity,
+      detected_domain: decision.features?.detected_domain || ''
+    },
+    checks
+  };
+}
+
+function summarize(results) {
+  const allChecks = results.flatMap((r) => r.checks);
+  const byType = (type) => allChecks.filter((c) => c.type === type);
+  const rate = (arr) => (arr.length ? arr.filter((x) => x.ok).length / arr.length : 1);
+
+  const complexity = rate(byType('complexity'));
+  const flags = rate(byType('flag'));
+  const domain = rate(byType('domain'));
+  const providerType = rate(byType('provider_type'));
+
+  const weighted = [complexity, flags, domain, providerType];
+  const overall = weighted.reduce((a, b) => a + b, 0) / weighted.length;
+
+  return {
+    dataset_size: results.length,
+    checks_count: allChecks.length,
+    complexity_accuracy: toFixed(complexity),
+    flag_accuracy: toFixed(flags),
+    domain_accuracy: toFixed(domain),
+    provider_type_accuracy: toFixed(providerType),
+    overall_score: toFixed(overall)
+  };
+}
+
+function gate(summary, thresholds, baseline) {
+  const failures = [];
+
+  if (summary.dataset_size < thresholds.min_dataset_size) {
+    failures.push(`dataset_size ${summary.dataset_size} < min_dataset_size ${thresholds.min_dataset_size}`);
+  }
+  if (summary.complexity_accuracy < thresholds.min_complexity_accuracy) {
+    failures.push(`complexity_accuracy ${summary.complexity_accuracy} < ${thresholds.min_complexity_accuracy}`);
+  }
+  if (summary.flag_accuracy < thresholds.min_flag_accuracy) {
+    failures.push(`flag_accuracy ${summary.flag_accuracy} < ${thresholds.min_flag_accuracy}`);
+  }
+  if (summary.domain_accuracy < thresholds.min_domain_accuracy) {
+    failures.push(`domain_accuracy ${summary.domain_accuracy} < ${thresholds.min_domain_accuracy}`);
+  }
+  if (summary.provider_type_accuracy < thresholds.min_provider_type_accuracy) {
+    failures.push(`provider_type_accuracy ${summary.provider_type_accuracy} < ${thresholds.min_provider_type_accuracy}`);
+  }
+  if (summary.overall_score < thresholds.min_overall_score) {
+    failures.push(`overall_score ${summary.overall_score} < ${thresholds.min_overall_score}`);
+  }
+
+  if (baseline?.summary) {
+    const delta = baseline.summary.overall_score - summary.overall_score;
+    if (delta > thresholds.max_regression_delta) {
+      failures.push(
+        `overall_score regression ${toFixed(delta)} > max_regression_delta ${thresholds.max_regression_delta}`
+      );
+    }
+  }
+
+  return failures;
+}
+
+function main() {
+  const evalDir = path.resolve(__dirname);
+  const datasetPath = path.join(evalDir, 'benchmark_dataset.jsonl');
+  const thresholdsPath = path.join(evalDir, 'thresholds.json');
+  const baselinePath = path.join(evalDir, 'baselines', 'main.json');
+  const resultsPath = path.join(evalDir, 'results', 'latest.json');
+
+  const dataset = readJsonl(datasetPath);
+  const thresholds = readJson(thresholdsPath);
+  const results = dataset.map(evaluateCase);
+  const summary = summarize(results);
+  const baseline = fs.existsSync(baselinePath) ? readJson(baselinePath) : null;
+  const failures = gate(summary, thresholds, baseline);
+
+  const output = {
+    timestamp_utc: new Date().toISOString(),
+    commit: process.env.GITHUB_SHA || null,
+    summary,
+    failures,
+    baseline_used: Boolean(baseline),
+    results
+  };
+
+  ensureDir(path.dirname(resultsPath));
+  fs.writeFileSync(resultsPath, JSON.stringify(output, null, 2));
+
+  console.log('\nA3M Routing Eval Summary');
+  console.log('------------------------');
+  console.log(JSON.stringify(summary, null, 2));
+  console.log(`Results file: ${resultsPath}`);
+
+  if (failures.length) {
+    appendExperimentRecord({
+      experiment_id: `routing_eval_${Date.now()}`,
+      dataset_version: 'core_regression_v1',
+      run_type: 'routing_eval',
+      metrics: summary,
+      decision: 'fail',
+      notes: failures
+    });
+    console.error('\nEval gate FAILED:');
+    for (const failure of failures) {
+      console.error(`- ${failure}`);
+    }
+    process.exit(1);
+  }
+
+  appendExperimentRecord({
+    experiment_id: `routing_eval_${Date.now()}`,
+    dataset_version: 'core_regression_v1',
+    run_type: 'routing_eval',
+    metrics: summary,
+    decision: 'pass',
+    notes: []
+  });
+
+  console.log('\nEval gate PASSED');
+}
+
+main();