lynkr 9.0.2 → 9.1.3

package/scripts/build-knn-index.js

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+/**
+ * Build the kNN router index from telemetry (and optional RouterBench bootstrap).
+ *
+ * Phase 3.1 of the routing overhaul. Should be run nightly:
+ *   node scripts/build-knn-index.js [--days 30] [--bootstrap path/to/routerbench.jsonl]
+ *
+ * RouterBench bootstrap format (one JSON per line):
+ *   { "query": "...", "provider": "anthropic", "model": "claude-...",
+ *     "quality": 87, "cost": 0.0034, "latency": 1200, "tier": "COMPLEX" }
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { generateEmbedding } = require('../src/cache/embeddings');
+const { getKnnRouter } = require('../src/routing/knn-router');
+
+const DEFAULT_DAYS = 30;
+const TELEMETRY_DB_CANDIDATES = [
+  path.join(__dirname, '../.lynkr/telemetry.db'),
+  path.join(__dirname, '../data/lynkr.db'),
+];
+
+function _findDb() {
+  for (const p of TELEMETRY_DB_CANDIDATES) if (fs.existsSync(p)) return p;
+  return null;
+}
+
+function _parseArgs(argv) {
+  const out = { days: DEFAULT_DAYS, bootstrap: null };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--days') out.days = Number(argv[++i]) || DEFAULT_DAYS;
+    else if (argv[i] === '--bootstrap') out.bootstrap = argv[++i];
+  }
+  return out;
+}
+
+async function _readTelemetry(days) {
+  const dbPath = _findDb();
+  if (!dbPath) return [];
+  let Database;
+  try {
+    Database = require('better-sqlite3');
+  } catch {
+    console.error('better-sqlite3 not installed');
+    return [];
+  }
+  const db = new Database(dbPath, { readonly: true, fileMustExist: true });
+  try {
+    const since = Date.now() - days * 24 * 3600 * 1000;
+    return db
+      .prepare(
+        `SELECT request_text AS query, provider, model, quality_score AS quality,
+                cost, total_latency_ms AS latency, tier
+           FROM routing_telemetry
+          WHERE timestamp >= ?
+            AND quality_score IS NOT NULL
+            AND request_text IS NOT NULL
+            AND request_text != ''`
+      )
+      .all(since);
+  } catch (err) {
+    console.error(`Telemetry query failed: ${err.message}`);
+    return [];
+  } finally {
+    try { db.close(); } catch {}
+  }
+}
+
+async function _readBootstrap(filePath) {
+  if (!filePath || !fs.existsSync(filePath)) return [];
+  const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
+  const out = [];
+  for (const line of lines) {
+    try {
+      out.push(JSON.parse(line));
+    } catch {
+      // skip malformed
+    }
+  }
+  return out;
+}
+
+async function build({ days = DEFAULT_DAYS, bootstrap = null } = {}) {
+  const router = getKnnRouter();
+  if (!router.ready) {
+    console.error('Router index not ready (hnswlib-node may be missing). Aborting.');
+    process.exit(2);
+  }
+
+  const teleRows = await _readTelemetry(days);
+  const bootRows = await _readBootstrap(bootstrap);
+  const all = [...bootRows, ...teleRows];
+  console.log(`Building index from ${bootRows.length} bootstrap + ${teleRows.length} telemetry rows`);
+
+  let added = 0;
+  let failed = 0;
+  for (const row of all) {
+    const text = row.query || row.request_text;
+    if (!text) continue;
+    try {
+      const emb = await generateEmbedding(text);
+      router.add(emb, {
+        provider: row.provider,
+        model: row.model,
+        quality: row.quality,
+        cost: row.cost,
+        latency: row.latency,
+        tier: row.tier,
+      });
+      added++;
+      if (added % 100 === 0) console.log(`  ${added} indexed...`);
+    } catch (err) {
+      failed++;
+    }
+  }
+
+  router.save();
+  console.log(`Indexed ${added}, failed ${failed}. Index size: ${router.size}`);
+}
+
+if (require.main === module) {
+  const opts = _parseArgs(process.argv.slice(2));
+  build(opts).catch((err) => {
+    console.error(err.stack || err.message);
+    process.exit(1);
+  });
+}
+
+module.exports = { build };

package/scripts/calibrate-thresholds.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+/**
+ * Calibrate tier thresholds from telemetry.
+ *
+ * Phase 1.4 of the routing overhaul. Reads quality_score history from the
+ * routing_telemetry table, finds where each tier's median quality drops below
+ * acceptable, and writes adjusted [lo, hi] ranges to
+ * data/calibrated-thresholds.json. ModelTierSelector picks the file up on
+ * next start.
+ *
+ * Usage: node scripts/calibrate-thresholds.js [--days N] [--dry-run]
+ *        npx lynkr calibrate
+ *
+ * Behavior when telemetry is sparse (<100 rows with quality_score):
+ *   - No file is written and existing calibration is left alone.
+ *   - Exits 0 with a "skipped" message.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_DAYS = 7;
+const MIN_SAMPLES = 100;
+/** Quality score below which a complexity bucket is "underperforming" for its tier. */
+const QUALITY_FLOOR = {
+  SIMPLE: 55,
+  MEDIUM: 60,
+  COMPLEX: 65,
+  REASONING: 70,
+};
+
+const OUTPUT_PATH = path.join(__dirname, '../data/calibrated-thresholds.json');
+const TELEMETRY_DB_CANDIDATES = [
+  path.join(__dirname, '../.lynkr/telemetry.db'),
+  path.join(__dirname, '../data/lynkr.db'),
+];
+
+function _findDb() {
+  for (const p of TELEMETRY_DB_CANDIDATES) {
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+function _parseArgs(argv) {
+  const out = { days: DEFAULT_DAYS, dryRun: false };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--days') out.days = Number(argv[++i]) || DEFAULT_DAYS;
+    else if (a === '--dry-run') out.dryRun = true;
+  }
+  return out;
+}
+
+const DEFAULT_RANGES = {
+  SIMPLE: [0, 25],
+  MEDIUM: [26, 50],
+  COMPLEX: [51, 75],
+  REASONING: [76, 100],
+};
+
+function _openDb(dbPath) {
+  let Database;
+  try {
+    Database = require('better-sqlite3');
+  } catch (err) {
+    console.error('better-sqlite3 not installed. Install with: npm install --save-optional better-sqlite3');
+    process.exit(2);
+  }
+  return new Database(dbPath, { readonly: true, fileMustExist: true });
+}
+
+function calibrate({ days = DEFAULT_DAYS, dryRun = false } = {}) {
+  const dbPath = _findDb();
+  if (!dbPath) {
+    console.log('No telemetry DB found — skipping calibration.');
+    return { skipped: true, reason: 'no_db' };
+  }
+
+  let db;
+  try {
+    db = _openDb(dbPath);
+  } catch (err) {
+    console.error(`Failed to open telemetry DB: ${err.message}`);
+    return { skipped: true, reason: 'db_open_failed', error: err.message };
+  }
+
+  const since = Date.now() - days * 24 * 3600 * 1000;
+  let rows;
+  try {
+    rows = db
+      .prepare(
+        `SELECT tier, complexity_score AS score, quality_score AS q
+           FROM routing_telemetry
+          WHERE timestamp >= ?
+            AND quality_score IS NOT NULL
+            AND complexity_score IS NOT NULL
+            AND tier IS NOT NULL`
+      )
+      .all(since);
+  } catch (err) {
+    console.error(`Telemetry query failed (DB may be corrupt or schema missing): ${err.message}`);
+    return { skipped: true, reason: 'query_failed', error: err.message };
+  } finally {
+    try { db.close(); } catch {}
+  }
+
+  if (!rows || rows.length < MIN_SAMPLES) {
+    console.log(`Only ${rows ? rows.length : 0} rows with quality_score in last ${days}d (need ≥${MIN_SAMPLES}). Skipping.`);
+    return { skipped: true, reason: 'insufficient_samples', count: rows ? rows.length : 0 };
+  }
+
+  // Bucket by score (0-100 in width-5 buckets) per tier, compute median quality.
+  const buckets = new Map(); // tier -> Map<bucketLowerBound, q-values[]>
+  for (const row of rows) {
+    const s = Math.max(0, Math.min(100, Math.floor(row.score)));
+    const bucket = Math.floor(s / 5) * 5;
+    if (!buckets.has(row.tier)) buckets.set(row.tier, new Map());
+    const b = buckets.get(row.tier);
+    if (!b.has(bucket)) b.set(bucket, []);
+    b.get(bucket).push(row.q);
+  }
+
+  const _median = (arr) => {
+    const s = arr.slice().sort((a, b) => a - b);
+    const m = Math.floor(s.length / 2);
+    return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
+  };
+
+  // Default ranges; will adjust per-tier upper bound if late buckets show poor quality.
+  const ranges = { ...DEFAULT_RANGES };
+  const tierOrder = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
+  const stats = {};
+
+  for (const tier of tierOrder) {
+    const floor = QUALITY_FLOOR[tier];
+    const tierBuckets = buckets.get(tier);
+    if (!tierBuckets) {
+      stats[tier] = { samples: 0, adjusted: false };
+      continue;
+    }
+    const ordered = Array.from(tierBuckets.entries()).sort((a, b) => a[0] - b[0]);
+    let suggestedUpper = DEFAULT_RANGES[tier][1];
+    const buckets_summary = [];
+    for (const [lo, vals] of ordered) {
+      if (vals.length < 5) {
+        buckets_summary.push({ bucket: lo, samples: vals.length, median: null });
+        continue;
+      }
+      const med = _median(vals);
+      buckets_summary.push({ bucket: lo, samples: vals.length, median: med });
+      if (med < floor && lo + 4 < suggestedUpper) {
+        suggestedUpper = lo + 4; // shrink tier upper bound just below the failing bucket
+      }
+    }
+    if (suggestedUpper !== DEFAULT_RANGES[tier][1]) {
+      ranges[tier] = [DEFAULT_RANGES[tier][0], suggestedUpper];
+      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: true, buckets: buckets_summary };
+    } else {
+      stats[tier] = { samples: ordered.reduce((s, [, v]) => s + v.length, 0), adjusted: false, buckets: buckets_summary };
+    }
+  }
+
+  // Re-stitch ranges so they don't overlap or leave gaps.
+  for (let i = 1; i < tierOrder.length; i++) {
+    const prev = ranges[tierOrder[i - 1]];
+    const cur = ranges[tierOrder[i]];
+    if (cur[0] !== prev[1] + 1) cur[0] = prev[1] + 1;
+    if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; tier disabled in practice
+  }
+
+  const out = {
+    calibratedAt: new Date().toISOString(),
+    days,
+    sampleCount: rows.length,
+    ranges,
+    stats,
+  };
+
+  if (dryRun) {
+    console.log(JSON.stringify(out, null, 2));
+    return { ...out, dryRun: true };
+  }
+
+  fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
+  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(out, null, 2));
+  console.log(`Wrote ${OUTPUT_PATH}`);
+  console.log(`Ranges: ${tierOrder.map((t) => `${t}=${ranges[t].join('-')}`).join(', ')}`);
+  return out;
+}
+
+if (require.main === module) {
+  const opts = _parseArgs(process.argv.slice(2));
+  calibrate(opts);
+}
+
+module.exports = { calibrate };

package/scripts/compare-policies.js

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+/**
+ * Compare active vs shadow routing policies (Phase 4.4).
+ *
+ * Reads data/shadow-decisions.jsonl and reports agreement rate and the
+ * disagreement breakdown by (active model → shadow model).
+ *
+ * Run weekly: node scripts/compare-policies.js [--days 7]
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { LOG_PATH } = require('../src/routing/shadow-mode');
+
+function _parseArgs(argv) {
+  let days = 7;
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--days') days = Number(argv[++i]) || 7;
+  }
+  return { days };
+}
+
+function main() {
+  const { days } = _parseArgs(process.argv.slice(2));
+  if (!fs.existsSync(LOG_PATH)) {
+    console.log('No shadow decisions logged yet.');
+    return;
+  }
+  const since = Date.now() - days * 24 * 3600 * 1000;
+  const lines = fs.readFileSync(LOG_PATH, 'utf8').split('\n').filter(Boolean);
+
+  let total = 0;
+  let agree = 0;
+  const disagreement = new Map(); // "active → shadow" -> count
+  for (const line of lines) {
+    let entry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (entry.timestamp < since) continue;
+    total++;
+    if (entry.agree) {
+      agree++;
+    } else if (entry.shadow) {
+      const key = `${entry.active.provider}:${entry.active.model} → ${entry.shadow.provider}:${entry.shadow.model}`;
+      disagreement.set(key, (disagreement.get(key) || 0) + 1);
+    }
+  }
+
+  if (total === 0) {
+    console.log(`No decisions in last ${days} days.`);
+    return;
+  }
+
+  console.log(`Last ${days}d: ${total} decisions, ${(agree / total * 100).toFixed(1)}% agreement`);
+  if (disagreement.size > 0) {
+    console.log('\nTop disagreements:');
+    const sorted = Array.from(disagreement.entries()).sort((a, b) => b[1] - a[1]).slice(0, 10);
+    for (const [k, c] of sorted) {
+      console.log(`  ${c}× ${k}`);
+    }
+  }
+}
+
+main();

package/scripts/learn-output-ratios.js

@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+/**
+ * Learn per-task-type output-token ratios from telemetry.
+ *
+ * Phase 2.3 of the routing overhaul. The cost-optimizer's default assumption
+ * of `output = 0.5 × input` is wrong for code generation (typically 1.5-3×)
+ * and summarization (typically 0.1-0.2×). This script builds an empirical
+ * ratio table from past completions, written to data/output-ratios.json.
+ *
+ * The cost-optimizer reads this file when estimating cost during routing.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_DAYS = 30;
+const MIN_SAMPLES_PER_TASK = 30;
+const OUTPUT_PATH = path.join(__dirname, '../data/output-ratios.json');
+const TELEMETRY_DB_CANDIDATES = [
+  path.join(__dirname, '../.lynkr/telemetry.db'),
+  path.join(__dirname, '../data/lynkr.db'),
+];
+
+// Fallback ratios when no telemetry exists.
+// Derived from public benchmark data (RouterBench task distribution).
+const FALLBACK_RATIOS = {
+  simple_qa: 0.30,
+  code_gen: 2.10,
+  code_edit: 1.40,
+  summarization: 0.15,
+  reasoning: 1.50,
+  tool_use: 0.80,
+  default: 0.50,
+};
+
+function _findDb() {
+  for (const p of TELEMETRY_DB_CANDIDATES) if (fs.existsSync(p)) return p;
+  return null;
+}
+
+function _openDb(dbPath) {
+  let Database;
+  try {
+    Database = require('better-sqlite3');
+  } catch {
+    console.error('better-sqlite3 not installed. Install with: npm install --save-optional better-sqlite3');
+    process.exit(2);
+  }
+  return new Database(dbPath, { readonly: true, fileMustExist: true });
+}
+
+function _median(arr) {
+  const s = arr.slice().sort((a, b) => a - b);
+  const m = Math.floor(s.length / 2);
+  return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
+}
+
+function _parseArgs(argv) {
+  const out = { days: DEFAULT_DAYS, dryRun: false };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--days') out.days = Number(argv[++i]) || DEFAULT_DAYS;
+    else if (argv[i] === '--dry-run') out.dryRun = true;
+  }
+  return out;
+}
+
+function learn({ days = DEFAULT_DAYS, dryRun = false } = {}) {
+  const dbPath = _findDb();
+  if (!dbPath) {
+    console.log('No telemetry DB — writing fallback ratios.');
+    if (!dryRun) {
+      fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
+      fs.writeFileSync(OUTPUT_PATH, JSON.stringify({
+        learnedAt: new Date().toISOString(),
+        source: 'fallback',
+        ratios: FALLBACK_RATIOS,
+      }, null, 2));
+    }
+    return { source: 'fallback', ratios: FALLBACK_RATIOS };
+  }
+
+  let db;
+  try {
+    db = _openDb(dbPath);
+  } catch (err) {
+    console.error(`Failed to open telemetry DB: ${err.message}. Writing fallback ratios.`);
+    if (!dryRun) {
+      fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
+      fs.writeFileSync(OUTPUT_PATH, JSON.stringify({
+        learnedAt: new Date().toISOString(),
+        source: 'fallback',
+        ratios: FALLBACK_RATIOS,
+      }, null, 2));
+    }
+    return { source: 'fallback', ratios: FALLBACK_RATIOS };
+  }
+
+  const since = Date.now() - days * 24 * 3600 * 1000;
+  let rows;
+  try {
+    rows = db
+      .prepare(
+        `SELECT task_type, input_tokens AS i, output_tokens AS o
+           FROM routing_telemetry
+          WHERE timestamp >= ?
+            AND input_tokens > 0
+            AND output_tokens > 0
+            AND task_type IS NOT NULL`
+      )
+      .all(since);
+  } catch (err) {
+    console.error(`Query failed: ${err.message}. Writing fallback.`);
+    rows = [];
+  } finally {
+    try { db.close(); } catch {}
+  }
+
+  // Bucket by task type
+  const buckets = new Map();
+  for (const row of rows) {
+    const key = String(row.task_type || 'default').toLowerCase();
+    if (!buckets.has(key)) buckets.set(key, []);
+    buckets.get(key).push(row.o / row.i);
+  }
+
+  const ratios = { ...FALLBACK_RATIOS };
+  const stats = {};
+  for (const [task, vals] of buckets) {
+    if (vals.length >= MIN_SAMPLES_PER_TASK) {
+      ratios[task] = +_median(vals).toFixed(3);
+      stats[task] = { samples: vals.length, median: ratios[task] };
+    } else {
+      stats[task] = { samples: vals.length, median: null, used_fallback: true };
+    }
+  }
+
+  const out = {
+    learnedAt: new Date().toISOString(),
+    days,
+    source: rows.length > 0 ? 'telemetry' : 'fallback',
+    sampleCount: rows.length,
+    ratios,
+    stats,
+  };
+
+  if (dryRun) {
+    console.log(JSON.stringify(out, null, 2));
+    return out;
+  }
+
+  fs.mkdirSync(path.dirname(OUTPUT_PATH), { recursive: true });
+  fs.writeFileSync(OUTPUT_PATH, JSON.stringify(out, null, 2));
+  console.log(`Wrote ${OUTPUT_PATH} (source=${out.source}, samples=${out.sampleCount})`);
+  return out;
+}
+
+if (require.main === module) {
+  const opts = _parseArgs(process.argv.slice(2));
+  learn(opts);
+}
+
+module.exports = { learn, FALLBACK_RATIOS };

package/scripts/refresh-pricing.js

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+/**
+ * Refresh model pricing data.
+ *
+ * Phase 2.2 of the routing overhaul. Cron-friendly entrypoint that forces a
+ * fresh pull of LiteLLM + models.dev pricing, compares to the last cached
+ * snapshot, and logs anything that moved more than 5%.
+ *
+ * Usage: node scripts/refresh-pricing.js [--diff-only] [--threshold 0.05]
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const CACHE_FILE = path.join(__dirname, '../data/model-prices-cache.json');
+const PREV_FILE = path.join(__dirname, '../data/model-prices-cache.prev.json');
+const DEFAULT_THRESHOLD = 0.05;
+
+function _parseArgs(argv) {
+  const out = { diffOnly: false, threshold: DEFAULT_THRESHOLD };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--diff-only') out.diffOnly = true;
+    else if (argv[i] === '--threshold') out.threshold = Number(argv[++i]) || DEFAULT_THRESHOLD;
+  }
+  return out;
+}
+
+function _readJson(p) {
+  try {
+    if (!fs.existsSync(p)) return null;
+    return JSON.parse(fs.readFileSync(p, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function _diff(prev, next, threshold) {
+  if (!prev || !next) return [];
+  const prevModels = prev.modelIndex || prev;
+  const nextModels = next.modelIndex || next;
+  const moves = [];
+  for (const [modelId, oldCost] of Object.entries(prevModels)) {
+    const newCost = nextModels[modelId];
+    if (!newCost) {
+      moves.push({ model: modelId, type: 'removed', oldCost });
+      continue;
+    }
+    const oldTotal = (oldCost.input || 0) + (oldCost.output || 0);
+    const newTotal = (newCost.input || 0) + (newCost.output || 0);
+    if (oldTotal === 0) continue;
+    const delta = (newTotal - oldTotal) / oldTotal;
+    if (Math.abs(delta) >= threshold) {
+      moves.push({
+        model: modelId,
+        type: delta > 0 ? 'increased' : 'decreased',
+        oldInput: oldCost.input,
+        newInput: newCost.input,
+        oldOutput: oldCost.output,
+        newOutput: newCost.output,
+        deltaPct: (delta * 100).toFixed(2) + '%',
+      });
+    }
+  }
+  for (const modelId of Object.keys(nextModels)) {
+    if (!prevModels[modelId]) {
+      moves.push({ model: modelId, type: 'added', newCost: nextModels[modelId] });
+    }
+  }
+  return moves;
+}
+
+async function refresh({ diffOnly = false, threshold = DEFAULT_THRESHOLD } = {}) {
+  if (!diffOnly) {
+    // Snapshot current cache as "previous" before fetching
+    if (fs.existsSync(CACHE_FILE)) {
+      try {
+        fs.copyFileSync(CACHE_FILE, PREV_FILE);
+      } catch (err) {
+        console.error(`Failed to snapshot previous cache: ${err.message}`);
+      }
+    }
+
+    const { getModelRegistry } = require('../src/routing/model-registry');
+    const registry = await getModelRegistry();
+    // Force a refresh
+    if (typeof registry._fetchAll === 'function') {
+      await registry._fetchAll();
+    }
+    console.log(`Refreshed pricing data (cache: ${CACHE_FILE})`);
+  }
+
+  const prev = _readJson(PREV_FILE);
+  const next = _readJson(CACHE_FILE);
+  const moves = _diff(prev, next, threshold);
+
+  if (moves.length === 0) {
+    console.log(`No pricing changes ≥${(threshold * 100).toFixed(1)}%.`);
+    return { moves: [] };
+  }
+
+  console.log(`${moves.length} pricing change(s) ≥${(threshold * 100).toFixed(1)}%:`);
+  for (const move of moves) {
+    if (move.type === 'added') {
+      console.log(`  + ${move.model}: input=${move.newCost.input}, output=${move.newCost.output}`);
+    } else if (move.type === 'removed') {
+      console.log(`  - ${move.model}: was input=${move.oldCost.input}, output=${move.oldCost.output}`);
+    } else {
+      console.log(`  ${move.type === 'increased' ? '↑' : '↓'} ${move.model}: ${move.oldInput}/${move.oldOutput} → ${move.newInput}/${move.newOutput} (${move.deltaPct})`);
+    }
+  }
+  return { moves };
+}
+
+if (require.main === module) {
+  const opts = _parseArgs(process.argv.slice(2));
+  refresh(opts).catch((err) => {
+    console.error(err.message);
+    process.exit(1);
+  });
+}
+
+module.exports = { refresh };

package/scripts/run-routerarena.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+/**
+ * RouterArena evaluation harness (Phase 6.4 — STUB).
+ *
+ * This is intentionally not wired to CI yet. The plan defers RouterArena
+ * integration until after Phases 1-4 have produced 2-4 weeks of telemetry
+ * to baseline against.
+ *
+ * To wire it up:
+ *   1. Clone https://github.com/RouteWorks/RouterArena into ./routerarena/
+ *   2. Install RouterArena's Python dependencies (transformers, datasets,
+ *      anthropic, openai)
+ *   3. Decide on a subset size for PR-blocking CI (recommend 100-200 queries
+ *      sampled stratified by difficulty); leave the full benchmark for nightly
+ *   4. Wire to GitHub Actions with `paths: [src/routing/**]` trigger
+ *   5. Compare PR's router decisions vs main's router on the same query set,
+ *      report cost/quality delta as a PR comment
+ *
+ * The intent is to use RouterArena to *catch regressions*, not to gate
+ * routing changes on absolute benchmark scores.
+ */
+
+console.log('RouterArena integration is a stub.');
+console.log('See scripts/run-routerarena.js for setup steps.');
+console.log('Phase 6.4 of docs/routing-improvement-plan.md.');
+process.exit(0);