@machinespirits/eval 0.2.0 → 0.3.0

package/scripts/test-latency.js

@@ -0,0 +1,210 @@
+#!/usr/bin/env node
+/**
+ * Lightweight latency test for all configured models.
+ *
+ * Sends a single short prompt ("Say hello in one sentence.") to each model
+ * and reports latency, token counts, and cost in a compact table.
+ *
+ * Usage:
+ *   node scripts/test-latency.js                  # all openrouter models (default)
+ *   node scripts/test-latency.js --provider all    # all providers
+ *   node scripts/test-latency.js --models nemotron,glm5,kimi-k2.5
+ *   node scripts/test-latency.js --serial          # one at a time
+ *   node scripts/test-latency.js --prompt "Explain Hegel in one sentence."
+ *   node scripts/test-latency.js --max-tokens 500  # override max output tokens
+ *   node scripts/test-latency.js --input "I don't understand why Hegel matters"
+ */
+
+import 'dotenv/config';
+import { unifiedAIProvider } from '@machinespirits/tutor-core';
+import * as evalConfigLoader from '../services/evalConfigLoader.js';
+
+// ── CLI args ────────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+const getArg = (flag) => {
+  const idx = args.indexOf(flag);
+  return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : null;
+};
+const hasFlag = (flag) => args.includes(flag);
+
+const serial = hasFlag('--serial');
+const maxTokens = parseInt(getArg('--max-tokens') || '200', 10);
+const providerFilter = getArg('--provider') || 'openrouter';
+const modelsFilter = getArg('--models')?.split(',').map(s => s.trim()) || null;
+
+const defaultInput = 'I keep reading about Hegel\'s master-slave dialectic but I don\'t really get why it matters. Can you explain it simply?';
+const learnerInput = getArg('--input') || getArg('--prompt') || defaultInput;
+const systemPrompt = 'You are a philosophy tutor. Respond helpfully and concisely to the learner.';
+
+// ── Discover models ─────────────────────────────────────────────────────────
+
+function discoverModels() {
+  const providers = evalConfigLoader.loadProviders();
+  if (!providers?.providers) {
+    console.error('No providers found in config/providers.yaml');
+    process.exit(1);
+  }
+
+  const targets = [];
+
+  for (const [provName, provConfig] of Object.entries(providers.providers)) {
+    if (providerFilter !== 'all' && provName !== providerFilter) continue;
+    if (provName === 'local') continue;
+
+    const models = provConfig.models || {};
+    for (const [alias, modelId] of Object.entries(models)) {
+      if (modelsFilter && !modelsFilter.includes(alias)) continue;
+      targets.push({ provider: provName, alias, modelId });
+    }
+  }
+
+  return targets;
+}
+
+// ── Test a single model ─────────────────────────────────────────────────────
+
+async function testModel({ provider, alias, modelId }) {
+  const label = providerFilter === 'all' ? `${provider}.${alias}` : alias;
+  try {
+    const resolved = evalConfigLoader.resolveModel({ provider, model: alias });
+    if (!resolved.isConfigured) {
+      return { label, modelId, status: 'skip', reason: 'no API key' };
+    }
+
+    const start = Date.now();
+    const response = await unifiedAIProvider.call({
+      provider,
+      model: resolved.model,
+      systemPrompt,
+      messages: [{ role: 'user', content: learnerInput }],
+      config: {
+        temperature: 0.3,
+        maxTokens,
+      },
+    });
+    const wallMs = Date.now() - start;
+
+    return {
+      label,
+      modelId: resolved.model,
+      status: 'ok',
+      latencyMs: response.latencyMs || wallMs,
+      wallMs,
+      inputTokens: response.usage?.inputTokens || 0,
+      outputTokens: response.usage?.outputTokens || 0,
+      cost: response.usage?.cost || 0,
+      content: (response.content || '').replace(/\s+/g, ' ').trim(),
+    };
+  } catch (error) {
+    return { label, modelId, status: 'error', reason: error.message.substring(0, 100) };
+  }
+}
+
+// ── Formatting helpers ──────────────────────────────────────────────────────
+
+function formatLatency(ms) {
+  return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`;
+}
+
+function formatCost(cost) {
+  if (!cost || cost === 0) return '  --';
+  if (cost < 0.001) return `$${cost.toFixed(6)}`;
+  return `$${cost.toFixed(4)}`;
+}
+
+function bar(ms, maxMs) {
+  const width = 20;
+  const filled = Math.round((ms / maxMs) * width);
+  return '█'.repeat(filled) + '░'.repeat(width - filled);
+}
+
+// ── Main ────────────────────────────────────────────────────────────────────
+
+const targets = discoverModels();
+if (targets.length === 0) {
+  console.error('No models matched. Check --provider / --models flags.');
+  process.exit(1);
+}
+
+console.log(`\nTesting ${targets.length} model(s) ${serial ? 'sequentially' : 'in parallel'} (max ${maxTokens} tokens)...`);
+console.log(`Input: "${learnerInput}"\n`);
+
+let results;
+if (serial) {
+  results = [];
+  for (const t of targets) {
+    process.stdout.write(`  ${t.alias} ... `);
+    const r = await testModel(t);
+    if (r.status === 'ok') {
+      process.stdout.write(`${formatLatency(r.latencyMs)} (${r.inputTokens}→${r.outputTokens} tok)\n`);
+    } else {
+      process.stdout.write(`${r.status}: ${r.reason || ''}\n`);
+    }
+    results.push(r);
+  }
+} else {
+  results = await Promise.all(targets.map(t => testModel(t)));
+}
+
+// ── Table output ────────────────────────────────────────────────────────────
+
+const ok = results.filter(r => r.status === 'ok').sort((a, b) => a.latencyMs - b.latencyMs);
+const failed = results.filter(r => r.status !== 'ok');
+
+if (ok.length > 0) {
+  const maxMs = ok[ok.length - 1].latencyMs;
+  const labelW = Math.max(12, ...ok.map(r => r.label.length));
+  const modelW = Math.max(15, ...ok.map(r => r.modelId.length));
+  const sep = '─'.repeat(labelW + modelW + 68);
+
+  console.log(`\n${sep}`);
+  console.log(
+    ' ' + 'Alias'.padEnd(labelW) +
+    '  ' + 'Model'.padEnd(modelW) +
+    '  ' + 'Latency'.padStart(7) +
+    '  ' + 'In'.padStart(4) +
+    '  ' + 'Out'.padStart(4) +
+    '  ' + 'Cost'.padStart(9) +
+    '  ' + 'Bar'.padEnd(20) +
+    '  Response'
+  );
+  console.log(sep);
+
+  for (const r of ok) {
+    console.log(
+      ' ' + r.label.padEnd(labelW) +
+      '  ' + r.modelId.padEnd(modelW) +
+      '  ' + formatLatency(r.latencyMs).padStart(7) +
+      '  ' + String(r.inputTokens).padStart(4) +
+      '  ' + String(r.outputTokens).padStart(4) +
+      '  ' + formatCost(r.cost).padStart(9) +
+      '  ' + bar(r.latencyMs, maxMs) +
+      '  ' + r.content.substring(0, 35)
+    );
+  }
+  console.log(sep);
+}
+
+if (failed.length > 0) {
+  console.log('\nFailed/Skipped:');
+  for (const r of failed) {
+    console.log(`  ${r.label}: ${r.reason || r.status}`);
+  }
+}
+
+// ── Summary ─────────────────────────────────────────────────────────────────
+
+if (ok.length > 0) {
+  const fastest = ok[0];
+  const slowest = ok[ok.length - 1];
+  const median = ok[Math.floor(ok.length / 2)];
+  const totalCost = ok.reduce((s, r) => s + r.cost, 0);
+  const avgLatency = Math.round(ok.reduce((s, r) => s + r.latencyMs, 0) / ok.length);
+  console.log(`\n${ok.length} succeeded, ${failed.length} failed`);
+  console.log(`  Fastest:  ${fastest.label} (${formatLatency(fastest.latencyMs)})`);
+  console.log(`  Median:   ${median.label} (${formatLatency(median.latencyMs)})`);
+  console.log(`  Slowest:  ${slowest.label} (${formatLatency(slowest.latencyMs)})`);
+  console.log(`  Average:  ${formatLatency(avgLatency)}`);
+  if (totalCost > 0) console.log(`  Total cost: ${formatCost(totalCost)}`);
+}

package/scripts/test-rate-limit.js

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+/**
+ * Quick rate-limit probe for OpenRouter models.
+ * Usage: node scripts/test-rate-limit.js [model-alias]
+ * Default: nemotron
+ */
+
+const MODEL_MAP = {
+  nemotron: 'nvidia/nemotron-3-nano-30b-a3b:free',
+  glm47: 'z-ai/glm-4.7',
+  'kimi-k2.5': 'moonshotai/kimi-k2.5',
+  deepseek: 'deepseek/deepseek-v3.2',
+  haiku: 'anthropic/claude-haiku-4.5',
+};
+
+const alias = process.argv[2] || 'nemotron';
+const model = MODEL_MAP[alias] || alias;
+const apiKey = process.env.OPENROUTER_API_KEY;
+
+if (!apiKey) {
+  console.error('OPENROUTER_API_KEY not set');
+  process.exit(1);
+}
+
+function formatReset(resetValue) {
+  const ts = Number(resetValue);
+  if (!ts || isNaN(ts)) return resetValue;
+  const resetDate = new Date(ts);
+  const now = new Date();
+  const diffMs = resetDate - now;
+  const local = resetDate.toLocaleString('en-AU', { timeZone: 'Australia/Melbourne' });
+  if (diffMs <= 0) return `${local} AEDT (already passed)`;
+  const mins = Math.ceil(diffMs / 60000);
+  if (mins < 60) return `${local} AEDT (in ${mins}m)`;
+  const hrs = Math.floor(mins / 60);
+  const remMins = mins % 60;
+  return `${local} AEDT (in ${hrs}h ${remMins}m)`;
+}
+
+async function probe() {
+  console.log(`Probing ${alias} (${model})...\n`);
+  const start = Date.now();
+
+  const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      model,
+      messages: [{ role: 'user', content: 'Say "hello" and nothing else.' }],
+      max_tokens: 10,
+    }),
+  });
+
+  const elapsed = Date.now() - start;
+  const headers = Object.fromEntries(res.headers.entries());
+
+  // Rate limit headers
+  const rl = {
+    limit: headers['x-ratelimit-limit-requests'] || headers['x-ratelimit-limit'] || '?',
+    remaining: headers['x-ratelimit-remaining-requests'] || headers['x-ratelimit-remaining'] || '?',
+    reset: headers['x-ratelimit-reset-requests'] || headers['x-ratelimit-reset'] || '?',
+  };
+
+  const body = await res.json();
+
+  console.log(`Status: ${res.status} (${elapsed}ms)`);
+  console.log(`Rate limit: ${rl.remaining}/${rl.limit} remaining`);
+  console.log(`Resets: ${formatReset(rl.reset)}`);
+
+  if (res.status === 429) {
+    console.log('\n*** RATE LIMITED ***');
+    console.log('Error:', body.error?.message || JSON.stringify(body));
+    process.exit(2);
+  }
+
+  if (res.status !== 200) {
+    console.log('\nError:', body.error?.message || JSON.stringify(body));
+    process.exit(1);
+  }
+
+  const reply = body.choices?.[0]?.message?.content || '(empty)';
+  const usage = body.usage || {};
+  console.log(`Reply: "${reply.trim()}"`);
+  console.log(`Tokens: ${usage.prompt_tokens || '?'} in / ${usage.completion_tokens || '?'} out`);
+  if (body.id) console.log(`Request ID: ${body.id}`);
+}
+
+probe().catch(err => {
+  console.error('Fetch error:', err.message);
+  process.exit(1);
+});

package/scripts/test-token-budget.js

@@ -0,0 +1,332 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+/**
+ * Token Budget Sensitivity Test
+ *
+ * Runs a dose-response curve measuring how constraining max_tokens affects
+ * evaluation scores. Useful for optimizing cost/latency without sacrificing quality.
+ *
+ * Usage:
+ *   node scripts/test-token-budget.js [options]
+ *
+ * Options:
+ *   --model <model>       Ego model (default: openrouter.haiku)
+ *   --levels <csv>        Comma-separated max_tokens levels (default: 256,512,1024,2048,4000)
+ *   --runs <n>            Runs per level×cell (default: 4)
+ *   --profiles <csv>      Cell profiles (default: cell_1_base_single_unified,cell_5_recog_single_unified)
+ *   --skip-judge          Skip rubric evaluation (generate only, judge later)
+ *   --parallelism <n>     Parallelism per run (default: 2)
+ *   --report-only <csv>   Skip generation, just build report from existing run IDs
+ */
+
+import { execSync, execFileSync } from 'child_process';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+import * as evaluationStore from '../services/evaluationStore.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const CLI_PATH = path.join(__dirname, 'eval-cli.js');
+const EXPORTS_DIR = path.join(__dirname, '..', 'exports');
+
+// Parse CLI arguments
+function getOption(name) {
+  const idx = process.argv.indexOf(`--${name}`);
+  return idx !== -1 && idx + 1 < process.argv.length ? process.argv[idx + 1] : null;
+}
+function getFlag(name) {
+  return process.argv.includes(`--${name}`);
+}
+
+const model = getOption('model') || 'openrouter.haiku';
+const levels = (getOption('levels') || '256,512,1024,2048,4000').split(',').map(s => parseInt(s.trim(), 10));
+const runsPerLevel = parseInt(getOption('runs') || '4', 10);
+const profiles = (getOption('profiles') || 'cell_1_base_single_unified,cell_5_recog_single_unified').split(',').map(s => s.trim());
+const skipJudge = getFlag('skip-judge');
+const parallelism = getOption('parallelism') || '2';
+const reportOnly = getOption('report-only');
+
+// Summary
+const totalEvals = levels.length * profiles.length * runsPerLevel * 3; // 3 scenarios
+console.log('\n╔══════════════════════════════════════════════════╗');
+console.log('║       Token Budget Sensitivity Test              ║');
+console.log('╚══════════════════════════════════════════════════╝');
+console.log(`  Model:      ${model}`);
+console.log(`  Levels:     ${levels.join(', ')}`);
+console.log(`  Profiles:   ${profiles.join(', ')}`);
+console.log(`  Runs/level: ${runsPerLevel}`);
+console.log(`  Total evals: ~${totalEvals}`);
+console.log(`  Skip judge: ${skipJudge}`);
+console.log('');
+
+/**
+ * Run evaluations for each token budget level and collect run IDs.
+ */
+async function runAllLevels() {
+  if (reportOnly) {
+    const ids = reportOnly.split(',').map(s => s.trim());
+    console.log(`Report-only mode: using ${ids.length} existing run IDs\n`);
+    return ids;
+  }
+
+  const runIds = [];
+
+  for (const level of levels) {
+    console.log(`\n${'─'.repeat(60)}`);
+    console.log(`Running max_tokens=${level}...`);
+    console.log(`${'─'.repeat(60)}`);
+
+    const args = [
+      CLI_PATH,
+      'run',
+      '--profiles', profiles.join(','),
+      '--runs', String(runsPerLevel),
+      '--max-tokens', String(level),
+      '--model', model,
+      '--parallelism', parallelism,
+      '--description', `Token budget test: max_tokens=${level}`,
+    ];
+    if (skipJudge) args.push('--skip-rubric');
+
+    try {
+      const output = execFileSync('node', args, {
+        encoding: 'utf-8',
+        stdio: ['inherit', 'pipe', 'inherit'],
+        timeout: 600_000, // 10 min per level
+      });
+
+      // Extract run ID from output (format: "Run ID: eval-YYYY-MM-DD-XXXXXXXX")
+      const match = output.match(/Run ID:\s*(eval-[\w-]+)/);
+      if (match) {
+        runIds.push(match[1]);
+        console.log(`  ✓ Completed: ${match[1]}`);
+      } else {
+        // Try alternative format
+        const altMatch = output.match(/(eval-\d{4}-\d{2}-\d{2}-[a-f0-9]+)/);
+        if (altMatch) {
+          runIds.push(altMatch[1]);
+          console.log(`  ✓ Completed: ${altMatch[1]}`);
+        } else {
+          console.error(`  ✗ Could not extract run ID for max_tokens=${level}`);
+          console.error('    Output:', output.slice(-200));
+        }
+      }
+    } catch (err) {
+      console.error(`  ✗ Failed for max_tokens=${level}:`, err.message);
+    }
+  }
+
+  return runIds;
+}
+
+/**
+ * Build the dose-response report from completed run IDs.
+ */
+function buildReport(runIds) {
+  console.log(`\n${'═'.repeat(60)}`);
+  console.log('  BUILDING DOSE-RESPONSE REPORT');
+  console.log(`${'═'.repeat(60)}\n`);
+
+  // Collect data per level × profile
+  const data = new Map(); // key: `${level}|${profileName}` → { scores, outputTokens, budget }
+
+  for (const runId of runIds) {
+    const results = evaluationStore.getResults(runId);
+    if (results.length === 0) {
+      console.warn(`  Warning: no results for ${runId}`);
+      continue;
+    }
+
+    // Extract max_tokens from hyperparameters of first result
+    const firstHyper = typeof results[0].hyperparameters === 'string'
+      ? JSON.parse(results[0].hyperparameters || '{}')
+      : (results[0].hyperparameters || {});
+    const budget = firstHyper.max_tokens || null;
+
+    if (!budget) {
+      console.warn(`  Warning: no max_tokens in hyperparameters for ${runId}`);
+      continue;
+    }
+
+    for (const r of results) {
+      const profile = r.profile_name || r.profileName;
+      const score = r.overall_score;
+      const outTokens = r.output_tokens || r.outputTokens || 0;
+      const apiCalls = r.api_calls || r.apiCalls || 1;
+
+      if (score == null) continue; // unjudged
+
+      const key = `${budget}|${profile}`;
+      if (!data.has(key)) {
+        data.set(key, { budget, profile, scores: [], outputTokens: [], apiCalls: [] });
+      }
+      const entry = data.get(key);
+      entry.scores.push(score);
+      entry.outputTokens.push(outTokens);
+      entry.apiCalls.push(apiCalls);
+    }
+  }
+
+  if (data.size === 0) {
+    console.log('No scored data found. Run without --skip-judge or judge the runs first.');
+    return;
+  }
+
+  // Compute statistics
+  const stats = (arr) => {
+    if (arr.length === 0) return { mean: 0, sd: 0, n: 0 };
+    const n = arr.length;
+    const mean = arr.reduce((a, b) => a + b, 0) / n;
+    const sd = n > 1
+      ? Math.sqrt(arr.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1))
+      : 0;
+    return { mean, sd, n };
+  };
+
+  // Extract model alias from the model string
+  const modelAlias = model.includes('.') ? model.split('.').slice(1).join('.') : model;
+
+  // Build table rows grouped by profile
+  const profileNames = [...new Set([...data.values()].map(d => d.profile))].sort();
+  const budgetLevels = [...new Set([...data.values()].map(d => d.budget))].sort((a, b) => a - b);
+
+  // Format the report
+  const lines = [];
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+
+  lines.push(`# Token Budget Sensitivity Test`);
+  lines.push('');
+  lines.push(`- **Date:** ${new Date().toISOString().slice(0, 10)}`);
+  lines.push(`- **Model:** ${modelAlias}`);
+  lines.push(`- **Runs per level×cell:** ${runsPerLevel}`);
+  lines.push(`- **Run IDs:** ${runIds.join(', ')}`);
+  lines.push('');
+
+  // Build the table
+  const profileLabels = profileNames.map(p => {
+    if (p.includes('base')) return `Base (${p})`;
+    if (p.includes('recog')) return `Recognition (${p})`;
+    return p;
+  });
+
+  // Header
+  const colWidth = 28;
+  let header = ' Budget  |';
+  let divider = '---------|';
+  for (const label of profileLabels) {
+    const shortLabel = label.length > colWidth - 2 ? label.slice(0, colWidth - 5) + '...' : label;
+    header += ` ${shortLabel.padEnd(colWidth)}|`;
+    divider += `${'-'.repeat(colWidth + 1)}|`;
+  }
+  lines.push('## Dose-Response Table');
+  lines.push('');
+  lines.push('```');
+  lines.push(header);
+  lines.push(`         | ${profileLabels.map(() => 'Mean   SD    N   Trunc%'.padEnd(colWidth)).join('| ')}|`);
+  lines.push(divider);
+
+  for (const budget of budgetLevels) {
+    let row = ` ${String(budget).padStart(5)}   |`;
+    for (const profile of profileNames) {
+      const key = `${budget}|${profile}`;
+      const entry = data.get(key);
+      if (!entry) {
+        row += ` ${'—'.padEnd(colWidth)}|`;
+        continue;
+      }
+
+      const s = stats(entry.scores);
+      // Truncation: per-row check whether output tokens >= budget × api_calls (within 95%).
+      // output_tokens is cumulative across all API calls (including inner retries),
+      // so we scale the threshold by api_calls to avoid false positives.
+      const truncCount = entry.outputTokens.filter((t, i) => {
+        const calls = entry.apiCalls[i] || 1;
+        return t >= Math.floor(budget * calls * 0.95);
+      }).length;
+      const truncPct = entry.outputTokens.length > 0
+        ? Math.round(100 * truncCount / entry.outputTokens.length)
+        : 0;
+
+      const cell = `${s.mean.toFixed(1).padStart(5)}  ${s.sd.toFixed(1).padStart(5)}  ${String(s.n).padStart(3)}   ${String(truncPct).padStart(3)}%`;
+      row += ` ${cell.padEnd(colWidth)}|`;
+    }
+    lines.push(row);
+  }
+  lines.push('```');
+  lines.push('');
+
+  // Effect size summary
+  if (profileNames.length >= 2 && budgetLevels.length >= 2) {
+    lines.push('## Key Observations');
+    lines.push('');
+
+    const highBudget = budgetLevels[budgetLevels.length - 1];
+    const lowBudget = budgetLevels[0];
+
+    for (const profile of profileNames) {
+      const highKey = `${highBudget}|${profile}`;
+      const lowKey = `${lowBudget}|${profile}`;
+      const highEntry = data.get(highKey);
+      const lowEntry = data.get(lowKey);
+
+      if (highEntry && lowEntry) {
+        const highStats = stats(highEntry.scores);
+        const lowStats = stats(lowEntry.scores);
+        const delta = highStats.mean - lowStats.mean;
+        const pooledSD = Math.sqrt(((highStats.sd ** 2) + (lowStats.sd ** 2)) / 2);
+        const d = pooledSD > 0 ? delta / pooledSD : 0;
+
+        lines.push(`- **${profile}**: ${highBudget} vs ${lowBudget} tokens → Δ=${delta.toFixed(1)} pts (d=${d.toFixed(2)})`);
+      }
+    }
+    lines.push('');
+  }
+
+  // Raw data table
+  lines.push('## Raw Data');
+  lines.push('');
+  lines.push('| Budget | Profile | N | Mean | SD | Trunc% |');
+  lines.push('|--------|---------|---|------|-----|--------|');
+  for (const budget of budgetLevels) {
+    for (const profile of profileNames) {
+      const key = `${budget}|${profile}`;
+      const entry = data.get(key);
+      if (!entry) continue;
+      const s = stats(entry.scores);
+      const truncCount = entry.outputTokens.filter((t, i) => {
+        const calls = entry.apiCalls[i] || 1;
+        return t >= Math.floor(budget * calls * 0.95);
+      }).length;
+      const truncPct = entry.outputTokens.length > 0
+        ? Math.round(100 * truncCount / entry.outputTokens.length)
+        : 0;
+      lines.push(`| ${budget} | ${profile} | ${s.n} | ${s.mean.toFixed(1)} | ${s.sd.toFixed(1)} | ${truncPct}% |`);
+    }
+  }
+
+  const report = lines.join('\n');
+
+  // Print to console
+  console.log(report);
+
+  // Write to file
+  if (!fs.existsSync(EXPORTS_DIR)) fs.mkdirSync(EXPORTS_DIR, { recursive: true });
+  const outPath = path.join(EXPORTS_DIR, `token-budget-sensitivity-${timestamp}.md`);
+  fs.writeFileSync(outPath, report + '\n');
+  console.log(`\nReport written to: ${outPath}`);
+}
+
+// Main
+(async () => {
+  try {
+    const runIds = await runAllLevels();
+    if (runIds.length > 0) {
+      buildReport(runIds);
+    } else {
+      console.log('\nNo runs completed. Nothing to report.');
+    }
+  } catch (err) {
+    console.error('Fatal error:', err);
+    process.exit(1);
+  }
+})();