@machinespirits/eval 0.2.0 → 0.3.0

package/scripts/analyze-run.mjs

@@ -1,282 +0,0 @@
-/**
- * Detailed statistical analysis of an evaluation run.
- * Usage: node scripts/analyze-run.mjs [run_id]
- */
-import Database from 'better-sqlite3';
-const db = new Database('data/evaluations.db');
-
-const RUN_ID = process.argv[2] || db.prepare(
-  'SELECT run_id FROM evaluation_results ORDER BY created_at DESC LIMIT 1'
-).get()?.run_id;
-
-if (!RUN_ID) { console.error('No run found'); process.exit(1); }
-console.log(`Analyzing run: ${RUN_ID}\n`);
-
-// ============================================================
-// Helper functions
-// ============================================================
-function std(values) {
-  if (values.length < 2) return 0;
-  const mean = values.reduce((a, b) => a + b, 0) / values.length;
-  const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (values.length - 1);
-  return Math.sqrt(variance);
-}
-
-function cohensD(group1, group2) {
-  const m1 = group1.reduce((a, b) => a + b, 0) / group1.length;
-  const m2 = group2.reduce((a, b) => a + b, 0) / group2.length;
-  const s1 = std(group1);
-  const s2 = std(group2);
-  const pooled = Math.sqrt(((group1.length - 1) * s1 ** 2 + (group2.length - 1) * s2 ** 2) / (group1.length + group2.length - 2));
-  return pooled === 0 ? 0 : (m1 - m2) / pooled;
-}
-
-function percentile(values, p) {
-  const sorted = [...values].sort((a, b) => a - b);
-  const idx = (p / 100) * (sorted.length - 1);
-  const lo = Math.floor(idx);
-  const hi = Math.ceil(idx);
-  return lo === hi ? sorted[lo] : sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
-}
-
-// ============================================================
-// 1. Summary statistics
-// ============================================================
-const allScores = db.prepare(`
-  SELECT overall_score FROM evaluation_results
-  WHERE run_id = ? AND overall_score IS NOT NULL
-`).all(RUN_ID).map(r => r.overall_score);
-
-const mean = allScores.reduce((a, b) => a + b, 0) / allScores.length;
-const sd = std(allScores);
-const median = percentile(allScores, 50);
-const q1 = percentile(allScores, 25);
-const q3 = percentile(allScores, 75);
-
-console.log('=== DESCRIPTIVE STATISTICS ===');
-console.log(`N = ${allScores.length}`);
-console.log(`Mean: ${mean.toFixed(1)} (SD: ${sd.toFixed(1)})`);
-console.log(`Median: ${median.toFixed(1)} (IQR: ${q1.toFixed(1)} – ${q3.toFixed(1)})`);
-console.log(`Range: ${Math.min(...allScores).toFixed(1)} – ${Math.max(...allScores).toFixed(1)}`);
-
-// ============================================================
-// 2. Per-model statistics
-// ============================================================
-console.log('\n=== PER-MODEL STATISTICS ===');
-const models = db.prepare(`
-  SELECT DISTINCT model FROM evaluation_results
-  WHERE run_id = ? AND overall_score IS NOT NULL
-`).all(RUN_ID).map(r => r.model);
-
-const modelData = {};
-for (const m of models) {
-  const scores = db.prepare(`
-    SELECT overall_score FROM evaluation_results
-    WHERE run_id = ? AND model = ? AND overall_score IS NOT NULL
-  `).all(RUN_ID, m).map(r => r.overall_score);
-  modelData[m] = scores;
-  const mn = scores.reduce((a, b) => a + b, 0) / scores.length;
-  const s = std(scores);
-  console.log(`${m}: M=${mn.toFixed(1)}, SD=${s.toFixed(1)}, N=${scores.length}, Range=[${Math.min(...scores).toFixed(1)}, ${Math.max(...scores).toFixed(1)}]`);
-}
-
-// ============================================================
-// 3. Pairwise effect sizes between models
-// ============================================================
-console.log('\n=== PAIRWISE EFFECT SIZES (Cohen\'s d) ===');
-const modelNames = Object.keys(modelData).filter(m => modelData[m].length >= 3);
-for (let i = 0; i < modelNames.length; i++) {
-  for (let j = i + 1; j < modelNames.length; j++) {
-    const d = cohensD(modelData[modelNames[i]], modelData[modelNames[j]]);
-    const label = d > 0.8 ? 'large' : d > 0.5 ? 'medium' : d > 0.2 ? 'small' : 'negligible';
-    console.log(`${modelNames[i]} vs ${modelNames[j]}: d=${d.toFixed(2)} (${label})`);
-  }
-}
-
-// ============================================================
-// 4. Per-dimension statistics
-// ============================================================
-console.log('\n=== DIMENSION STATISTICS ===');
-const dims = ['relevance', 'specificity', 'pedagogical', 'personalization', 'actionability', 'tone'];
-const dimCols = dims.map(d => `score_${d}`);
-
-for (const dim of dims) {
-  const col = `score_${dim}`;
-  const vals = db.prepare(`
-    SELECT ${col} as v FROM evaluation_results
-    WHERE run_id = ? AND ${col} IS NOT NULL
-  `).all(RUN_ID).map(r => r.v);
-  if (vals.length === 0) continue;
-  const mn = vals.reduce((a, b) => a + b, 0) / vals.length;
-  const s = std(vals);
-  console.log(`${dim.padEnd(20)} M=${mn.toFixed(2)}, SD=${s.toFixed(2)}, N=${vals.length}`);
-}
-
-// ============================================================
-// 5. Per-dimension per-model
-// ============================================================
-console.log('\n=== DIMENSION × MODEL BREAKDOWN ===');
-const header = 'Model'.padEnd(25) + dims.map(d => d.substring(0, 8).padStart(9)).join('');
-console.log(header);
-for (const m of modelNames) {
-  let line = m.padEnd(25);
-  for (const dim of dims) {
-    const col = `score_${dim}`;
-    const val = db.prepare(`
-      SELECT AVG(${col}) as v FROM evaluation_results
-      WHERE run_id = ? AND model = ? AND ${col} IS NOT NULL
-    `).get(RUN_ID, m);
-    line += (val?.v?.toFixed(2) || 'N/A').padStart(9);
-  }
-  console.log(line);
-}
-
-// ============================================================
-// 6. Scenario difficulty ranking
-// ============================================================
-console.log('\n=== SCENARIO DIFFICULTY RANKING (hardest → easiest) ===');
-const scenarioStats = db.prepare(`
-  SELECT scenario_id,
-    AVG(overall_score) as mean,
-    COUNT(*) as n
-  FROM evaluation_results
-  WHERE run_id = ? AND overall_score IS NOT NULL
-  GROUP BY scenario_id
-  ORDER BY mean ASC
-`).all(RUN_ID);
-
-for (const s of scenarioStats) {
-  const scores = db.prepare(`
-    SELECT overall_score FROM evaluation_results
-    WHERE run_id = ? AND scenario_id = ? AND overall_score IS NOT NULL
-  `).all(RUN_ID, s.scenario_id).map(r => r.overall_score);
-  const s_sd = std(scores);
-  const bar = '█'.repeat(Math.round(s.mean / 5));
-  console.log(`${s.scenario_id.padEnd(40)} ${s.mean.toFixed(1).padStart(5)} (SD=${s_sd.toFixed(1).padStart(5)}) ${bar}`);
-}
-
-// ============================================================
-// 7. Inter-model agreement (scenario-level correlation)
-// ============================================================
-console.log('\n=== INTER-MODEL AGREEMENT ===');
-const scenarios = db.prepare(`
-  SELECT DISTINCT scenario_id FROM evaluation_results
-  WHERE run_id = ? AND overall_score IS NOT NULL
-`).all(RUN_ID).map(r => r.scenario_id);
-
-// Check if models rank scenarios similarly
-for (let i = 0; i < modelNames.length; i++) {
-  for (let j = i + 1; j < modelNames.length; j++) {
-    const pairs = [];
-    for (const s of scenarios) {
-      const s1 = db.prepare(`SELECT overall_score FROM evaluation_results WHERE run_id = ? AND model = ? AND scenario_id = ? AND overall_score IS NOT NULL`).get(RUN_ID, modelNames[i], s);
-      const s2 = db.prepare(`SELECT overall_score FROM evaluation_results WHERE run_id = ? AND model = ? AND scenario_id = ? AND overall_score IS NOT NULL`).get(RUN_ID, modelNames[j], s);
-      if (s1 && s2) pairs.push([s1.overall_score, s2.overall_score]);
-    }
-    if (pairs.length >= 3) {
-      // Spearman rank correlation
-      const ranked = pairs.map(([a, b], idx) => ({ a, b, idx }));
-      ranked.sort((x, y) => x.a - y.a);
-      ranked.forEach((r, i) => r.rankA = i + 1);
-      ranked.sort((x, y) => x.b - y.b);
-      ranked.forEach((r, i) => r.rankB = i + 1);
-      const n = ranked.length;
-      const dSquared = ranked.reduce((sum, r) => sum + (r.rankA - r.rankB) ** 2, 0);
-      const rho = 1 - (6 * dSquared) / (n * (n * n - 1));
-      const agreement = rho > 0.7 ? 'strong' : rho > 0.4 ? 'moderate' : rho > 0 ? 'weak' : 'none';
-      console.log(`${modelNames[i]} vs ${modelNames[j]}: Spearman ρ=${rho.toFixed(2)} (${agreement} agreement, N=${n})`);
-    }
-  }
-}
-
-// ============================================================
-// 8. Base vs Recognition score analysis
-// ============================================================
-console.log('\n=== BASE vs RECOGNITION SCORE ANALYSIS ===');
-const dualRows = db.prepare(`
-  SELECT model, base_score, recognition_score, overall_score
-  FROM evaluation_results
-  WHERE run_id = ? AND base_score IS NOT NULL AND recognition_score IS NOT NULL
-`).all(RUN_ID);
-
-if (dualRows.length > 0) {
-  const bases = dualRows.map(r => r.base_score);
-  const recogs = dualRows.map(r => r.recognition_score);
-  const overalls = dualRows.map(r => r.overall_score);
-
-  console.log(`N (with both scores): ${dualRows.length}`);
-  console.log(`Base:        M=${(bases.reduce((a,b)=>a+b,0)/bases.length).toFixed(1)}, SD=${std(bases).toFixed(1)}`);
-  console.log(`Recognition: M=${(recogs.reduce((a,b)=>a+b,0)/recogs.length).toFixed(1)}, SD=${std(recogs).toFixed(1)}`);
-  console.log(`Overall:     M=${(overalls.reduce((a,b)=>a+b,0)/overalls.length).toFixed(1)}, SD=${std(overalls).toFixed(1)}`);
-
-  const gap = cohensD(bases, recogs);
-  console.log(`Base vs Recognition gap: d=${gap.toFixed(2)} (${gap > 0.8 ? 'large' : gap > 0.5 ? 'medium' : 'small'})`);
-
-  // Per-model breakdown
-  console.log('\nPer-model dual scores:');
-  for (const m of modelNames) {
-    const mRows = dualRows.filter(r => r.model === m);
-    if (mRows.length === 0) continue;
-    const mb = mRows.map(r => r.base_score);
-    const mr = mRows.map(r => r.recognition_score);
-    console.log(`  ${m}: Base=${(mb.reduce((a,b)=>a+b,0)/mb.length).toFixed(1)}, Recog=${(mr.reduce((a,b)=>a+b,0)/mr.length).toFixed(1)}, Gap=${((mb.reduce((a,b)=>a+b,0)/mb.length) - (mr.reduce((a,b)=>a+b,0)/mr.length)).toFixed(1)}, N=${mRows.length}`);
-  }
-} else {
-  console.log('No results with both base_score and recognition_score');
-}
-
-// ============================================================
-// 9. Variance decomposition (eta-squared)
-// ============================================================
-console.log('\n=== VARIANCE DECOMPOSITION ===');
-// How much variance is explained by model vs scenario?
-const grandMean = mean;
-const SSTotal = allScores.reduce((sum, s) => sum + (s - grandMean) ** 2, 0);
-
-// SS between models
-let SSModel = 0;
-for (const m of modelNames) {
-  const mScores = modelData[m];
-  const mMean = mScores.reduce((a, b) => a + b, 0) / mScores.length;
-  SSModel += mScores.length * (mMean - grandMean) ** 2;
-}
-
-// SS between scenarios
-let SSScenario = 0;
-for (const s of scenarioStats) {
-  const sMean = s.mean;
-  SSScenario += s.n * (sMean - grandMean) ** 2;
-}
-
-const etaModel = SSModel / SSTotal;
-const etaScenario = SSScenario / SSTotal;
-const etaResidual = 1 - etaModel - etaScenario;
-
-console.log(`Total SS: ${SSTotal.toFixed(1)}`);
-console.log(`Model effect (η²):    ${(etaModel * 100).toFixed(1)}% — ${etaModel < 0.01 ? 'negligible' : etaModel < 0.06 ? 'small' : etaModel < 0.14 ? 'medium' : 'large'}`);
-console.log(`Scenario effect (η²): ${(etaScenario * 100).toFixed(1)}% — ${etaScenario < 0.01 ? 'negligible' : etaScenario < 0.06 ? 'small' : etaScenario < 0.14 ? 'medium' : 'large'}`);
-console.log(`Residual:              ${(etaResidual * 100).toFixed(1)}%`);
-
-// ============================================================
-// 10. High-variance scenarios (discriminating power)
-// ============================================================
-console.log('\n=== SCENARIO DISCRIMINATING POWER (cross-model variance) ===');
-const scenarioVariance = [];
-for (const s of scenarios) {
-  const scores = db.prepare(`
-    SELECT overall_score FROM evaluation_results
-    WHERE run_id = ? AND scenario_id = ? AND overall_score IS NOT NULL
-  `).all(RUN_ID, s).map(r => r.overall_score);
-  if (scores.length >= 2) {
-    const sv = std(scores);
-    scenarioVariance.push({ id: s, sd: sv, range: Math.max(...scores) - Math.min(...scores) });
-  }
-}
-scenarioVariance.sort((a, b) => b.sd - a.sd);
-console.log('Scenario'.padEnd(40), 'SD'.padStart(6), 'Range'.padStart(7));
-for (const s of scenarioVariance) {
-  console.log(s.id.padEnd(40), s.sd.toFixed(1).padStart(6), s.range.toFixed(1).padStart(7));
-}
-
-console.log('\n=== ANALYSIS COMPLETE ===');
-db.close();

package/scripts/compare-runs.js

@@ -1,44 +0,0 @@
-#!/usr/bin/env node
-import fs from 'fs';
-
-function getScores(runId, scenario) {
-  const logPath = `./logs/eval-progress/${runId}.jsonl`;
-  const lines = fs.readFileSync(logPath, 'utf8').split('\n').filter(l => l.trim());
-  const events = lines.map(l => JSON.parse(l));
-  const successful = events.filter(e =>
-    e.eventType === 'test_complete' &&
-    e.success === true &&
-    e.overallScore != null &&
-    e.scenarioId === scenario
-  );
-
-  const byProfile = {};
-  for (const r of successful) {
-    if (!byProfile[r.profileName]) byProfile[r.profileName] = [];
-    byProfile[r.profileName].push(r.overallScore);
-  }
-  return byProfile;
-}
-
-const run1 = process.argv[2] || 'eval-2026-02-03-f5d4dd93';
-const run2 = process.argv[3] || 'eval-2026-02-03-b391d999';
-
-const scenarios = ['mood_frustration_to_breakthrough', 'misconception_correction_flow', 'mutual_transformation_journey'];
-
-for (const scenario of scenarios) {
-  console.log(`=== ${scenario} ===`);
-  const s1 = getScores(run1, scenario);
-  const s2 = getScores(run2, scenario);
-
-  const profiles = ['cell_1_base_single_unified', 'cell_3_base_multi_unified', 'cell_5_recog_single_unified', 'cell_7_recog_multi_unified'];
-
-  for (const p of profiles) {
-    const scores1 = s1[p] || [];
-    const scores2 = s2[p] || [];
-    const avg1 = scores1.length > 0 ? (scores1.reduce((a, b) => a + b, 0) / scores1.length).toFixed(1) : 'N/A';
-    const avg2 = scores2.length > 0 ? (scores2.reduce((a, b) => a + b, 0) / scores2.length).toFixed(1) : 'N/A';
-    const shortP = p.replace('cell_', '').replace('_single_unified', '').replace('_multi_unified', '_m');
-    console.log(`  ${shortP.padEnd(12)} ${run1.slice(-8)}=${avg1.padStart(5)} (${scores1.join(',').padEnd(20)})  ${run2.slice(-8)}=${avg2.padStart(5)} (${scores2.join(',')})`);
-  }
-  console.log('');
-}

package/scripts/compare-suggestions.js

@@ -1,80 +0,0 @@
-#!/usr/bin/env node
-import fs from 'fs';
-import path from 'path';
-
-const dir = 'logs/tutor-dialogues';
-const files = fs.readdirSync(dir).filter(f => f.endsWith('.json'));
-
-// Get recent files from Feb 3
-const recentFiles = files.filter(f => {
-  const stat = fs.statSync(path.join(dir, f));
-  const date = new Date(stat.mtime);
-  return date >= new Date('2026-02-03');
-});
-
-console.log(`Analyzing ${recentFiles.length} dialogue files from Feb 3+\n`);
-
-// Collect examples by profile and scenario
-const examples = { budget: [], recognition: [] };
-
-for (const f of recentFiles) {
-  try {
-    const d = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
-    const profile = d.profileName;
-    const scenario = d.scenario?.scenarioId || d.scenarioId || 'unknown';
-
-    if (profile !== 'budget' && profile !== 'recognition') continue;
-
-    // Only look at struggling_learner and concept_confusion
-    if (!scenario.includes('struggling') && !scenario.includes('confusion')) continue;
-
-    const suggestions = d.suggestions || [];
-    const firstSuggestion = suggestions[0];
-    if (!firstSuggestion) continue;
-
-    const title = firstSuggestion.title || '';
-    const message = firstSuggestion.message || '';
-    const fullText = (title + ' ' + message).toLowerCase();
-
-    const hasReview = fullText.includes('review');
-    const hasForbidden = ['next lecture', 'move on to', 'continue with'].some(fb => fullText.includes(fb));
-
-    examples[profile].push({
-      file: f,
-      scenario,
-      title,
-      messagePreview: message.substring(0, 150),
-      hasReview,
-      hasForbidden,
-      passed: hasReview && !hasForbidden
-    });
-  } catch (e) {}
-}
-
-// Show failing examples for each profile
-for (const profile of ['budget', 'recognition']) {
-  const data = examples[profile];
-  const passing = data.filter(x => x.passed);
-  const failing = data.filter(x => !x.passed);
-
-  console.log(`=== ${profile.toUpperCase()} ===`);
-  console.log(`Total: ${data.length}, Passing: ${passing.length}, Failing: ${failing.length}`);
-  console.log(`Pass rate: ${(passing.length / data.length * 100).toFixed(1)}%\n`);
-
-  console.log('FAILING examples (missing review or has forbidden):');
-  for (const ex of failing.slice(0, 5)) {
-    console.log(`  Scenario: ${ex.scenario}`);
-    console.log(`  Title: "${ex.title}"`);
-    console.log(`  Has review: ${ex.hasReview}, Has forbidden: ${ex.hasForbidden}`);
-    console.log(`  Preview: ${ex.messagePreview}...`);
-    console.log(`  File: ${ex.file}\n`);
-  }
-
-  console.log('PASSING examples:');
-  for (const ex of passing.slice(0, 3)) {
-    console.log(`  Scenario: ${ex.scenario}`);
-    console.log(`  Title: "${ex.title}"`);
-    console.log('');
-  }
-  console.log('');
-}

package/scripts/dig-into-run.js

@@ -1,158 +0,0 @@
-#!/usr/bin/env node
-import fs from 'fs';
-import path from 'path';
-
-const runId = process.argv[2] || 'eval-2026-02-03-f5d4dd93';
-const logPath = `./logs/eval-progress/${runId}.jsonl`;
-
-const lines = fs.readFileSync(logPath, 'utf8').split('\n').filter(l => l.trim());
-const events = lines.map(l => JSON.parse(l));
-const tests = events.filter(e => e.eventType === 'test_complete' && e.success && e.overallScore != null);
-
-// Group by profile and scenario
-const byProfileScenario = {};
-for (const t of tests) {
-  const key = t.profileName + '|' + t.scenarioId;
-  if (!byProfileScenario[key]) byProfileScenario[key] = [];
-  byProfileScenario[key].push(t.overallScore);
-}
-
-const scenarios = [...new Set(tests.map(t => t.scenarioId))];
-
-// Compare cell_2 vs cell_6 (both single+psycho, but base vs recog)
-console.log('=== cell_2 (base+single+psycho) vs cell_6 (recog+single+psycho) ===');
-console.log('Scenario'.padEnd(35), 'cell_2', 'cell_6', 'Delta');
-console.log('-'.repeat(60));
-let total2 = 0, total6 = 0, count = 0;
-for (const s of scenarios) {
-  const c2 = byProfileScenario['cell_2_base_single_psycho|' + s] || [];
-  const c6 = byProfileScenario['cell_6_recog_single_psycho|' + s] || [];
-  const avg2 = c2.length ? (c2.reduce((a,b)=>a+b,0)/c2.length) : null;
-  const avg6 = c6.length ? (c6.reduce((a,b)=>a+b,0)/c6.length) : null;
-  const delta = (avg2 !== null && avg6 !== null) ? (avg2 - avg6) : null;
-  console.log(
-    s.padEnd(35),
-    (avg2 !== null ? avg2.toFixed(1) : 'N/A').padStart(5),
-    (avg6 !== null ? avg6.toFixed(1) : 'N/A').padStart(5),
-    (delta !== null ? (delta > 0 ? '+' : '') + delta.toFixed(1) : '-').padStart(6)
-  );
-  if (avg2 !== null && avg6 !== null) {
-    total2 += avg2;
-    total6 += avg6;
-    count++;
-  }
-}
-console.log('-'.repeat(60));
-console.log('Average'.padEnd(35), (total2/count).toFixed(1).padStart(5), (total6/count).toFixed(1).padStart(5), ((total2-total6)/count > 0 ? '+' : '') + ((total2-total6)/count).toFixed(1).padStart(5));
-
-// Now compare all base vs all recognition
-console.log('\n=== All BASE profiles vs All RECOGNITION profiles ===');
-const baseProfiles = ['cell_1_base_single_unified', 'cell_2_base_single_psycho', 'cell_3_base_multi_unified', 'cell_4_base_multi_psycho'];
-const recogProfiles = ['cell_5_recog_single_unified', 'cell_6_recog_single_psycho', 'cell_7_recog_multi_unified', 'cell_8_recog_multi_psycho'];
-
-console.log('Scenario'.padEnd(35), 'Base', 'Recog', 'Delta');
-console.log('-'.repeat(60));
-let totalBase = 0, totalRecog = 0, countScen = 0;
-for (const s of scenarios) {
-  let baseScores = [], recogScores = [];
-  for (const p of baseProfiles) {
-    baseScores.push(...(byProfileScenario[p + '|' + s] || []));
-  }
-  for (const p of recogProfiles) {
-    recogScores.push(...(byProfileScenario[p + '|' + s] || []));
-  }
-  const avgBase = baseScores.length ? baseScores.reduce((a,b)=>a+b,0)/baseScores.length : null;
-  const avgRecog = recogScores.length ? recogScores.reduce((a,b)=>a+b,0)/recogScores.length : null;
-  const delta = (avgBase !== null && avgRecog !== null) ? (avgRecog - avgBase) : null;
-  console.log(
-    s.padEnd(35),
-    (avgBase !== null ? avgBase.toFixed(1) : 'N/A').padStart(5),
-    (avgRecog !== null ? avgRecog.toFixed(1) : 'N/A').padStart(5),
-    (delta !== null ? (delta > 0 ? '+' : '') + delta.toFixed(1) : '-').padStart(6)
-  );
-  if (avgBase !== null && avgRecog !== null) {
-    totalBase += avgBase;
-    totalRecog += avgRecog;
-    countScen++;
-  }
-}
-console.log('-'.repeat(60));
-console.log('Average'.padEnd(35), (totalBase/countScen).toFixed(1).padStart(5), (totalRecog/countScen).toFixed(1).padStart(5), ((totalRecog-totalBase)/countScen > 0 ? '+' : '') + ((totalRecog-totalBase)/countScen).toFixed(1).padStart(5));
-
-// Check raw scores for cell_2 to see variance
-console.log('\n=== cell_2 raw scores (to check variance) ===');
-for (const s of scenarios) {
-  const scores = byProfileScenario['cell_2_base_single_psycho|' + s] || [];
-  if (scores.length > 0) {
-    console.log(s.padEnd(35), scores.join(', '));
-  }
-}
-
-// Check which scenarios cell_2 wins vs loses
-console.log('\n=== Where cell_2 beats cell_6 vs where it loses ===');
-let wins = [], losses = [], ties = [];
-for (const s of scenarios) {
-  const c2 = byProfileScenario['cell_2_base_single_psycho|' + s] || [];
-  const c6 = byProfileScenario['cell_6_recog_single_psycho|' + s] || [];
-  if (c2.length && c6.length) {
-    const avg2 = c2.reduce((a,b)=>a+b,0)/c2.length;
-    const avg6 = c6.reduce((a,b)=>a+b,0)/c6.length;
-    const delta = avg2 - avg6;
-    if (delta > 5) wins.push({ scenario: s, delta });
-    else if (delta < -5) losses.push({ scenario: s, delta });
-    else ties.push({ scenario: s, delta });
-  }
-}
-console.log('cell_2 WINS (>5 pts):');
-wins.sort((a,b) => b.delta - a.delta).forEach(w => console.log(`  ${w.scenario}: +${w.delta.toFixed(1)}`));
-console.log('cell_2 LOSES (<-5 pts):');
-losses.sort((a,b) => a.delta - b.delta).forEach(l => console.log(`  ${l.scenario}: ${l.delta.toFixed(1)}`));
-console.log('TIES (±5 pts):');
-ties.forEach(t => console.log(`  ${t.scenario}: ${t.delta > 0 ? '+' : ''}${t.delta.toFixed(1)}`));
-
-// Analyze dialogue files for validation patterns
-console.log('\n=== Dialogue Validation Analysis ===');
-
-const dir = 'logs/tutor-dialogues';
-const files = fs.readdirSync(dir).filter(f => f.startsWith('dialogue-177008') || f.startsWith('dialogue-177009'));
-
-const forbidden = ['next lecture', 'move on to', 'continue with'];
-
-let results = { budget: [], recognition: [] };
-
-for (const f of files) {
-  try {
-    const d = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
-    const s = d.suggestions?.[0];
-    if (s == null) continue;
-
-    const userText = ((s.title || '') + ' ' + (s.message || '')).toLowerCase();
-    const fullText = JSON.stringify(d.suggestions).toLowerCase();
-
-    const hasReview = fullText.includes('review');
-    const hasForbidden = forbidden.some(fb => userText.includes(fb));
-
-    const profile = d.profileName === 'budget' ? 'budget' : d.profileName === 'recognition' ? 'recognition' : null;
-    if (profile) {
-      results[profile].push({
-        hasReview,
-        hasForbidden,
-        score: (hasReview ? 50 : 0) + (hasForbidden ? 0 : 50)
-      });
-    }
-  } catch (e) {}
-}
-
-for (const profile of ['budget', 'recognition']) {
-  const r = results[profile];
-  if (r.length === 0) continue;
-  const avgScore = r.reduce((a, b) => a + b.score, 0) / r.length;
-  const reviewPass = r.filter(x => x.hasReview).length;
-  const forbiddenPass = r.filter(x => x.hasForbidden === false).length;
-  console.log(profile + ':');
-  console.log('  Samples:', r.length);
-  console.log('  Review present (required):', reviewPass, '(' + (reviewPass/r.length*100).toFixed(1) + '%)');
-  console.log('  Forbidden absent:', forbiddenPass, '(' + (forbiddenPass/r.length*100).toFixed(1) + '%)');
-  console.log('  Avg validation score:', avgScore.toFixed(1));
-  console.log('');
-}

package/scripts/show-failed-suggestions.js

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-import fs from 'fs';
-import path from 'path';
-
-const dir = 'logs/tutor-dialogues';
-const files = fs.readdirSync(dir).filter(f => f.endsWith('.json'));
-
-// Get recent files
-const recentFiles = files.filter(f => {
-  const stat = fs.statSync(path.join(dir, f));
-  return new Date(stat.mtime) >= new Date('2026-02-03');
-});
-
-console.log(`Scanning ${recentFiles.length} dialogue files...\n`);
-
-// Find recognition profile dialogues
-let recognitionExamples = [];
-
-for (const f of recentFiles) {
-  try {
-    const d = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
-    if (d.profileName !== 'recognition') continue;
-
-    const suggestions = d.suggestions || [];
-    if (suggestions.length === 0) continue;
-
-    const first = suggestions[0];
-    const text = ((first.title || '') + ' ' + (first.message || '')).toLowerCase();
-    const hasReview = text.includes('review');
-
-    // Store suggestion content
-    recognitionExamples.push({
-      file: f,
-      title: first.title || '',
-      message: first.message || '',
-      hasReview,
-      text
-    });
-  } catch (e) {}
-}
-
-// Show examples without "review"
-const failingExamples = recognitionExamples.filter(e => !e.hasReview);
-const passingExamples = recognitionExamples.filter(e => e.hasReview);
-
-console.log(`Recognition profile: ${recognitionExamples.length} total dialogues`);
-console.log(`  With "review": ${passingExamples.length}`);
-console.log(`  Without "review": ${failingExamples.length}\n`);
-
-console.log('=== FAILING EXAMPLES (no "review" in text) ===\n');
-for (const ex of failingExamples.slice(0, 6)) {
-  console.log(`File: ${ex.file}`);
-  console.log(`Title: "${ex.title}"`);
-  console.log(`Message: ${ex.message.substring(0, 300)}...`);
-  console.log('---\n');
-}
-
-console.log('=== PASSING EXAMPLES (has "review") ===\n');
-for (const ex of passingExamples.slice(0, 3)) {
-  console.log(`File: ${ex.file}`);
-  console.log(`Title: "${ex.title}"`);
-  console.log(`Message: ${ex.message.substring(0, 200)}...`);
-  console.log('---\n');
-}