@machinespirits/eval 0.1.2 → 0.2.0

package/scripts/seed-db.js

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+/**
+ * Seed Database
+ *
+ * Creates a small sample dataset so new users can explore the CLI
+ * (runs, report, export) without running a full evaluation.
+ *
+ * Usage: node scripts/seed-db.js
+ */
+
+import * as evaluationStore from '../services/evaluationStore.js';
+
+const SEED_RUN_ID = 'seed-sample-factorial';
+
+// Check if seed data already exists
+const existing = evaluationStore.getRun(SEED_RUN_ID);
+if (existing) {
+  console.log(`Seed run '${SEED_RUN_ID}' already exists. Delete it first to re-seed.`);
+  process.exit(0);
+}
+
+console.log('Creating seed evaluation run...');
+
+// Insert run directly (createRun auto-generates IDs, so use the store's db)
+const run = evaluationStore.createRun({
+  description: 'Sample 2x2x2 factorial (seed data for demonstration)',
+  totalScenarios: 1,
+  totalConfigurations: 8,
+});
+
+// We need the auto-generated ID — use it going forward
+const runId = run.id;
+
+// 8 factorial cells with representative scores (matching paper Table 5 means)
+const cells = [
+  { profile: 'cell_1_base_single_unified',   recog: false, multi: false, learner: 'unified',       score: 77.6 },
+  { profile: 'cell_2_base_single_psycho',     recog: false, multi: false, learner: 'ego_superego',  score: 80.0 },
+  { profile: 'cell_3_base_multi_unified',     recog: false, multi: true,  learner: 'unified',       score: 76.6 },
+  { profile: 'cell_4_base_multi_psycho',      recog: false, multi: true,  learner: 'ego_superego',  score: 81.5 },
+  { profile: 'cell_5_recog_single_unified',   recog: true,  multi: false, learner: 'unified',       score: 92.8 },
+  { profile: 'cell_6_recog_single_psycho',    recog: true,  multi: false, learner: 'ego_superego',  score: 83.4 },
+  { profile: 'cell_7_recog_multi_unified',    recog: true,  multi: true,  learner: 'unified',       score: 92.3 },
+  { profile: 'cell_8_recog_multi_psycho',     recog: true,  multi: true,  learner: 'ego_superego',  score: 86.7 },
+];
+
+for (const cell of cells) {
+  const base = cell.score / 20; // scale 0-100 → approx 1-5
+  evaluationStore.storeResult(runId, {
+    scenarioId: 'struggling_learner',
+    scenarioName: 'Struggling Learner',
+    provider: 'openrouter',
+    model: 'moonshotai/kimi-k2.5',
+    profileName: cell.profile,
+    suggestions: [{
+      type: 'review',
+      priority: 'high',
+      title: 'Sample suggestion',
+      message: `Sample ${cell.recog ? 'recognition-theory' : 'base'} tutor response for a struggling learner.`,
+    }],
+    latencyMs: 5000 + Math.floor(Math.random() * 10000),
+    scores: {
+      relevance: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+      specificity: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+      pedagogical: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+      personalization: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+      actionability: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+      tone: Math.min(5, Math.max(1, Math.round(base + (Math.random() - 0.5)))),
+    },
+    overallScore: cell.score,
+    judgeModel: 'seed-data',
+    success: true,
+    factors: {
+      recognition: cell.recog,
+      multi_agent_tutor: cell.multi,
+      multi_agent_learner: cell.learner === 'ego_superego',
+    },
+    learnerArchitecture: cell.learner,
+  });
+}
+
+// Mark run complete
+evaluationStore.completeRun(runId);
+
+console.log(`Seed run created: ${runId}`);
+console.log('  8 factorial cells, 1 scenario each');
+console.log(`  Try: node scripts/eval-cli.js runs`);
+console.log(`  Try: node scripts/eval-cli.js report ${runId}`);

package/scripts/show-failed-suggestions.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import path from 'path';
+
+const dir = 'logs/tutor-dialogues';
+const files = fs.readdirSync(dir).filter(f => f.endsWith('.json'));
+
+// Get recent files
+const recentFiles = files.filter(f => {
+  const stat = fs.statSync(path.join(dir, f));
+  return new Date(stat.mtime) >= new Date('2026-02-03');
+});
+
+console.log(`Scanning ${recentFiles.length} dialogue files...\n`);
+
+// Find recognition profile dialogues
+let recognitionExamples = [];
+
+for (const f of recentFiles) {
+  try {
+    const d = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
+    if (d.profileName !== 'recognition') continue;
+
+    const suggestions = d.suggestions || [];
+    if (suggestions.length === 0) continue;
+
+    const first = suggestions[0];
+    const text = ((first.title || '') + ' ' + (first.message || '')).toLowerCase();
+    const hasReview = text.includes('review');
+
+    // Store suggestion content
+    recognitionExamples.push({
+      file: f,
+      title: first.title || '',
+      message: first.message || '',
+      hasReview,
+      text
+    });
+  } catch (e) {}
+}
+
+// Show examples without "review"
+const failingExamples = recognitionExamples.filter(e => !e.hasReview);
+const passingExamples = recognitionExamples.filter(e => e.hasReview);
+
+console.log(`Recognition profile: ${recognitionExamples.length} total dialogues`);
+console.log(`  With "review": ${passingExamples.length}`);
+console.log(`  Without "review": ${failingExamples.length}\n`);
+
+console.log('=== FAILING EXAMPLES (no "review" in text) ===\n');
+for (const ex of failingExamples.slice(0, 6)) {
+  console.log(`File: ${ex.file}`);
+  console.log(`Title: "${ex.title}"`);
+  console.log(`Message: ${ex.message.substring(0, 300)}...`);
+  console.log('---\n');
+}
+
+console.log('=== PASSING EXAMPLES (has "review") ===\n');
+for (const ex of passingExamples.slice(0, 3)) {
+  console.log(`File: ${ex.file}`);
+  console.log(`Title: "${ex.title}"`);
+  console.log(`Message: ${ex.message.substring(0, 200)}...`);
+  console.log('---\n');
+}

package/scripts/validate-content.js

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+
+/**
+ * Content Validation CLI
+ *
+ * Validates that the content package is accessible and all lectures load
+ * correctly. Also previews parsed content for debugging.
+ *
+ * Usage:
+ *   node scripts/validate-content.js                        # Validate all content
+ *   node scripts/validate-content.js --lecture 479-lecture-3 # Show parsed lecture
+ *   node scripts/validate-content.js --preview 479-lecture-3 # Show full curriculum context
+ *   node scripts/validate-content.js --scenarios             # Check all scenarios' content refs
+ */
+
+import * as contentResolver from '../services/contentResolver.js';
+import * as evalConfigLoader from '../services/evalConfigLoader.js';
+
+// ── Helpers ────────────────────────────────────────────────────────────────────
+
+function initContentResolver() {
+  const contentConfig = evalConfigLoader.getContentConfig();
+  if (!contentConfig?.content_package_path) {
+    console.error('Error: No content.content_package_path in config/eval-settings.yaml');
+    process.exit(1);
+  }
+
+  contentResolver.configure({
+    contentPackagePath: contentConfig.content_package_path,
+    maxLectureChars: contentConfig.max_lecture_chars,
+    includeSpeakerNotes: contentConfig.include_speaker_notes,
+  });
+
+  if (!contentResolver.isConfigured()) {
+    console.error(`Error: Content directory not found at: ${contentConfig.content_package_path}`);
+    process.exit(1);
+  }
+
+  return contentConfig;
+}
+
+// ── Commands ──────────────────────────────────────────────────────────────────
+
+function validateAll() {
+  const config = initContentResolver();
+  console.log(`Content package: ${config.content_package_path}`);
+  console.log('');
+
+  const courses = contentResolver.listAvailableCourses();
+  console.log(`Found ${courses.length} course(s): ${courses.join(', ')}`);
+  console.log('');
+
+  for (const courseId of courses) {
+    const meta = contentResolver.loadCourseMeta(courseId);
+    if (meta) {
+      console.log(`  [${courseId}] ${meta.title || '(no title)'}`);
+      if (meta.instructor) console.log(`         Instructor: ${meta.instructor}`);
+      if (meta.objectives?.length) console.log(`         Objectives: ${meta.objectives.length}`);
+    } else {
+      console.log(`  [${courseId}] ERROR: could not load course.md`);
+    }
+  }
+
+  console.log('');
+  const errors = contentResolver.validateContent();
+  if (errors.length === 0) {
+    console.log('Validation PASSED - all content loads correctly.');
+  } else {
+    console.log(`Validation FAILED - ${errors.length} error(s):`);
+    for (const err of errors) {
+      console.log(`  - ${err}`);
+    }
+    process.exit(1);
+  }
+}
+
+function showLecture(lectureRef) {
+  initContentResolver();
+
+  const raw = contentResolver.loadLecture(lectureRef);
+  if (!raw) {
+    console.error(`Error: Could not load lecture "${lectureRef}"`);
+    process.exit(1);
+  }
+
+  const parsed = contentResolver.parseLectureMarkdown(raw);
+  console.log(`Lecture: ${lectureRef}`);
+  console.log(`Total characters: ${raw.length}`);
+  console.log(`Slides: ${parsed.slides.length}`);
+  console.log(`Speaker notes blocks: ${parsed.notes.length}`);
+  console.log('');
+
+  for (let i = 0; i < parsed.slides.length; i++) {
+    const slide = parsed.slides[i];
+    const preview = slide.slice(0, 120).replace(/\n/g, ' ');
+    console.log(`  Slide ${i + 1}: ${preview}${slide.length > 120 ? '...' : ''}`);
+  }
+
+  if (parsed.notes.length > 0) {
+    console.log('');
+    console.log('Speaker Notes:');
+    for (let i = 0; i < parsed.notes.length; i++) {
+      const preview = parsed.notes[i].slice(0, 100).replace(/\n/g, ' ');
+      console.log(`  [${i + 1}] ${preview}${parsed.notes[i].length > 100 ? '...' : ''}`);
+    }
+  }
+}
+
+function previewCurriculum(lectureRef) {
+  initContentResolver();
+
+  const context = contentResolver.buildCurriculumContext({
+    currentContent: lectureRef,
+  });
+
+  if (!context) {
+    console.error(`Error: Could not build curriculum context for "${lectureRef}"`);
+    process.exit(1);
+  }
+
+  console.log(`Curriculum context for: ${lectureRef}`);
+  console.log(`Total characters: ${context.length}`);
+  console.log('='.repeat(80));
+  console.log(context);
+  console.log('='.repeat(80));
+}
+
+function checkScenarios() {
+  initContentResolver();
+
+  const scenarios = evalConfigLoader.listScenarios();
+  console.log(`Checking ${scenarios.length} scenario(s) for content references...\n`);
+
+  let resolved = 0;
+  let unresolved = 0;
+  let noContent = 0;
+
+  for (const scenarioMeta of scenarios) {
+    const scenario = evalConfigLoader.getScenario(scenarioMeta.id);
+    const { currentContent } = contentResolver.resolveScenarioContent(scenario);
+
+    if (currentContent) {
+      const raw = contentResolver.loadLecture(currentContent);
+      if (raw) {
+        console.log(`  [OK]   ${scenarioMeta.id} → ${currentContent} (${raw.length} chars)`);
+        resolved++;
+      } else {
+        console.log(`  [FAIL] ${scenarioMeta.id} → ${currentContent} (NOT FOUND)`);
+        unresolved++;
+      }
+    } else {
+      console.log(`  [NONE] ${scenarioMeta.id} → no content reference (will use course overview only)`);
+      noContent++;
+    }
+  }
+
+  console.log('');
+  console.log(`Results: ${resolved} resolved, ${unresolved} failed, ${noContent} no content ref`);
+
+  if (unresolved > 0) {
+    process.exit(1);
+  }
+}
+
+// ── Main ──────────────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  console.log(`Usage:
+  node scripts/validate-content.js                        # Validate all content
+  node scripts/validate-content.js --lecture 479-lecture-3 # Show parsed lecture
+  node scripts/validate-content.js --preview 479-lecture-3 # Show full curriculum context
+  node scripts/validate-content.js --scenarios             # Check all scenarios' content refs`);
+  process.exit(0);
+}
+
+if (args.includes('--lecture')) {
+  const idx = args.indexOf('--lecture');
+  const ref = args[idx + 1];
+  if (!ref) { console.error('Missing lecture ref'); process.exit(1); }
+  showLecture(ref);
+} else if (args.includes('--preview')) {
+  const idx = args.indexOf('--preview');
+  const ref = args[idx + 1];
+  if (!ref) { console.error('Missing lecture ref'); process.exit(1); }
+  previewCurriculum(ref);
+} else if (args.includes('--scenarios')) {
+  checkScenarios();
+} else {
+  validateAll();
+}

package/server.js

@@ -20,10 +20,11 @@
 import express from 'express';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { existsSync, mkdirSync } from 'fs';
+import { existsSync, mkdirSync, readFileSync } from 'fs';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
+const pkg = JSON.parse(readFileSync(path.join(__dirname, 'package.json'), 'utf-8'));
 
 const app = express();
 const PORT = Number(process.env.PORT) || 8081;
@@ -44,7 +45,7 @@ app.get('/health', (req, res) => {
   res.json({
     status: 'ok',
     package: '@machinespirits/eval',
-    version: '0.1.0',
+    version: pkg.version,
     mode: isStandalone ? 'standalone' : 'mounted',
   });
 });