@machinespirits/eval 0.2.0 → 0.3.0

package/scripts/download-data.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Download evaluation databases from the GitHub release.
+# Creates symlinks in data/ pointing to ~/.machinespirits-data/.
+#
+# Usage: ./scripts/download-data.sh [--tag v2.3.14]
+#
+# Requires: gh (GitHub CLI), authenticated
+
+set -euo pipefail
+
+REPO="liammagee/machinespirits-eval"
+TAG="${1:-v2.3.14}"
+DATA_DIR="$HOME/.machinespirits-data"
+LINK_DIR="$(cd "$(dirname "$0")/.." && pwd)/data"
+
+# Strip --tag prefix if provided
+TAG="${TAG#--tag }"
+TAG="${TAG#--tag=}"
+
+echo "Downloading evaluation databases from release ${TAG}..."
+echo "  Target: ${DATA_DIR}"
+echo ""
+
+mkdir -p "${DATA_DIR}"
+
+DB_FILES=(evaluations.db learner-writing-pad.db tutor-writing-pad.db writing-pads.db)
+
+for f in "${DB_FILES[@]}"; do
+  if [ -f "${DATA_DIR}/${f}" ]; then
+    echo "  [skip] ${f} (already exists)"
+  else
+    echo "  [download] ${f}..."
+    gh release download "${TAG}" --repo "${REPO}" --pattern "${f}" --dir "${DATA_DIR}"
+  fi
+done
+
+echo ""
+echo "Creating symlinks in ${LINK_DIR}/..."
+
+for f in "${DB_FILES[@]}"; do
+  target="${DATA_DIR}/${f}"
+  link="${LINK_DIR}/${f}"
+  if [ -L "${link}" ]; then
+    echo "  [skip] ${f} (symlink exists)"
+  elif [ -f "${link}" ]; then
+    echo "  [skip] ${f} (regular file exists — remove manually if you want a symlink)"
+  else
+    ln -s "${target}" "${link}"
+    echo "  [link] ${f} -> ${target}"
+  fi
+done
+
+echo ""
+echo "Done. Verify with: node -e \"import Database from 'better-sqlite3'; const db = new Database('data/evaluations.db'); console.log(db.prepare('SELECT COUNT(*) as n FROM evaluation_results').get())\""

package/scripts/eval-cli.js

@@ -44,6 +44,7 @@ import 'dotenv/config';
  *   --refresh <ms>         Refresh interval for 'watch' (default: 2000) or 'evaluate --follow' (default: 5000)
  *   --force                Actually complete stale runs (for 'cleanup'; dry-run without it)
  *   --older-than <min>     Staleness threshold in minutes (for 'cleanup', default: 30)
+ *   --dry-run              Use mock data instead of API calls (no API keys required)
  *
  * The default `run` uses the 2x2x2 factorial design:
  *   Factor A: Recognition prompts (off / on)
@@ -68,6 +69,7 @@ import { buildLearnerEvaluationPrompt, calculateLearnerOverallScore } from '../s
 import { readProgressLog, getProgressLogPath } from '../services/progressLogger.js';
 import * as evalConfigLoader from '../services/evalConfigLoader.js';
 const { getScenario } = evalConfigLoader;
+import { formatTranscript } from '../services/transcriptFormatter.js';
 import { spawn } from 'child_process';
 import readline from 'readline';
 import fs from 'fs';
@@ -767,15 +769,17 @@ async function main() {
         const scenarioId = getOption('scenario', 'new_user_first_visit');
         const profile = getOption('profile', 'budget');
         const verbose = getFlag('verbose');
+        const dryRun = getFlag('dry-run');
         const evalSettingsQt = evalConfigLoader.getEvalSettings();
-        const skipRubricEval = getFlag('skip-rubric') || !evalSettingsQt.useAIJudge;
+        const skipRubricEval = dryRun ? false : (getFlag('skip-rubric') || !evalSettingsQt.useAIJudge);
         const config = { profileName: profile };
 
-        console.log(`\nRunning quick test (profile: ${profile}, scenario: ${scenarioId})...\n`);
+        console.log(`\nRunning quick test (profile: ${profile}, scenario: ${scenarioId}${dryRun ? ', dry-run' : ''})...\n`);
         const result = await evaluationRunner.quickTest(config, {
           scenarioId,
           verbose,
           skipRubricEval,
+          dryRun,
         });
         console.log('\nResult:');
         console.log(JSON.stringify(result, null, 2));
@@ -784,9 +788,11 @@ async function main() {
 
       case 'run': {
         const verbose = getFlag('verbose');
+        const dryRun = getFlag('dry-run');
         // CLI --use-rubric forces rubric on; --skip-rubric forces off; otherwise use config default
+        // --dry-run always enables rubric (mock judge has no cost)
         const evalSettings = evalConfigLoader.getEvalSettings();
-        const skipRubricEval = getFlag('use-rubric') ? false : (getFlag('skip-rubric') || !evalSettings.useAIJudge);
+        const skipRubricEval = dryRun ? false : (getFlag('use-rubric') ? false : (getFlag('skip-rubric') || !evalSettings.useAIJudge));
         const runsPerConfig = parseInt(getOption('runs', '1'), 10);
         const parallelism = parseInt(getOption('parallelism', '2'), 10);
         const description = getOption('description');
@@ -796,6 +802,9 @@ async function main() {
         const modelOverride = getOption('model');
         const egoModelOverride = getOption('ego-model');
         const superegoModelOverride = getOption('superego-model');
+        const learnerModelOverride = getOption('learner-model');
+        const transcriptMode = getFlag('transcript');
+        const maxTokensOverride = getOption('max-tokens');
 
         // --cluster and --scenario are mutually exclusive
         if (clusterOpt && scenarioOpt) {
@@ -847,6 +856,8 @@ async function main() {
             if (egoModelOverride) console.log(`  Ego model override: ${egoModelOverride}`);
             if (superegoModelOverride) console.log(`  Superego model override: ${superegoModelOverride}`);
           }
+          if (learnerModelOverride) console.log(`  Learner model override: ${learnerModelOverride}`);
+          if (maxTokensOverride) console.log(`  Max tokens override: ${maxTokensOverride}`);
           console.log('');
         }
 
@@ -860,12 +871,16 @@ async function main() {
           runsPerConfig,
           parallelism,
           skipRubricEval,
-          description: description || (isFactorial ? '2x2x2 Factorial Evaluation' : null),
+          description: description || (dryRun ? 'Dry-run evaluation (mock data)' : (isFactorial ? '2x2x2 Factorial Evaluation' : null)),
           verbose,
           scenarioFilter: clusterOpt || null,
           modelOverride: modelOverride || null,
           egoModelOverride: egoModelOverride || null,
           superegoModelOverride: superegoModelOverride || null,
+          learnerModelOverride: learnerModelOverride || null,
+          dryRun,
+          transcriptMode,
+          maxTokensOverride: maxTokensOverride ? parseInt(maxTokensOverride, 10) : null,
         });
         // Extract unique model aliases used across all configs (ego + superego)
         const extractAlias = (raw) => {
@@ -884,6 +899,63 @@ async function main() {
         if (modelAliases.length > 0) {
           console.log(`Models: ${modelAliases.join(', ')}`);
         }
+
+        // Token / cost / latency summary report
+        if (result.runId) {
+          const runResults = evaluationStore.getResults(result.runId);
+          if (runResults.length > 0) {
+            console.log('\n' + '='.repeat(80));
+            console.log('  TOKEN & COST SUMMARY');
+            console.log('='.repeat(80));
+
+            // Per-result breakdown
+            const header = '  #  | Scenario                         | In Tok  | Out Tok | API  | Rounds | Latency   | Cost';
+            const divider = '  ' + '-'.repeat(header.length - 2);
+            console.log(header);
+            console.log(divider);
+
+            let totalIn = 0, totalOut = 0, totalApi = 0, totalRounds = 0, totalLatency = 0, totalCost = 0;
+
+            runResults.forEach((r, i) => {
+              const inTok = r.input_tokens || r.inputTokens || 0;
+              const outTok = r.output_tokens || r.outputTokens || 0;
+              const apiCalls = r.api_calls || r.apiCalls || 0;
+              const rounds = r.dialogue_rounds || r.dialogueRounds || 0;
+              const latMs = r.latency_ms || r.latencyMs || 0;
+              const cost = r.cost || 0;
+
+              totalIn += inTok;
+              totalOut += outTok;
+              totalApi += apiCalls;
+              totalRounds += rounds;
+              totalLatency += latMs;
+              totalCost += cost;
+
+              const scenLabel = (r.scenario_id || r.scenarioId || '').substring(0, 32).padEnd(32);
+              const latStr = latMs >= 1000 ? `${(latMs / 1000).toFixed(1)}s` : `${latMs}ms`;
+              const costStr = cost > 0 ? `$${cost.toFixed(4)}` : '-';
+              console.log(`  ${String(i + 1).padStart(2)} | ${scenLabel} | ${String(inTok).padStart(7)} | ${String(outTok).padStart(7)} | ${String(apiCalls).padStart(4)} | ${String(rounds).padStart(6)} | ${latStr.padStart(9)} | ${costStr}`);
+            });
+
+            console.log(divider);
+            const totalLatStr = totalLatency >= 1000 ? `${(totalLatency / 1000).toFixed(1)}s` : `${totalLatency}ms`;
+            const totalCostStr = totalCost > 0 ? `$${totalCost.toFixed(4)}` : '-';
+            console.log(`  ${'TOTAL'.padStart(2)} | ${''.padEnd(32)} | ${String(totalIn).padStart(7)} | ${String(totalOut).padStart(7)} | ${String(totalApi).padStart(4)} | ${String(totalRounds).padStart(6)} | ${totalLatStr.padStart(9)} | ${totalCostStr}`);
+
+            // Per-token cost efficiency
+            const totalTok = totalIn + totalOut;
+            if (totalTok > 0) {
+              const avgLatPerCall = totalApi > 0 ? (totalLatency / totalApi / 1000).toFixed(2) : '-';
+              console.log(`\n  Tokens: ${totalTok.toLocaleString()} total (${totalIn.toLocaleString()} in + ${totalOut.toLocaleString()} out)`);
+              console.log(`  Avg latency/API call: ${avgLatPerCall}s  |  Results: ${runResults.length}  |  API calls: ${totalApi}`);
+              if (totalCost > 0) {
+                console.log(`  Cost/1K tokens: $${(totalCost / totalTok * 1000).toFixed(4)}`);
+              }
+            }
+            console.log('='.repeat(80));
+          }
+        }
+
         console.log(JSON.stringify(result, null, 2));
 
         // Factorial post-analysis: print cell means and ANOVA for each score type
@@ -964,6 +1036,11 @@ async function main() {
           } else if (run.completedResults > 0) {
             progress = `${run.completedResults} done`;
           }
+          // Show per-turn progress for running multi-turn tests
+          const turnProgress = run.metadata?.turnProgress;
+          if (run.status === 'running' && turnProgress) {
+            progress += ` T${turnProgress.current}/${turnProgress.total}`;
+          }
           const avg = run.avgScore != null ? run.avgScore.toFixed(1) : '--';
           // Duration formatting
           let duration = '--';
@@ -1223,11 +1300,18 @@ async function main() {
       case 'transcript': {
         const runId = args.find(a => !a.startsWith('--') && a !== 'transcript');
         if (!runId) {
-          console.error('Usage: eval-cli.js transcript <runId> [--scenario <id>]');
+          console.error('Usage: eval-cli.js transcript <runId> [--scenario <id>] [--detail play|compact|messages-only|full|bilateral]');
           process.exit(1);
         }
 
         const scenarioFilter = getOption('scenario');
+        // Determine detail level: --compact and --messages-only are shortcuts, --detail is explicit
+        let detailLevel = getOption('detail') || 'play';
+        if (getFlag('compact')) detailLevel = 'compact';
+        if (getFlag('messages-only')) detailLevel = 'messages-only';
+        if (getFlag('full')) detailLevel = 'full';
+        if (getFlag('bilateral')) detailLevel = 'bilateral';
+
         const results = evaluationStore.getResults(runId, {
           scenarioId: scenarioFilter || null,
         });
@@ -1237,7 +1321,7 @@ async function main() {
           break;
         }
 
-        console.log(`\nTranscripts for run: ${runId} (${results.length} results)\n`);
+        console.log(`\nTranscripts for run: ${runId} (${results.length} results, detail: ${detailLevel})\n`);
 
         for (const result of results) {
           console.log('='.repeat(80));
@@ -1246,10 +1330,9 @@ async function main() {
           console.log(`Score:    ${result.overallScore != null ? result.overallScore.toFixed(1) : '--'}  |  Success: ${result.success}`);
           console.log('-'.repeat(80));
 
-          // Try dialogue log file first
+          // Try dialogue log file first (rich trace with metadata)
           let printed = false;
           if (result.dialogueId) {
-            // Search for the dialogue file (may include date prefix in filename)
             const files = fs.existsSync(LOGS_DIR)
               ? fs.readdirSync(LOGS_DIR).filter(f => f.includes(result.dialogueId))
               : [];
@@ -1258,24 +1341,29 @@ async function main() {
               try {
                 const dialogue = JSON.parse(fs.readFileSync(path.join(LOGS_DIR, files[0]), 'utf-8'));
                 const trace = dialogue.dialogueTrace || [];
-                for (const entry of trace) {
-                  console.log(formatTraceEntry(entry));
-                  console.log('');
+                if (trace.length > 0) {
+                  const formatted = formatTranscript(trace, {
+                    detail: detailLevel,
+                    scenarioName: result.scenarioName || result.scenarioId,
+                    profileName: result.profileName,
+                    totalTurns: dialogue.totalTurns || 0,
+                  });
+                  console.log(formatted);
+                  printed = true;
                 }
-                if (trace.length > 0) printed = true;
               } catch (e) {
-                // Fall through to suggestions
+                // Fall through to legacy format
               }
             }
           }
 
-          // Fall back to suggestions / raw response from DB
+          // Fall back to legacy format (suggestions / raw response from DB)
           if (!printed) {
             if (result.suggestions?.length > 0) {
               console.log('Suggestions:');
               for (const s of result.suggestions) {
                 const text = typeof s === 'string' ? s : (s.text || s.content || JSON.stringify(s));
-                console.log(`  • ${text}`);
+                console.log(`  \u2022 ${text}`);
               }
               console.log('');
             }
@@ -1831,7 +1919,7 @@ async function main() {
             requiredMissing: parsed.validation?.required_missing || [],
             forbiddenFound: parsed.validation?.forbidden_found || [],
             summary: parsed.summary,
-            judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-code/opus',
+            judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-opus-4.6',
           };
 
           evaluationStore.updateResultScores(result.id, evaluation);
@@ -2023,7 +2111,7 @@ async function main() {
                 recognitionScore,
                 scores: normalizedScores,
                 summary: parsed.summary,
-                judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-code/opus',
+                judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-opus-4.6',
               };
 
               // Save to dialogue log
@@ -2578,7 +2666,7 @@ async function main() {
             evaluationStore.updateResultLearnerScores(result.id, {
               scores: turnScores,
               overallScore: dialogueLearnerScore,
-              judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-code/opus',
+              judgeModel: modelOverride ? `claude-code/${modelOverride}` : 'claude-opus-4.6',
             });
 
             allScores.push(dialogueLearnerScore);