@machinespirits/eval 0.1.2 → 0.2.1

package/services/learnerTutorInteractionEngine.js

@@ -847,11 +847,386 @@ function calculateMemoryDelta(before, after) {
   };
 }
 
+// ============================================================================
+// Standalone Learner Response (for evaluation pipeline)
+// ============================================================================
+
+// Retry delays for 429 rate limits (matches evaluationRunner pattern)
+const LEARNER_RETRY_DELAYS = [2000, 4000, 8000];
+
+/**
+ * Call the LLM for a learner agent using the same raw fetch layer as
+ * tutorDialogueEngine.callAI — same headers, error handling, and response
+ * parsing per provider. This ensures learner and tutor calls go through
+ * identical network code paths.
+ *
+ * Includes built-in retry with exponential backoff for 429 rate limits.
+ *
+ * @param {Object} agentConfig - From learnerConfig.getAgentConfig()
+ * @param {string} systemPrompt - Static system/persona prompt (cacheable)
+ * @param {string} userPrompt - Dynamic per-call user content
+ * @param {string} agentRole - For logging (e.g. 'ego', 'superego', 'synthesis')
+ * @returns {Promise<Object>} { content, usage: { inputTokens, outputTokens }, latencyMs }
+ */
+async function callLearnerAI(agentConfig, systemPrompt, userPrompt, agentRole = 'learner') {
+  let lastError;
+  for (let attempt = 0; attempt <= LEARNER_RETRY_DELAYS.length; attempt++) {
+    try {
+      return await _callLearnerAIOnce(agentConfig, systemPrompt, userPrompt, agentRole);
+    } catch (error) {
+      lastError = error;
+      const is429 = error?.message?.includes('429') ||
+                    error?.message?.toLowerCase()?.includes('rate limit');
+      if (!is429 || attempt >= LEARNER_RETRY_DELAYS.length) throw error;
+      const delay = LEARNER_RETRY_DELAYS[attempt];
+      console.warn(`[${agentRole}] Rate limit hit, retrying in ${delay}ms (attempt ${attempt + 1}/${LEARNER_RETRY_DELAYS.length})`);
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  throw lastError;
+}
+
+/**
+ * Single-attempt LLM call. Mirrors tutorDialogueEngine.callAI per-provider
+ * fetch logic: same headers, same body format, same error parsing.
+ * Accepts system and user prompts separately for provider-level caching.
+ */
+async function _callLearnerAIOnce(agentConfig, systemPrompt, userPrompt, agentRole) {
+  const { provider, providerConfig, model, hyperparameters = {} } = agentConfig;
+  let { temperature = 0.7, max_tokens = 300, top_p } = hyperparameters;
+
+  // Thinking models (kimi-k2.5, deepseek-r1, etc.) use reasoning tokens that consume
+  // the max_tokens budget. Increase significantly to allow for both reasoning and output.
+  const isThinkingModel = model?.includes('kimi-k2') || model?.includes('deepseek-r1');
+  if (isThinkingModel && max_tokens < 2000) {
+    max_tokens = 2000;
+  }
+
+  if (!providerConfig?.isConfigured) {
+    throw new Error(`Learner provider ${provider} not configured (missing API key)`);
+  }
+
+  const startTime = Date.now();
+
+  // --- Anthropic ---
+  if (provider === 'anthropic') {
+    const bodyParams = {
+      model,
+      max_tokens,
+      temperature,
+      system: systemPrompt,
+      messages: [{ role: 'user', content: userPrompt }],
+    };
+    if (top_p !== undefined) {
+      delete bodyParams.temperature;
+      bodyParams.top_p = top_p;
+    }
+
+    const res = await fetch(providerConfig.base_url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': providerConfig.apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify(bodyParams),
+    });
+
+    if (!res.ok) {
+      const data = await res.json().catch(() => ({}));
+      throw new Error(`Anthropic API error: ${res.status} - ${data?.error?.message || 'Unknown error'}`);
+    }
+
+    const data = await res.json();
+    return {
+      content: data?.content?.[0]?.text?.trim() || '',
+      usage: {
+        inputTokens: data?.usage?.input_tokens || 0,
+        outputTokens: data?.usage?.output_tokens || 0,
+      },
+      latencyMs: Date.now() - startTime,
+    };
+  }
+
+  // --- OpenAI ---
+  if (provider === 'openai') {
+    const res = await fetch(providerConfig.base_url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${providerConfig.apiKey}`,
+      },
+      body: JSON.stringify({
+        model,
+        temperature,
+        max_tokens,
+        top_p,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      }),
+    });
+
+    if (!res.ok) {
+      const data = await res.json().catch(() => ({}));
+      throw new Error(`OpenAI API error: ${res.status} - ${data?.error?.message || 'Unknown error'}`);
+    }
+
+    const data = await res.json();
+    return {
+      content: data?.choices?.[0]?.message?.content?.trim() || '',
+      usage: {
+        inputTokens: data?.usage?.prompt_tokens || 0,
+        outputTokens: data?.usage?.completion_tokens || 0,
+      },
+      latencyMs: Date.now() - startTime,
+    };
+  }
+
+  // --- OpenRouter ---
+  if (provider === 'openrouter') {
+    const res = await fetch(providerConfig.base_url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${providerConfig.apiKey}`,
+        'HTTP-Referer': process.env.OPENROUTER_REFERER || 'https://machine-spirits.com',
+        'X-Title': 'Machine Spirits Tutor',
+      },
+      body: JSON.stringify({
+        model,
+        temperature,
+        max_tokens,
+        top_p,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      }),
+    });
+
+    if (!res.ok) {
+      const data = await res.json().catch(() => ({}));
+      throw new Error(`OpenRouter API error: ${res.status} - ${data?.error?.message || 'Unknown error'}`);
+    }
+
+    const data = await res.json();
+    const content = data?.choices?.[0]?.message?.content?.trim() || '';
+
+    if (!content) {
+      console.warn(`[${agentRole}] OpenRouter returned empty content. Model: ${model}, finish_reason: ${data?.choices?.[0]?.finish_reason}`);
+    }
+
+    return {
+      content,
+      usage: {
+        inputTokens: data?.usage?.prompt_tokens || 0,
+        outputTokens: data?.usage?.completion_tokens || 0,
+      },
+      latencyMs: Date.now() - startTime,
+    };
+  }
+
+  // --- Gemini ---
+  if (provider === 'gemini') {
+    const { GoogleGenAI } = await import('@google/genai');
+    const gemini = new GoogleGenAI({ apiKey: providerConfig.apiKey });
+
+    const result = await gemini.models.generateContent({
+      model,
+      systemInstruction: systemPrompt,
+      contents: [{ role: 'user', parts: [{ text: userPrompt }] }],
+      config: { temperature, maxOutputTokens: max_tokens, topP: top_p },
+    });
+
+    const content = result?.text?.() || result?.response?.text?.() || '';
+    return {
+      content,
+      usage: { inputTokens: 0, outputTokens: 0 },
+      latencyMs: Date.now() - startTime,
+    };
+  }
+
+  // --- Local (LM Studio / Ollama / llama.cpp) ---
+  if (provider === 'local') {
+    const res = await fetch(providerConfig.base_url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model,
+        temperature,
+        max_tokens,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      }),
+    });
+
+    if (!res.ok) {
+      const data = await res.json().catch(() => ({}));
+      throw new Error(`Local LLM error: ${res.status} - ${data?.error?.message || 'Is LM Studio running?'}`);
+    }
+
+    const data = await res.json();
+    return {
+      content: data?.choices?.[0]?.message?.content?.trim() || '',
+      usage: {
+        inputTokens: data?.usage?.prompt_tokens || 0,
+        outputTokens: data?.usage?.completion_tokens || 0,
+      },
+      latencyMs: Date.now() - startTime,
+    };
+  }
+
+  throw new Error(`Unsupported learner provider: ${provider}`);
+}
+
+/**
+ * Generate a single learner response for use by the evaluation pipeline.
+ * Runs ego→superego→synthesis if profile is multi-agent, or single call if unified.
+ *
+ * Uses callLearnerAI internally — the same raw fetch layer as the tutor's
+ * tutorDialogueEngine.callAI — so learner and tutor LLM calls go through
+ * identical provider code paths with identical retry logic.
+ *
+ * @param {Object} options
+ * @param {string} options.tutorMessage - The tutor's message to respond to
+ * @param {string} options.topic - Current topic
+ * @param {Array}  options.conversationHistory - [{role, content}, ...]
+ * @param {string} options.learnerProfile - Profile name ('ego_superego' or 'unified')
+ * @param {string} options.personaId - Persona identifier (default: 'eager_novice')
+ * @param {string|Object} [options.modelOverride] - Optional model override (e.g. 'openrouter.nemotron') applied to all learner agents
+ * @returns {Promise<Object>} { message, internalDeliberation, emotionalState, understandingLevel, tokenUsage }
+ */
+export async function generateLearnerResponse(options) {
+  const {
+    tutorMessage,
+    topic,
+    conversationHistory = [],
+    learnerProfile = 'unified',
+    personaId = 'eager_novice',
+    modelOverride,
+  } = options;
+
+  // Resolve model override once (if provided) so all learner agents use the same model
+  let resolvedOverride = null;
+  if (modelOverride) {
+    const r = learnerConfig.resolveModel(modelOverride);
+    const providerConfig = learnerConfig.getProviderConfig(r.provider);
+    const modelFullId = providerConfig.models?.[r.model] || r.model;
+    resolvedOverride = { provider: r.provider, providerConfig, model: modelFullId, modelAlias: r.model };
+  }
+
+  const applyOverride = (cfg) => {
+    if (!resolvedOverride || !cfg) return cfg;
+    return { ...cfg, provider: resolvedOverride.provider, providerConfig: resolvedOverride.providerConfig, model: resolvedOverride.model, modelAlias: resolvedOverride.modelAlias };
+  };
+
+  const persona = learnerConfig.getPersona(personaId);
+  const profile = learnerConfig.getActiveProfile(learnerProfile);
+  const agentRoles = learnerConfig.getProfileAgentRoles(profile.name);
+  const internalDeliberation = [];
+  const tokenUsage = { inputTokens: 0, outputTokens: 0, apiCalls: 0 };
+
+  // Build conversation context string from history
+  const conversationContext = conversationHistory
+    .slice(-6)
+    .map(m => `${m.role.toUpperCase()}: ${m.content}`)
+    .join('\n\n');
+
+  // Psychodynamic flow: Ego (initial) → Superego (critique) → Ego (revision/final)
+  // This mirrors the tutor architecture where the ego has final authority over output,
+  // accepting, rejecting, or modifying the superego's suggestions.
+
+  const hasMultiAgent = agentRoles.includes('ego') && agentRoles.includes('superego');
+
+  if (hasMultiAgent) {
+    // === STEP 1: Ego initial reaction ===
+    const egoConfig = applyOverride(learnerConfig.getAgentConfig('ego', profile.name));
+    const egoContext = `Topic: ${topic}\n\nRecent conversation:\n${conversationContext}\n\nThe tutor just said:\n"${tutorMessage}"\n\nGenerate your initial internal reaction as the learner's ego.`;
+    const egoSystemPrompt = buildLearnerPrompt(egoConfig, persona, egoContext);
+
+    const egoInitialResponse = await callLearnerAI(egoConfig, egoSystemPrompt, "React to the tutor's message.", 'learner_ego_initial');
+    internalDeliberation.push({ role: 'ego_initial', content: egoInitialResponse.content });
+    tokenUsage.inputTokens += egoInitialResponse.usage?.inputTokens || 0;
+    tokenUsage.outputTokens += egoInitialResponse.usage?.outputTokens || 0;
+    tokenUsage.apiCalls++;
+
+    // === STEP 2: Superego critique ===
+    const superegoConfig = applyOverride(learnerConfig.getAgentConfig('superego', profile.name));
+    const superegoContext = `Topic: ${topic}\n\nRecent conversation:\n${conversationContext}\n\nThe tutor just said:\n"${tutorMessage}"\n\nThe EGO's initial reaction was:\n"${egoInitialResponse.content}"\n\nReview the EGO's response. Is it accurate? What's being missed? What should be reconsidered?`;
+    const superegoSystemPrompt = buildLearnerPrompt(superegoConfig, persona, superegoContext);
+
+    const superegoResponse = await callLearnerAI(superegoConfig, superegoSystemPrompt, "Critique the EGO's reaction.", 'learner_superego');
+    internalDeliberation.push({ role: 'superego', content: superegoResponse.content });
+    tokenUsage.inputTokens += superegoResponse.usage?.inputTokens || 0;
+    tokenUsage.outputTokens += superegoResponse.usage?.outputTokens || 0;
+    tokenUsage.apiCalls++;
+
+    // === STEP 3: Ego revision (final authority) ===
+    // The ego considers the superego's feedback and decides what to actually say.
+    // It may accept, reject, or modify the superego's suggestions.
+    const egoRevisionContext = `Topic: ${topic}\n\nRecent conversation:\n${conversationContext}\n\nThe tutor just said:\n"${tutorMessage}"\n\nYour initial reaction was:\n"${egoInitialResponse.content}"\n\nThe SUPEREGO's critique:\n"${superegoResponse.content}"\n\nConsider the superego's feedback. You have final authority — accept, reject, or modify its suggestions as you see fit. Then produce a realistic external response (1-4 sentences) that the learner would actually say to the tutor.`;
+    const egoRevisionSystemPrompt = buildLearnerPrompt(egoConfig, persona, egoRevisionContext);
+
+    const egoFinalResponse = await callLearnerAI(egoConfig, egoRevisionSystemPrompt, "Produce your final response to the tutor.", 'learner_ego_revision');
+    internalDeliberation.push({ role: 'ego_revision', content: egoFinalResponse.content });
+    tokenUsage.inputTokens += egoFinalResponse.usage?.inputTokens || 0;
+    tokenUsage.outputTokens += egoFinalResponse.usage?.outputTokens || 0;
+    tokenUsage.apiCalls++;
+
+    // Log deliberation for debugging/analysis
+    if (process.env.LEARNER_DEBUG) {
+      console.log('\n┌─────────────────────────────────────────────────────────────');
+      console.log('│ LEARNER DELIBERATION (ego→superego→ego_revision)');
+      console.log('├─────────────────────────────────────────────────────────────');
+      console.log(`│ EGO INITIAL: ${egoInitialResponse.content.substring(0, 200)}...`);
+      console.log('├─────────────────────────────────────────────────────────────');
+      console.log(`│ SUPEREGO: ${superegoResponse.content.substring(0, 200)}...`);
+      console.log('├─────────────────────────────────────────────────────────────');
+      console.log(`│ EGO REVISION (FINAL): ${egoFinalResponse.content.substring(0, 200)}...`);
+      console.log('└─────────────────────────────────────────────────────────────\n');
+    }
+  } else {
+    // Single-agent (unified) flow — run each role sequentially as before
+    for (const role of agentRoles) {
+      const agentConfig = applyOverride(learnerConfig.getAgentConfig(role, profile.name));
+      if (!agentConfig) continue;
+
+      let roleContext = `Topic: ${topic}\n\nRecent conversation:\n${conversationContext}\n\nThe tutor just said:\n"${tutorMessage}"`;
+      roleContext += `\n\nGenerate your internal reaction as this dimension of the learner's experience.`;
+
+      const systemPrompt = buildLearnerPrompt(agentConfig, persona, roleContext);
+      const response = await callLearnerAI(agentConfig, systemPrompt, "React to the tutor's message.", `learner_${role}`);
+
+      internalDeliberation.push({ role, content: response.content });
+      tokenUsage.inputTokens += response.usage?.inputTokens || 0;
+      tokenUsage.outputTokens += response.usage?.outputTokens || 0;
+      tokenUsage.apiCalls++;
+    }
+  }
+
+  // Get final message from the last deliberation step
+  // For multi-agent: ego_revision. For unified: the single agent's output.
+  const finalDeliberation = internalDeliberation[internalDeliberation.length - 1];
+
+  return {
+    message: finalDeliberation.content,
+    internalDeliberation,
+    emotionalState: detectEmotionalState(internalDeliberation),
+    understandingLevel: detectUnderstandingLevel(internalDeliberation),
+    tokenUsage,
+  };
+}
+
 // ============================================================================
 // Exports
 // ============================================================================
 
 export default {
   runInteraction,
+  generateLearnerResponse,
   INTERACTION_OUTCOMES,
 };

package/services/processUtils.js

@@ -0,0 +1,18 @@
+/**
+ * Process utility functions shared across eval services and CLI.
+ */
+
+/**
+ * Check if a process with the given PID is still running.
+ * @param {number} pid - Process ID to check
+ * @returns {boolean|null} true if alive, false if dead, null if pid is falsy
+ */
+export function isPidAlive(pid) {
+  if (!pid || typeof pid !== 'number') return null;
+  try {
+    process.kill(pid, 0); // Signal 0 = check existence without killing
+    return true;
+  } catch (e) {
+    return e.code === 'EPERM'; // EPERM means process exists but we can't signal it
+  }
+}

package/services/progressLogger.js

@@ -0,0 +1,98 @@
+/**
+ * Progress Logger — JSONL event writer for cross-process eval monitoring.
+ *
+ * One file per run at logs/eval-progress/<runId>.jsonl.
+ * Each line is a self-contained JSON object with timestamp + runId + eventType.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const ROOT_DIR = path.resolve(__dirname, '..');
+const PROGRESS_DIR = path.join(ROOT_DIR, 'logs', 'eval-progress');
+
+export class ProgressLogger {
+  constructor(runId) {
+    this.runId = runId;
+    // Ensure directory exists
+    fs.mkdirSync(PROGRESS_DIR, { recursive: true });
+    this.filePath = path.join(PROGRESS_DIR, `${runId}.jsonl`);
+  }
+
+  /** Append a single JSON line */
+  writeEvent(eventType, data = {}) {
+    const event = {
+      timestamp: new Date().toISOString(),
+      runId: this.runId,
+      eventType,
+      ...data,
+    };
+    fs.appendFileSync(this.filePath, JSON.stringify(event) + '\n');
+  }
+
+  // ── Convenience methods ──────────────────────────────────────────
+
+  runStart({ totalTests, totalScenarios, totalConfigurations, scenarios, profiles, description }) {
+    this.writeEvent('run_start', {
+      totalTests,
+      totalScenarios,
+      totalConfigurations,
+      scenarios,
+      profiles,
+      description,
+    });
+  }
+
+  testStart({ scenarioId, scenarioName, profileName }) {
+    this.writeEvent('test_start', { scenarioId, scenarioName, profileName });
+  }
+
+  testComplete({ scenarioId, scenarioName, profileName, success, overallScore, baseScore, recognitionScore, latencyMs, completedCount, totalTests }) {
+    this.writeEvent('test_complete', {
+      scenarioId, scenarioName, profileName,
+      success, overallScore, baseScore, recognitionScore, latencyMs,
+      completedCount, totalTests,
+    });
+  }
+
+  testError({ scenarioId, scenarioName, profileName, errorMessage, completedCount, totalTests }) {
+    this.writeEvent('test_error', {
+      scenarioId, scenarioName, profileName, errorMessage,
+      completedCount, totalTests,
+    });
+  }
+
+  scenarioComplete({ scenarioId, scenarioName, profileNames, avgScore, completedScenarios, totalScenarios }) {
+    this.writeEvent('scenario_complete', {
+      scenarioId, scenarioName, profileNames, avgScore,
+      completedScenarios, totalScenarios,
+    });
+  }
+
+  runComplete({ totalTests, successfulTests, failedTests, durationMs }) {
+    this.writeEvent('run_complete', {
+      totalTests, successfulTests, failedTests, durationMs,
+    });
+  }
+}
+
+/** Resolve the JSONL path for a given runId (may not exist yet). */
+export function getProgressLogPath(runId) {
+  return path.join(PROGRESS_DIR, `${runId}.jsonl`);
+}
+
+/** Read all events from a JSONL progress file. Returns [] if missing. */
+export function readProgressLog(runId) {
+  const filePath = path.join(PROGRESS_DIR, `${runId}.jsonl`);
+  if (!fs.existsSync(filePath)) return [];
+  const lines = fs.readFileSync(filePath, 'utf-8').split('\n').filter(Boolean);
+  return lines.map(line => {
+    try { return JSON.parse(line); }
+    catch { return null; }
+  }).filter(Boolean);
+}
+
+export default { ProgressLogger, getProgressLogPath, readProgressLog };

package/services/promptRecommendationService.js

@@ -2,18 +2,17 @@
  * Prompt Recommendation Service
  *
  * Analyzes evaluation results and generates recommendations to improve
- * tutor prompts. Uses a powerful evaluator model to analyze failures
+ * tutor prompts. Uses a powerful recommender model to analyze failures
  * and weaknesses from weaker tutor models.
  *
- * Evaluator configuration is loaded from config/evaluation-rubric.yaml
+ * Recommender configuration is loaded from config/evaluation-rubric.yaml
  * Provider details are resolved from config/providers.yaml
  */
 
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import Anthropic from '@anthropic-ai/sdk';
-import { tutorApiService as tutorApi, tutorConfigLoader as configLoader } from '@machinespirits/tutor-core';
+import * as evalConfigLoader from './evalConfigLoader.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -22,12 +21,11 @@ const PROMPTS_DIR = path.join(ROOT_DIR, 'prompts');
 
 /**
  * Get recommender config, resolving model references via providers.yaml
- * Uses 'recommender' config from evaluation-rubric.yaml (falls back to 'evaluator')
+ * Uses 'recommender' config from evaluation-rubric.yaml
  */
-function getEvaluatorConfig() {
-  const rubric = tutorApi.loadRubric();
-  // Prefer 'recommender' for prompt analysis, fall back to legacy 'evaluator'
-  const evalConfig = rubric?.recommender || rubric?.evaluator;
+function getRecommenderConfig() {
+  const rubric = evalConfigLoader.loadRubric();
+  const evalConfig = rubric?.recommender;
 
   if (!evalConfig?.model) {
     console.warn('[promptRecommendation] No recommender in evaluation-rubric.yaml, using defaults');
@@ -40,7 +38,7 @@ function getEvaluatorConfig() {
 
   // Try to resolve primary model
   try {
-    const resolved = configLoader.resolveModel(evalConfig.model);
+    const resolved = evalConfigLoader.resolveModel(evalConfig.model);
     if (resolved.isConfigured) {
       return {
         provider: resolved.provider,
@@ -57,7 +55,7 @@ function getEvaluatorConfig() {
   // Try fallback
   if (evalConfig.fallback?.model) {
     try {
-      const fallback = configLoader.resolveModel(evalConfig.fallback.model);
+      const fallback = evalConfigLoader.resolveModel(evalConfig.fallback.model);
       if (fallback.isConfigured) {
         console.log(`[promptRecommendation] Using fallback: ${fallback.provider}/${fallback.model}`);
         return {
@@ -74,7 +72,7 @@ function getEvaluatorConfig() {
   }
 
   // Return primary anyway - will fail with helpful error
-  const resolved = configLoader.resolveModel(evalConfig.model);
+  const resolved = evalConfigLoader.resolveModel(evalConfig.model);
   return {
     provider: resolved.provider,
     model: resolved.model,
@@ -157,7 +155,7 @@ function analyzeResults(results) {
 }
 
 /**
- * Build the analysis prompt for the evaluator
+ * Build the analysis prompt for the recommender
  */
 function buildAnalysisPrompt(analysis, egoPrompt, superegoPrompt, profileName) {
   const sections = [];
@@ -273,14 +271,14 @@ Be specific and actionable. Quote exact text to change when possible.
 }
 
 /**
- * Call the evaluator model to generate recommendations
+ * Call the recommender model to generate recommendations
  * Uses config from evaluation-rubric.yaml
  */
-async function callEvaluator(prompt, options = {}) {
+async function callRecommender(prompt, options = {}) {
   const { budget = false } = options;
 
   // Get config from yaml (handles fallbacks automatically)
-  const config = getEvaluatorConfig();
+  const config = getRecommenderConfig();
   const { provider, model, hyperparameters } = config;
   const maxTokens = hyperparameters?.max_tokens ?? 4000;
   const temperature = hyperparameters?.temperature ?? 0.3;
@@ -298,6 +296,13 @@ async function callEvaluator(prompt, options = {}) {
     throw new Error('ANTHROPIC_API_KEY not set');
   }
 
+  let Anthropic;
+  try {
+    Anthropic = (await import('@anthropic-ai/sdk')).default;
+  } catch {
+    throw new Error('@anthropic-ai/sdk is not installed. Install it to use the Anthropic provider for recommendations.');
+  }
+
   const client = new Anthropic({ apiKey });
 
   const response = await client.messages.create({
@@ -377,9 +382,9 @@ async function callOpenRouterEvaluator(prompt, model, options = {}) {
  * @param {string} options.profileName - Profile that was evaluated
  * @param {string} options.egoPromptFile - Ego prompt file to analyze
  * @param {string} options.superegoPromptFile - Superego prompt file to analyze
- * @param {string} options.evaluatorModel - Model to use for analysis (default: claude-sonnet-4)
- * @param {string} options.evaluatorProvider - Provider: 'anthropic' or 'openrouter'
- * @param {boolean} options.budget - Use budget evaluator model
+ * @param {string} options.recommenderModel - Model to use for analysis (default: claude-sonnet-4)
+ * @param {string} options.recommenderProvider - Provider: 'anthropic' or 'openrouter'
+ * @param {boolean} options.budget - Use budget recommender model
  * @returns {Promise<Object>} Recommendations
  */
 export async function generateRecommendations(options = {}) {
@@ -388,8 +393,8 @@ export async function generateRecommendations(options = {}) {
     profileName = 'unknown',
     egoPromptFile = 'tutor-ego.md',
     superegoPromptFile = 'tutor-superego.md',
-    evaluatorModel = null,
-    evaluatorProvider = 'anthropic',
+    recommenderModel = null,
+    recommenderProvider = 'anthropic',
     budget = false,
   } = options;
 
@@ -421,18 +426,18 @@ export async function generateRecommendations(options = {}) {
   // Build analysis prompt
   const analysisPrompt = buildAnalysisPrompt(analysis, egoPrompt, superegoPrompt, profileName);
 
-  // Get evaluator config from yaml
-  const evalConfig = getEvaluatorConfig();
+  // Get recommender config from yaml
+  const evalConfig = getRecommenderConfig();
   console.log(`\nGenerating recommendations using ${evalConfig.provider}/${evalConfig.model}...`);
 
-  const evalResult = await callEvaluator(analysisPrompt);
+  const evalResult = await callRecommender(analysisPrompt);
 
   return {
     success: true,
     needsImprovement: true,
     analysis,
     recommendations: evalResult.content,
-    evaluatorModel: evalResult.model,
+    recommenderModel: evalResult.model,
     usage: {
       inputTokens: evalResult.inputTokens,
       outputTokens: evalResult.outputTokens,
@@ -479,7 +484,7 @@ export function formatRecommendations(result) {
   lines.push(result.recommendations);
   lines.push('');
   lines.push('─'.repeat(80));
-  lines.push(`Evaluator: ${result.evaluatorModel}`);
+  lines.push(`Recommender: ${result.recommenderModel}`);
   lines.push(`Tokens: ${result.usage.inputTokens} in / ${result.usage.outputTokens} out`);
   lines.push('═'.repeat(80));