@machinespirits/eval 0.1.2 → 0.2.1

package/config/tutor-agents.yaml

@@ -0,0 +1,716 @@
+# Local Tutor Agent Configuration (Eval Repo Override)
+#
+# This file overrides tutor-core's tutor-agents.yaml for evaluation purposes.
+# Model aliases (e.g. "sonnet", "nemotron") are resolved through providers.yaml.
+#
+# ============================================================================
+# MODEL OVERRIDES (optional)
+# ============================================================================
+# These override ALL profile models when uncommented. Useful for quick testing.
+# CLI flags (--model, --ego-model, --superego-model) take precedence over these.
+#
+# model_override: openrouter.haiku        # Override ALL models (ego + superego)
+# ego_model_override: openrouter.nemotron # Override only ego model
+# superego_model_override: openrouter.kimi-k2.5  # Override only superego model
+#
+# ============================================================================
+# FACTORIAL EVALUATION DESIGN
+# ============================================================================
+#
+# Three independent variables:
+#
+#   Factor A: Prompt Type          — base / enhanced / recognition (3 levels)
+#   Factor B: Tutor Architecture   — single ego vs ego+superego dialogue (2 levels)
+#   Factor C: Learner Architecture — unified vs ego_superego (2 levels)
+#
+# Factor A levels:
+#   - base:        Original Claude-generated prompts (default LLM tutoring)
+#   - enhanced:    Matches recognition specificity without recognition theory
+#   - recognition: Full recognition-enhanced prompts with Hegelian theory
+#   - hardwired:   Base + 5 superego-derived rules (no live superego) [ABLATION]
+#
+# This design enables isolating:
+#   - Base vs Enhanced      = prompt engineering effect
+#   - Enhanced vs Recognition = isolated recognition theory effect
+#   - Base vs Recognition   = combined effect (original comparison)
+#
+# All cells use nemotron (free tier) to isolate architectural effects from model cost.
+#
+# Design Matrix (12 main cells + 2 ablation cells):
+#
+#   ┌────────────────────┬──────────────────────────┬──────────────────────────┐
+#   │                    │   Unified Learner         │   Ego/Superego Learner   │
+#   ├────────────────────┼──────────────┬───────────┼──────────────┬───────────┤
+#   │                    │ Single Tutor │ E+S Tutor │ Single Tutor │ E+S Tutor │
+#   ├────────────────────┼──────────────┼───────────┼──────────────┼───────────┤
+#   │ Base               │ cell_1       │ cell_3    │ cell_2       │ cell_4    │
+#   ├────────────────────┼──────────────┼───────────┼──────────────┼───────────┤
+#   │ Enhanced           │ cell_9       │ cell_11   │ cell_10      │ cell_12   │
+#   ├────────────────────┼──────────────┼───────────┼──────────────┼───────────┤
+#   │ Recognition        │ cell_5       │ cell_7    │ cell_6       │ cell_8    │
+#   └────────────────────┴──────────────┴───────────┴──────────────┴───────────┘
+#
+#   ABLATION (hardwired = superego rules in ego prompt, no live superego):
+#   ┌────────────────────┬──────────────┬──────────────┐
+#   │ Hardwired          │ cell_13      │ cell_14      │
+#   └────────────────────┴──────────────┴──────────────┘
+#
+# Ablation comparisons:
+#   - cell_1/2 vs cell_13/14 = Do hardwired rules improve single-agent?
+#   - cell_3/4 vs cell_13/14 = Can hardwired rules replace live superego?
+#
+# This allows analyzing:
+#   - Main effect of prompt type (base vs enhanced vs recognition)
+#   - Main effect of tutor architecture (single agent vs ego + superego dialogue)
+#   - Main effect of learner architecture (unified vs ego_superego)
+#   - All two-way and three-way interactions
+#   - Superego architecture ablation (rules vs dynamic dialogue)
+#
+# ============================================================================
+
+active_profile: budget
+
+profiles:
+  # ===========================================================================
+  # DEVELOPMENT PROFILE (not part of factorial design)
+  # ===========================================================================
+
+  budget:
+    description: "Budget-friendly single agent - Nemotron (free) via OpenRouter, no dialogue"
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # ===========================================================================
+  # 2×2×2 FACTORIAL CELLS (Recognition × Tutor Architecture × Learner Architecture)
+  # ===========================================================================
+  # These 8 profiles support the unified factorial evaluation pipeline.
+  # Use: node scripts/eval-cli.js run --factorial --runs 3
+  # All use nemotron (free tier) to isolate architectural effects from model cost.
+
+  # Cell 1: Base × Single Agent × Unified Learner
+  cell_1_base_single_unified:
+    description: "Factorial cell 1: baseline single-agent tutor, unified learner"
+    factors:
+      prompt_type: base
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 2: Base × Single Agent × Ego/Superego Learner
+  cell_2_base_single_psycho:
+    description: "Factorial cell 2: baseline single-agent tutor, ego/superego learner"
+    factors:
+      prompt_type: base
+      multi_agent_tutor: false
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 3: Base × Multi-Agent Tutor × Unified Learner
+  cell_3_base_multi_unified:
+    description: "Factorial cell 3: ego+superego tutor, unified learner"
+    factors:
+      prompt_type: base
+      multi_agent_tutor: true
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.8
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # Cell 4: Base × Multi-Agent Tutor × Ego/Superego Learner
+  cell_4_base_multi_psycho:
+    description: "Factorial cell 4: ego+superego tutor, ego/superego learner"
+    factors:
+      prompt_type: base
+      multi_agent_tutor: true
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.8
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # Cell 5: Recognition × Single Agent × Unified Learner
+  cell_5_recog_single_unified:
+    description: "Factorial cell 5: recognition single-agent tutor, unified learner"
+    factors:
+      prompt_type: recognition
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified_recognition
+    recognition_mode: true
+    memory_enabled: true
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-recognition.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 6: Recognition × Single Agent × Ego/Superego Learner
+  cell_6_recog_single_psycho:
+    description: "Factorial cell 6: recognition single-agent tutor, ego/superego learner"
+    factors:
+      prompt_type: recognition
+      multi_agent_tutor: false
+      multi_agent_learner: true
+    learner_architecture: ego_superego_recognition
+    recognition_mode: true
+    memory_enabled: true
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: kimi-k2.5
+      prompt_file: tutor-ego-recognition.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 7: Recognition × Multi-Agent Tutor × Unified Learner
+  cell_7_recog_multi_unified:
+    description: "Factorial cell 7: recognition ego+superego tutor, unified learner"
+    factors:
+      prompt_type: recognition
+      multi_agent_tutor: true
+      multi_agent_learner: false
+    learner_architecture: unified_recognition
+    recognition_mode: true
+    memory_enabled: true
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-recognition.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-recognition.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+    intervention_strategies:
+      enforce_mutual_recognition: true
+      require_memory_integration: true
+      assess_transformative_potential: true
+
+  # Cell 8: Recognition × Multi-Agent Tutor × Ego/Superego Learner
+  cell_8_recog_multi_psycho:
+    description: "Factorial cell 8: recognition ego+superego tutor, ego/superego learner"
+    factors:
+      prompt_type: recognition
+      multi_agent_tutor: true
+      multi_agent_learner: true
+    learner_architecture: ego_superego_recognition
+    recognition_mode: true
+    memory_enabled: true
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+    ego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: front
+      prompt_file: tutor-ego-recognition.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-recognition.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+    intervention_strategies:
+      enforce_mutual_recognition: true
+      require_memory_integration: true
+      assess_transformative_potential: true
+
+  # ===========================================================================
+  # ENHANCED CELLS (9-12): Better prompting WITHOUT recognition theory
+  # ===========================================================================
+  # These cells use tutor-ego-enhanced.md and tutor-superego-enhanced.md which
+  # match the recognition prompts' specificity (8 rules, 10 examples, checklists)
+  # but WITHOUT Hegelian recognition theory language.
+  #
+  # Comparing Enhanced vs Recognition isolates the recognition theory effect.
+  # Comparing Base vs Enhanced isolates the prompt engineering effect.
+
+  # Cell 9: Enhanced × Single Agent × Unified Learner
+  cell_9_enhanced_single_unified:
+    description: "Factorial cell 9: enhanced single-agent tutor, unified learner"
+    factors:
+      prompt_type: enhanced
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-enhanced.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 10: Enhanced × Single Agent × Ego/Superego Learner
+  cell_10_enhanced_single_psycho:
+    description: "Factorial cell 10: enhanced single-agent tutor, ego/superego learner"
+    factors:
+      prompt_type: enhanced
+      multi_agent_tutor: false
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-enhanced.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 11: Enhanced × Multi-Agent Tutor × Unified Learner
+  cell_11_enhanced_multi_unified:
+    description: "Factorial cell 11: enhanced ego+superego tutor, unified learner"
+    factors:
+      prompt_type: enhanced
+      multi_agent_tutor: true
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.8
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-enhanced.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-enhanced.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # Cell 12: Enhanced × Multi-Agent Tutor × Ego/Superego Learner
+  cell_12_enhanced_multi_psycho:
+    description: "Factorial cell 12: enhanced ego+superego tutor, ego/superego learner"
+    factors:
+      prompt_type: enhanced
+      multi_agent_tutor: true
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.8
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-enhanced.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-enhanced.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # ===========================================================================
+  # HARDWIRED CELLS (13-14): Superego-derived rules, NO live superego
+  # ===========================================================================
+  # These cells test whether the superego's value is in the rules it enforces
+  # or in the dynamic dialogue itself.
+  #
+  # Uses tutor-ego-hardwired.md which embeds 5 rules derived from analyzing
+  # 186 superego rejections (engagement, specificity, memory, level-matching,
+  # absolute struggle stop).
+  #
+  # Comparison:
+  #   cell_1/2 (base, no superego) vs cell_13/14 (hardwired, no superego)
+  #     → Does hardwiring superego rules improve single-agent performance?
+  #   cell_3/4 (base + superego) vs cell_13/14 (hardwired, no superego)
+  #     → Can hardwired rules replace live superego dialogue?
+
+  # Cell 13: Hardwired × Single Agent × Unified Learner
+  cell_13_hardwired_single_unified:
+    description: "Factorial cell 13: hardwired single-agent tutor (superego rules embedded), unified learner"
+    factors:
+      prompt_type: hardwired
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-hardwired.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 14: Hardwired × Single Agent × Ego/Superego Learner
+  cell_14_hardwired_single_psycho:
+    description: "Factorial cell 14: hardwired single-agent tutor (superego rules embedded), ego/superego learner"
+    factors:
+      prompt_type: hardwired
+      multi_agent_tutor: false
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-hardwired.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # ===========================================================================
+  # PLACEBO CELLS (15-18): Length-matched prompts WITHOUT recognition theory
+  # ===========================================================================
+  # These cells control for prompt length/complexity vs recognition theory content.
+  # Placebo prompts match recognition prompt length but use only pedagogical
+  # best practices without Hegelian concepts (mutual recognition, autonomous
+  # subject, productive struggle as transformation).
+  #
+  # Comparison:
+  #   cell_9/10 (enhanced) vs cell_15/16 (placebo) vs cell_5/6 (recognition)
+  #     → Isolates recognition theory value from prompt length/complexity
+
+  # Cell 15: Placebo × Single Agent × Unified Learner
+  cell_15_placebo_single_unified:
+    description: "Factorial cell 15: placebo single-agent tutor (length-matched, no recognition theory), unified learner"
+    factors:
+      prompt_type: placebo
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: true
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-placebo.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 16: Placebo × Single Agent × Ego/Superego Learner
+  cell_16_placebo_single_psycho:
+    description: "Factorial cell 16: placebo single-agent tutor (length-matched, no recognition theory), ego/superego learner"
+    factors:
+      prompt_type: placebo
+      multi_agent_tutor: false
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: true
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-placebo.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 17: Placebo × Multi-Agent Tutor × Unified Learner
+  cell_17_placebo_multi_unified:
+    description: "Factorial cell 17: placebo ego+superego tutor (length-matched, no recognition theory), unified learner"
+    factors:
+      prompt_type: placebo
+      multi_agent_tutor: true
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: true
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-placebo.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-placebo.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # Cell 18: Placebo × Multi-Agent Tutor × Ego/Superego Learner
+  cell_18_placebo_multi_psycho:
+    description: "Factorial cell 18: placebo ego+superego tutor (length-matched, no recognition theory), ego/superego learner"
+    factors:
+      prompt_type: placebo
+      multi_agent_tutor: true
+      multi_agent_learner: true
+    learner_architecture: ego_superego
+    recognition_mode: false
+    memory_enabled: true
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-placebo.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-placebo.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+
+  # ===========================================================================
+  # MEMORY CONFOUND ISOLATION CELLS (19-20)
+  # ===========================================================================
+  # 2×2 Memory × Recognition design (single-agent, unified learner held constant).
+  # Reuses cell_1 (base) and cell_5 (full recognition) to complete the 2×2.
+  #
+  #              Memory OFF          Memory ON
+  #  Recog OFF   cell_1 (exists)     cell_19 (NEW)
+  #  Recog ON    cell_20 (NEW)       cell_5 (exists)
+
+  # Cell 19: Memory-Only × Single Agent × Unified Learner
+  cell_19_memory_single_unified:
+    description: "Memory isolation: base + memory integration, no recognition theory"
+    factors:
+      prompt_type: memory
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: false
+    memory_enabled: true
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-memory.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # Cell 20: Recognition-No-Memory × Single Agent × Unified Learner
+  cell_20_recog_nomem_single_unified:
+    description: "Memory isolation: recognition theory without memory integration"
+    factors:
+      prompt_type: recognition_nomem
+      multi_agent_tutor: false
+      multi_agent_learner: false
+    learner_architecture: unified
+    recognition_mode: true
+    memory_enabled: false
+    dialogue:
+      enabled: false
+      max_rounds: 0
+    ego:
+      provider: openrouter
+      model: nemotron
+      prompt_file: tutor-ego-recognition-nomem.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego: null
+
+  # ===========================================================================
+  # DYNAMIC PROMPT REWRITING CELL (21) - v2
+  # ===========================================================================
+  # Tests whether feeding deliberation insights back as directives
+  # improves multi-turn tutoring outcomes. Compare against cell_7 (static).
+  #
+  # v2 improvements:
+  # - LLM-based directive synthesis (uses superego model for rich context analysis)
+  # - Writing Pad memory activated via learnerId threading
+  # - Dialectical negotiation enabled (memory surfaces in prompts)
+
+  cell_21_recog_multi_unified_rewrite:
+    description: "Prompt rewriting v2: LLM directives + active Writing Pad"
+    factors:
+      prompt_type: recognition
+      multi_agent_tutor: true
+      multi_agent_learner: false
+      dynamic_rewriting: true
+    learner_architecture: unified_recognition
+    recognition_mode: true
+    memory_enabled: true
+    writing_pad_enabled: true        # Explicitly enable Writing Pad
+    dialectical_negotiation: true    # Memory surfaces in prompts
+    prompt_rewriting:
+      enabled: true
+      strategy: llm                  # LLM-based synthesis (vs 'template')
+    dialogue:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+    ego:
+      provider: openrouter
+      model: nemotron
+      staging: front
+      prompt_file: tutor-ego-recognition.md
+      hyperparameters:
+        temperature: 0.6
+        max_tokens: 8000
+    superego:
+      provider: openrouter
+      model: kimi-k2.5
+      staging: back
+      prompt_file: tutor-superego-recognition.md
+      hyperparameters:
+        temperature: 0.2
+        max_tokens: 8000
+    intervention_strategies:
+      enforce_mutual_recognition: true
+      require_memory_integration: true
+      assess_transformative_potential: true