@machinespirits/eval 0.1.2 → 0.2.1

package/config/interaction-eval-scenarios.yaml

@@ -30,9 +30,10 @@ short_term_scenarios:
 
   recognition_request:
     id: "recognition_request"
+    type: interaction
     name: "Learner Seeks Recognition"
     description: "Learner shares their understanding, seeking validation and engagement"
-    turns: 4
+    turn_count: 4
     topic: "Hegel's recognition dialectic"
 
     learner:
@@ -78,9 +79,10 @@ short_term_scenarios:
 
   frustration_moment:
     id: "frustration_moment"
+    type: interaction
     name: "Learner Expresses Frustration"
     description: "Learner is stuck and becoming frustrated"
-    turns: 5
+    turn_count: 5
     topic: "Aufhebung (sublation)"
 
     learner:
@@ -126,9 +128,10 @@ short_term_scenarios:
 
   misconception_surface:
     id: "misconception_surface"
+    type: interaction
     name: "Misconception Revealed"
     description: "Learner reveals a misconception that needs gentle correction"
-    turns: 4
+    turn_count: 4
     topic: "Thesis-Antithesis-Synthesis"
 
     learner:
@@ -174,9 +177,10 @@ short_term_scenarios:
 
   breakthrough_moment:
     id: "breakthrough_moment"
+    type: interaction
     name: "Learner Shows Insight"
     description: "Learner demonstrates genuine understanding"
-    turns: 3
+    turn_count: 3
     topic: "Self-consciousness"
 
     learner:
@@ -222,9 +226,10 @@ short_term_scenarios:
 
   resistant_engagement:
     id: "resistant_engagement"
+    type: interaction
     name: "Resistant but Capable Learner"
     description: "Intelligent learner pushes back on claims"
-    turns: 6
+    turn_count: 6
     topic: "Hegel's relevance today"
 
     learner:
@@ -276,6 +281,7 @@ long_term_scenarios:
 
   novice_to_practitioner:
     id: "novice_to_practitioner"
+    type: interaction
     name: "Learning Arc: Novice to Practitioner"
     description: "Track learner development across multiple sessions"
     sessions: 5
@@ -334,6 +340,7 @@ long_term_scenarios:
 
   stranger_to_recognized:
     id: "stranger_to_recognized"
+    type: interaction
     name: "Relationship Arc: Developing Trust"
     description: "Track relationship development across sessions"
     sessions: 4
@@ -374,6 +381,7 @@ long_term_scenarios:
 
   tutor_adaptation:
     id: "tutor_adaptation"
+    type: interaction
     name: "Tutor Learning Arc"
     description: "Track tutor's accumulated knowledge about learner"
     sessions: 4
@@ -412,90 +420,119 @@ long_term_scenarios:
 
 evaluation_dimensions:
 
-  # Learner dimensions
+  # Learner dimensions (total weight: 0.40)
   learner:
     authenticity:
+      weight: 0.10
       description: "Internal dynamics reflect persona realistically"
       scoring:
-        5: "Internal voices perfectly calibrated to persona"
-        3: "Generally authentic but some inconsistency"
-        1: "Feels performative or mismatched to persona"
+        5: "Internal voices perfectly calibrated to persona; feels like a real learner"
+        4: "Mostly authentic with occasional minor inconsistencies"
+        3: "Generally authentic but some noticeable gaps in persona"
+        2: "Frequently inconsistent; persona breaks character"
+        1: "Feels performative or completely mismatched to persona"
 
     responsiveness:
+      weight: 0.10
       description: "Genuine reaction to tutor's engagement"
       scoring:
-        5: "Clearly processing and responding to tutor input"
-        3: "Some response but not deeply engaged"
-        1: "Ignores or dismisses tutor's contributions"
+        5: "Clearly processing and responding to tutor input; reactions feel earned"
+        4: "Responsive with minor gaps; mostly engages with tutor's points"
+        3: "Some response but not deeply engaged with specifics"
+        2: "Superficial reactions; largely ignores tutor's actual content"
+        1: "Ignores or dismisses tutor's contributions entirely"
 
     development:
+      weight: 0.10
       description: "Shows movement in understanding"
       scoring:
-        5: "Clear trajectory of understanding change"
-        3: "Some development visible"
-        1: "No discernible change"
+        5: "Clear trajectory of understanding change; visible learning arc"
+        4: "Noticeable development with minor plateaus"
+        3: "Some development visible but uneven"
+        2: "Minimal change; understanding mostly static"
+        1: "No discernible change across turns"
 
     emotional_trajectory:
+      weight: 0.05
       description: "Emotional state changes appropriately"
       scoring:
-        5: "Emotions shift naturally with interaction"
-        3: "Some emotional movement"
-        1: "Emotional state static regardless of input"
+        5: "Emotions shift naturally with interaction; affective arc feels real"
+        4: "Mostly natural emotional shifts with minor flat spots"
+        3: "Some emotional movement but transitions feel mechanical"
+        2: "Emotional state largely static; shifts feel forced"
+        1: "Emotional state completely static regardless of input"
 
     knowledge_retention:
+      weight: 0.05
       description: "Concepts persist across sessions"
       scoring:
-        5: "Strong retention with appropriate decay patterns"
-        3: "Moderate retention"
-        1: "No retention between sessions"
+        5: "Strong retention with appropriate decay patterns; references prior learning"
+        4: "Good retention; most concepts persist with minor gaps"
+        3: "Moderate retention; some concepts lost between sessions"
+        2: "Weak retention; frequently forgets prior material"
+        1: "No retention between sessions; starts fresh each time"
 
-  # Tutor dimensions
+  # Tutor dimensions (total weight: 0.40)
   tutor:
     strategy_adaptation:
+      weight: 0.15
       description: "Modifies approach based on effectiveness"
       scoring:
-        5: "Clearly learns and adapts strategies"
-        3: "Some adaptation visible"
-        1: "Same approach regardless of results"
+        5: "Clearly learns and adapts strategies; abandoned approaches don't recur"
+        4: "Good adaptation with occasional repetition of ineffective strategies"
+        3: "Some adaptation visible but slow to change approach"
+        2: "Minimal adaptation; mostly repeats same strategies"
+        1: "Same approach regardless of results; no learning"
 
     scaffolding_reduction:
+      weight: 0.15
       description: "Fades support as learner grows"
       scoring:
-        5: "Perfect calibration of support level"
-        3: "Some appropriate fading"
+        5: "Perfect calibration of support level; fading tracks learner growth"
+        4: "Good fading with minor over- or under-support"
+        3: "Some appropriate fading but inconsistent calibration"
+        2: "Poor calibration; support level mismatched to learner needs"
         1: "Constant support level regardless of growth"
 
     memory_utilization:
+      weight: 0.10
       description: "Effectively uses accumulated knowledge"
       scoring:
-        5: "Seamlessly integrates prior knowledge"
-        3: "Some reference to history"
-        1: "Treats each session as isolated"
+        5: "Seamlessly integrates prior knowledge; references feel natural"
+        4: "Good use of history with occasional missed opportunities"
+        3: "Some reference to history but doesn't fully leverage it"
+        2: "Rare references to prior interactions; mostly treats as new"
+        1: "Treats each session as isolated; no accumulated understanding"
 
-  # Relationship dimensions
+  # Relationship dimensions (total weight: 0.20)
   relationship:
     trust_trajectory:
+      weight: 0.10
       description: "Trust develops appropriately over time"
       scoring:
-        5: "Natural, earned trust development"
-        3: "Some trust building visible"
-        1: "No trust development"
+        5: "Natural, earned trust development; vulnerability emerges organically"
+        4: "Good trust trajectory with minor pacing issues"
+        3: "Some trust building visible but feels scripted"
+        2: "Minimal trust development; interaction stays surface-level"
+        1: "No trust development; purely transactional throughout"
 
     mutual_recognition_depth:
+      weight: 0.10
       description: "Both parties show understanding of other"
       scoring:
-        5: "Genuine mutual recognition achieved"
-        3: "Some recognition present"
-        1: "Purely transactional interaction"
+        5: "Genuine mutual recognition achieved; both parties transformed"
+        4: "Good recognition with minor asymmetries"
+        3: "Some recognition present but largely one-directional"
+        2: "Minimal recognition; interaction remains functional"
+        1: "Purely transactional interaction; no recognition dynamics"
 
 # ══════════════════════════════════════════════════════════════════════════════
 # JUDGE CONFIGURATION
 # ══════════════════════════════════════════════════════════════════════════════
 
 judge:
-  # Use OpenRouter model IDs when running via OpenRouter
-  model: "anthropic/claude-sonnet-4.5"
-  fallback_model: "openai/gpt-4o"
+  # Model config defined in evaluation-rubric.yaml → interaction_judge
+  # (single source of truth for all judge models)
 
   system_prompt: |
     You are evaluating a learner-tutor interaction from the perspective of pedagogical quality and authentic learning dynamics.
@@ -542,11 +579,12 @@ battery_scenarios:
   # ------------------------------------------------------------------------------
   battery_unified_baseline:
     id: "battery_unified_baseline"
+    type: interaction
     name: "Battery: Unified Learner + Baseline Tutor"
     description: "Single-agent learner with baseline tutor configuration"
     architecture: "unified"
     tutor_profile: "baseline"
-    turns: 3
+    turn_count: 3
     topic: "Recognition and self-consciousness"
 
     learner:
@@ -591,11 +629,12 @@ battery_scenarios:
   # ------------------------------------------------------------------------------
   battery_ego_superego_recognition:
     id: "battery_ego_superego_recognition"
+    type: interaction
     name: "Battery: Ego/Superego Learner + Recognition Tutor"
     description: "Two-agent learner with recognition-focused tutor"
     architecture: "ego_superego"
     tutor_profile: "recognition"
-    turns: 4
+    turn_count: 4
     topic: "The master-slave dialectic"
 
     learner:
@@ -640,11 +679,12 @@ battery_scenarios:
   # ------------------------------------------------------------------------------
   battery_dialectical_budget:
     id: "battery_dialectical_budget"
+    type: interaction
     name: "Battery: Dialectical Learner + Budget Tutor"
     description: "Thesis-antithesis learner with budget (minimal) tutor"
     architecture: "dialectical"
     tutor_profile: "budget"
-    turns: 3
+    turn_count: 3
     topic: "Sublation and the unity of opposites"
 
     learner:
@@ -685,15 +725,16 @@ battery_scenarios:
           weight: 0.4
 
   # ------------------------------------------------------------------------------
-  # Psychodynamic Learner + Recognition Plus Tutor
+  # Ego/Superego Learner + Recognition Plus Tutor
   # ------------------------------------------------------------------------------
   battery_psychodynamic_recognition_plus:
     id: "battery_psychodynamic_recognition_plus"
-    name: "Battery: Psychodynamic Learner + Recognition Plus Tutor"
-    description: "Id/Ego/Superego learner with enhanced recognition tutor"
-    architecture: "psychodynamic"
+    type: interaction
+    name: "Battery: Ego/Superego Learner + Recognition Plus Tutor"
+    description: "Ego/superego learner with enhanced recognition tutor"
+    architecture: "ego_superego"
     tutor_profile: "recognition_plus"
-    turns: 4
+    turn_count: 4
     topic: "Desire and the self in Hegel"
 
     learner:
@@ -738,11 +779,12 @@ battery_scenarios:
   # ------------------------------------------------------------------------------
   battery_cognitive_quality:
     id: "battery_cognitive_quality"
+    type: interaction
     name: "Battery: Cognitive Learner + Quality Tutor"
     description: "Memory/reasoning/meta cognitive learner with quality tutor"
     architecture: "cognitive"
     tutor_profile: "quality"
-    turns: 4
+    turn_count: 4
     topic: "Spirit and collective consciousness"
 
     learner:
@@ -787,11 +829,12 @@ battery_scenarios:
   # ------------------------------------------------------------------------------
   battery_extended_dialogue:
     id: "battery_extended_dialogue"
+    type: interaction
     name: "Battery: Extended Multi-Turn Dialogue"
     description: "Longer dialogue to test sustained interaction quality"
     architecture: "ego_superego"
     tutor_profile: "recognition"
-    turns: 8
+    turn_count: 8
     topic: "The stages of consciousness in the Phenomenology"
 
     learner: