@machinespirits/eval 0.1.0

package/config/learner-agents.yaml

@@ -0,0 +1,248 @@
+# Learner Agent Architecture Configuration
+# Defines how the simulated learner generates responses during evaluation
+#
+# ============================================================================
+# LEARNER ARCHITECTURES
+# ============================================================================
+#
+# The learner architecture determines whether the simulated learner has
+# internal deliberation before generating responses. This enables testing
+# whether multi-agent learner simulation improves evaluation validity.
+#
+# Architectures:
+# 1. unified: Single learner agent (no internal dialogue)
+# 2. psychodynamic: Freudian desire/intellect/aspiration deliberation
+# 3. dialectical: Hegelian thesis/antithesis/synthesis process
+#
+# ============================================================================
+
+# Active architecture (can be overridden by tutor profile)
+active_architecture: unified
+
+# ============================================================================
+# ARCHITECTURE DEFINITIONS
+# ============================================================================
+
+architectures:
+  # Unified: Single agent with no internal deliberation
+  # Simple, fast, good for baseline comparisons
+  unified:
+    name: "Unified Learner"
+    description: "Single learner agent without internal deliberation"
+    deliberation:
+      enabled: false
+      max_rounds: 0
+
+    agent:
+      provider: openrouter
+      model: nemotron
+      prompt_file: learner-unified.md
+      hyperparameters:
+        temperature: 0.7
+        max_tokens: 500
+
+  # Psychodynamic: Freudian-inspired desire/intellect/aspiration
+  # Internal deliberation between:
+  # - Desire (Id-like): What the learner wants emotionally
+  # - Intellect (Ego-like): Rational analysis of the situation
+  # - Aspiration (Superego-like): Idealized learning goals
+  psychodynamic:
+    name: "Psychodynamic Learner"
+    description: "Freudian-inspired internal deliberation between desire, intellect, and aspiration"
+    deliberation:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+
+    agents:
+      desire:
+        role: "id"
+        description: "Emotional/affective responses - what the learner WANTS"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-desire.md
+        hyperparameters:
+          temperature: 0.8  # Higher temp for more emotional/varied responses
+          max_tokens: 400
+
+      intellect:
+        role: "ego"
+        description: "Rational analysis - what the learner THINKS"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-intellect.md
+        hyperparameters:
+          temperature: 0.5  # Lower temp for more analytical responses
+          max_tokens: 400
+
+      aspiration:
+        role: "superego"
+        description: "Idealized goals - what the learner SHOULD want"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-aspiration.md
+        hyperparameters:
+          temperature: 0.6
+          max_tokens: 400
+
+      synthesizer:
+        description: "Integrates the three voices into a coherent response"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-synthesizer.md
+        hyperparameters:
+          temperature: 0.6
+          max_tokens: 500
+
+    deliberation_process: |
+      The psychodynamic learner simulates internal conflict:
+
+      1. DESIRE (Id): "I want to skip ahead / I'm bored / This is frustrating"
+         - Immediate emotional reactions
+         - Avoidance tendencies
+         - Curiosity and excitement
+
+      2. INTELLECT (Ego): "This doesn't make sense yet / I need more examples"
+         - Rational assessment of understanding
+         - Strategic thinking about learning path
+         - Reality testing
+
+      3. ASPIRATION (Superego): "I should master this / I want to be an expert"
+         - Long-term learning goals
+         - Internalized expectations
+         - Self-improvement drives
+
+      4. SYNTHESIS: Integration into coherent learner response
+         - Balances immediate desires with long-term goals
+         - Produces realistic learner behavior
+
+  # Dialectical: Hegelian thesis/antithesis/synthesis
+  # Internal deliberation where:
+  # - Thesis: Initial position/understanding
+  # - Antithesis: Challenge/complication to that position
+  # - Synthesis: New integrated understanding
+  dialectical:
+    name: "Dialectical Learner"
+    description: "Hegelian-inspired internal dialectic between thesis, antithesis, and synthesis"
+    deliberation:
+      enabled: true
+      max_rounds: 2
+      convergence_threshold: 0.7
+
+    agents:
+      thesis:
+        role: "thesis"
+        description: "Initial understanding or position"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-thesis.md
+        hyperparameters:
+          temperature: 0.6
+          max_tokens: 400
+
+      antithesis:
+        role: "antithesis"
+        description: "Challenge or complication to the thesis"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-antithesis.md
+        hyperparameters:
+          temperature: 0.7
+          max_tokens: 400
+
+      synthesis:
+        role: "synthesis"
+        description: "Integration that preserves and overcomes the tension"
+        provider: openrouter
+        model: nemotron
+        prompt_file: learner-synthesis.md
+        hyperparameters:
+          temperature: 0.6
+          max_tokens: 500
+
+    deliberation_process: |
+      The dialectical learner simulates Hegelian movement:
+
+      1. THESIS: "I understand X as..."
+         - Initial grasp of the concept
+         - Current mental model
+         - Working hypothesis
+
+      2. ANTITHESIS: "But wait, what about Y? That complicates things..."
+         - Internal contradiction discovered
+         - New information that doesn't fit
+         - Productive confusion
+
+      3. SYNTHESIS: "So actually, X and Y together mean..."
+         - New understanding that integrates both
+         - Aufhebung: preserves while overcoming
+         - Readiness for next dialectical cycle
+
+# ============================================================================
+# PERSONA MODIFIERS
+# ============================================================================
+# These modifiers adjust the base architecture based on learner persona
+
+persona_modifiers:
+  confused_novice:
+    desire_weight: 0.4      # Strong emotional reactions to confusion
+    intellect_weight: 0.3   # Struggles with analysis
+    aspiration_weight: 0.3  # Wants to succeed but uncertain
+
+  eager_explorer:
+    desire_weight: 0.5      # Curiosity-driven
+    intellect_weight: 0.3   # Quick but sometimes shallow analysis
+    aspiration_weight: 0.2  # Less concerned with "should"
+
+  focused_achiever:
+    desire_weight: 0.2      # Controlled emotional reactions
+    intellect_weight: 0.4   # Strong analytical focus
+    aspiration_weight: 0.4  # Clear goals
+
+  struggling_anxious:
+    desire_weight: 0.5      # Strong anxiety-driven responses
+    intellect_weight: 0.2   # Anxiety impairs analysis
+    aspiration_weight: 0.3  # High expectations create pressure
+
+  adversarial_tester:
+    desire_weight: 0.3      # Enjoys challenging
+    intellect_weight: 0.4   # Analytical about finding weaknesses
+    aspiration_weight: 0.3  # Wants to be thorough
+
+# ============================================================================
+# EVALUATION SETTINGS
+# ============================================================================
+
+evaluation:
+  # Track internal deliberation traces for analysis
+  log_deliberation: true
+  log_path: logs/learner-deliberation
+
+  # Metrics specific to multi-agent learner
+  metrics:
+    - deliberation_rounds
+    - internal_coherence
+    - desire_intellect_tension
+    - aspiration_alignment
+    - response_authenticity
+
+# ============================================================================
+# ABLATION STUDY SUPPORT
+# ============================================================================
+
+ablation:
+  # Mapping of ablation profiles to learner architectures
+  profile_architectures:
+    ablation_baseline_unified: unified
+    ablation_baseline_multilearner: psychodynamic
+    ablation_multiagent_unified: unified
+    ablation_multiagent_multilearner: psychodynamic
+    ablation_recognition_unified: unified
+    ablation_recognition_multilearner: psychodynamic
+    ablation_recognition_multiagent_unified: unified
+    ablation_recognition_multiagent_multilearner: psychodynamic
+
+  # Which architectures to compare in ablation studies
+  architectures_to_compare:
+    - unified
+    - psychodynamic

package/docs/research/ABLATION-DIALOGUE-ROUNDS.md

@@ -0,0 +1,52 @@
+# Ablation Study: Dialogue Rounds
+
+**Generated:** 2026-01-14T10:23:56.877Z
+
+## Research Question
+
+Does increasing the number of Ego-Superego dialogue rounds improve tutor suggestion quality?
+
+## Method
+
+Compared evaluation scores across profiles with different `max_rounds` settings:
+- **0 rounds**: Single-agent (no Superego review)
+- **1 round**: Single critique-revise cycle
+- **2 rounds**: Two critique-revise cycles (default)
+- **3 rounds**: Three critique-revise cycles
+
+## Results
+
+### Descriptive Statistics
+
+| Rounds | N | Mean | SD | 95% CI |
+|--------|---|------|-----|--------|
+| 0 | 483 | 91.58 | 15.75 | [90.2, 93.0] |
+| 1 | 1 | 50.00 | 0.00 | [50.0, 50.0] |
+| 2 | 247 | 88.05 | 19.77 | [85.6, 90.5] |
+| 3 | 2 | 96.25 | 1.77 | [93.8, 98.7] |
+
+### One-Way ANOVA
+
+| Source | SS | df | MS | F | p | η² |
+|--------|-----|-----|-----|-----|-----|-----|
+| Between | 3738.46 | 3 | 1246.15 | 4.212 | 0.050 | 0.017 |
+| Within | 215696.89 | 729 | 295.88 | | | |
+| Total | 219435.35 | 732 | | | | |
+
+## Interpretation
+
+The effect of dialogue rounds on suggestion quality was not statistically significant (F(3, 729) = 4.21, p = 0.050, η² = 0.017).
+
+Moving from single-agent (0 rounds) to multi-agent with 2 dialogue rounds shows a -3.9% improvement in mean score.
+
+## Limitations
+
+- Confounded with profile differences (model selection, prompts)
+- Unbalanced sample sizes across conditions
+- No randomized controlled comparison
+
+## Implications for System Design
+
+Based on these results:
+- Dialogue rounds may have limited impact compared to other factors
+- Consider whether additional API costs are justified

package/docs/research/ABLATION-MODEL-SELECTION.md

@@ -0,0 +1,53 @@
+# Ablation Study: Model Selection
+
+**Generated:** 2026-01-14T10:25:59.574Z
+
+## Research Question
+
+Does the choice of LLM model (for the Ego agent) affect tutor suggestion quality?
+
+## Method
+
+Analyzed evaluation scores grouped by the Ego model used in each profile.
+
+## Results
+
+### Descriptive Statistics
+
+| Model | N | Mean | SD | 95% CI |
+|-------|---|------|-----|--------|
+| deepseek | 442 | 93.31 | 13.10 | [92.1, 94.5] |
+| nemotron | 299 | 86.44 | 20.35 | [84.1, 88.7] |
+| haiku | 29 | 84.20 | 21.58 | [76.3, 92.1] |
+| gpt-5.2 | 1 | 97.50 | 0.00 | [97.5, 97.5] |
+| sonnet | 1 | 97.50 | 0.00 | [97.5, 97.5] |
+
+### One-Way ANOVA
+
+- F(4, 767) = 8.729
+- p < .05
+- η² = 0.044 (Small effect)
+
+### Model Ranking
+
+1. **gpt-5.2**: M = 97.50 (n=1)
+2. **sonnet**: M = 97.50 (n=1)
+3. **deepseek**: M = 93.31 (n=442)
+4. **nemotron**: M = 86.44 (n=299)
+5. **haiku**: M = 84.20 (n=29)
+
+## Interpretation
+
+Model selection has a statistically significant effect on suggestion quality (F(4, 767) = 8.73, p < .05, η² = 0.044).
+
+## Limitations
+
+- Confounded with profile differences (prompts, dialogue settings)
+- Unbalanced sample sizes across models
+- No direct A/B comparison with identical prompts
+
+## Implications
+
+- gpt-5.2 shows the highest mean score but with n=1 observations
+- deepseek offers good quality at minimal cost
+- Consider running controlled experiments varying only the model

package/docs/research/ADVANCED-EVAL-ANALYSIS.md

@@ -0,0 +1,60 @@
+# Advanced Evaluation Analysis
+
+**Generated:** 2026-01-14T11:07:14.890Z
+
+## Extended Recognition Scenarios
+
+These scenarios test recognition quality across multiple conversation turns, where learner responses are contingent on tutor suggestions.
+
+### Results Summary
+
+| Scenario | Turns | Baseline | Recognition | Diff | Cohen's d | Sig |
+|----------|-------|----------|-------------|------|-----------|-----|
+| Sustained Dialogue | 8 | 46.3 | 61.0 | +14.7 | 3.60 | * |
+| Breakdown Recovery | 6 | 57.5 | 71.3 | +13.8 | 2.23 | * |
+| Productive Struggle | 5 | 46.5 | 73.2 | +26.7 | 3.32 | * |
+| Mutual Transformation | 5 | 45.1 | 64.3 | +19.1 | 2.89 | * |
+
+**Aggregate Statistics:**
+- Average improvement: +18.6 points
+- Average effect size: d = 3.01
+- Significant effects: 4/4
+
+## Contingent Learner Analysis
+
+Multi-turn scenarios simulate realistic interactions where learner behavior depends on tutor suggestions. Recognition-enhanced tutoring maintains quality advantage even as:
+- Learners follow or reject suggestions
+- Conversations extend over multiple turns
+- Learners express frustration or confusion
+- Repair cycles become necessary
+
+## Bilateral Measurement Framework
+
+### Tutor Evaluation Dimensions
+1. **Mutual Recognition**: Acknowledges learner as autonomous subject
+2. **Dialectical Responsiveness**: Shaped by learner's specific input
+3. **Transformative Potential**: Enables genuine growth
+
+### Learner Evaluation Dimensions (Simulated)
+1. **Authenticity**: Genuine perspective contribution
+2. **Responsiveness**: Engagement with tutor suggestions
+3. **Development**: Growth across turns
+
+### Bilateral Metric
+> "Does engagement produce genuine mutual development?"
+
+## Integration with Statistical Findings
+
+The extended scenario results align with our factorial ANOVA findings:
+
+1. **Recognition Effect Persists**: The large recognition effect (η² = .422) is maintained across extended interactions, suggesting recognition-oriented prompting produces robust improvements.
+
+2. **Architecture Effect Context-Dependent**: The marginal architecture effect (η² = .034) may become more important in complex multi-turn scenarios requiring repair cycles.
+
+3. **Additive Benefits Confirmed**: No interaction effects suggest recognition benefits transfer across different scenario types and lengths.
+
+## Implications
+
+1. **Scalability**: Recognition-oriented design scales to longer interactions
+2. **Robustness**: Benefits persist even with contingent learner responses
+3. **Cost-Effectiveness**: Free-tier models achieve recognition quality with proper prompting

package/docs/research/ANOVA-RESULTS-2026-01-14.md

@@ -0,0 +1,257 @@
+# Two-Way ANOVA Results
+
+**Generated:** 2026-01-14T10:22:17.071Z
+**Data Source:** evaluations.db (factorial evaluation runs)
+
+```
+======================================================================
+TWO-WAY ANOVA RESULTS: 2×2 Factorial Design
+======================================================================
+
+EXPERIMENTAL DESIGN
+----------------------------------------------------------------------
+Factor A: Architecture (Single-Agent vs Multi-Agent)
+Factor B: Recognition (Standard vs Recognition-Enhanced Prompts)
+Total N: 76
+Grand Mean: 60.87
+
+CELL STATISTICS
+----------------------------------------------------------------------
+Cell                      N     Mean      SD      95% CI
+----------------------------------------------------------------------
+Single + Standard          8    39.99     9.12   [33.5, 46.4]
+Single + Recognition       6    70.36    26.15   [49.0, 91.7]
+Multi + Standard          31    50.16    15.00   [44.8, 55.5]
+Multi + Recognition       31    75.14    14.68   [69.9, 80.4]
+
+MARGINAL MEANS
+----------------------------------------------------------------------
+Architecture:  Single = 53.00,  Multi = 62.65
+Recognition:   Standard = 48.07,  Recognition = 74.37
+
+ANOVA TABLE
+----------------------------------------------------------------------
+Source              SS        df       MS         F        p       Sig
+----------------------------------------------------------------------
+Architecture (A)     1063.08       1     1063.08      4.445     0.050  *
+Recognition (B)     13123.82       1    13123.82     54.877     0.001  ***
+A × B                 124.13       1      124.13      0.519     0.250  
+Error               17218.77      72      239.15
+----------------------------------------------------------------------
+Total               31115.95      75            
+
+Significance: *** p < .05, * p < .10
+
+EFFECT SIZES
+----------------------------------------------------------------------
+Source              η²      Partial η²    Cohen's d    Interpretation
+----------------------------------------------------------------------
+Architecture (A)     0.034       0.058         0.62       Small
+Recognition (B)      0.422       0.433         1.70       Large
+A × B                0.004       0.007        N/A         Negligible
+
+MAIN EFFECTS (Raw Differences)
+----------------------------------------------------------------------
+Architecture Effect: Multi - Single = +9.65 points
+Recognition Effect:  Recognition - Standard = +26.29 points
+
+ASSUMPTION CHECKS
+----------------------------------------------------------------------
+Normality (Shapiro-Wilk approx): PASSED ✓
+Homogeneity of Variance (Levene): F = 3.31, p = 0.010 - VIOLATED ✗
+
+INTERPRETATION
+----------------------------------------------------------------------
+✓ Recognition prompts have a SIGNIFICANT effect (F = 54.88, p < .05)
+  Effect size: large (η² = 0.422)
+✗ Architecture effect is NOT significant (F = 4.45, p = 0.050)
+✗ No significant interaction (F = 0.52, p = 0.250)
+
+======================================================================
+```
+
+## JSON Results
+
+```json
+{
+  "grandMean": 60.87384654818866,
+  "N": 76,
+  "cellStats": {
+    "a0b0": {
+      "n": 8,
+      "mean": 39.98579545454547,
+      "std": 9.11985690330198,
+      "values": [
+        38.44696969696971,
+        38.63636363636365,
+        43.18181818181819,
+        34.848484848484865,
+        52.272727272727295,
+        37.500000000000014,
+        23.86363636363637,
+        51.136363636363654
+      ]
+    },
+    "a0b1": {
+      "n": 6,
+      "mean": 70.3598484848485,
+      "std": 26.148369552557515,
+      "values": [
+        50.3787878787879,
+        76.13636363636364,
+        100.00000000000003,
+        58.14393939393941,
+        37.50000000000002,
+        100.00000000000003
+      ]
+    },
+    "a1b0": {
+      "n": 31,
+      "mean": 50.16175580691711,
+      "std": 14.99861019348702,
+      "values": [
+        42.99242424242427,
+        32.95454545454547,
+        48.86363636363639,
+        49.05303030303031,
+        45.45454545454547,
+        59.09090909090911,
+        63.47402597402599,
+        50.974025974026,
+        58.11688311688312,
+        44.94949494949497,
+        52.02020202020204,
+        41.919191919191945,
+        38.63636363636365,
+        57.00757575757578,
+        48.4848484848485,
+        43.37121212121213,
+        43.181818181818194,
+        50.00000000000002,
+        45.90909090909093,
+        47.72727272727274,
+        53.863636363636395,
+        48.63636363636365,
+        42.80303030303031,
+        51.59090909090911,
+        43.93939393939396,
+        31.818181818181834,
+        18.181818181818194,
+        45.45454545454547,
+        87.5,
+        100,
+        67.04545454545458
+      ]
+    },
+    "a1b1": {
+      "n": 31,
+      "mean": 75.14040171298237,
+      "std": 14.684605144018311,
+      "values": [
+        75.18939393939395,
+        67.04545454545458,
+        100.00000000000003,
+        65.15151515151518,
+        56.818181818181834,
+        95.45454545454548,
+        78.08441558441562,
+        66.23376623376626,
+        69.64285714285717,
+        58.0808080808081,
+        62.12121212121214,
+        62.878787878787904,
+        60.03787878787881,
+        72.9166666666667,
+        82.00757575757578,
+        66.47727272727275,
+        63.636363636363654,
+        52.840909090909115,
+        67.72727272727275,
+        75.68181818181822,
+        60.68181818181819,
+        71.5909090909091,
+        84.09090909090911,
+        68.1818181818182,
+        74.05303030303033,
+        79.54545454545456,
+        93.18181818181822,
+        100.00000000000003,
+        100,
+        100,
+        100
+      ]
+    }
+  },
+  "marginalMeans": {
+    "architecture": {
+      "single": 53.003246753246756,
+      "multi": 62.65107875994973
+    },
+    "recognition": {
+      "standard": 48.07437932437934,
+      "recognition": 74.36517686517688
+    }
+  },
+  "anovaTable": {
+    "architecture": {
+      "SS": 1063.0791445902653,
+      "df": 1,
+      "MS": 1063.0791445902653,
+      "F": 4.445248575427072,
+      "p": 0.05,
+      "sig": false
+    },
+    "recognition": {
+      "SS": 13123.81985503855,
+      "df": 1,
+      "MS": 13123.81985503855,
+      "F": 54.87704449065894,
+      "p": 0.001,
+      "sig": true
+    },
+    "interaction": {
+      "SS": 124.12853011664384,
+      "df": 1,
+      "MS": 124.12853011664384,
+      "F": 0.5190414791586724,
+      "p": 0.25,
+      "sig": false
+    },
+    "error": {
+      "SS": 17218.76675999957,
+      "df": 72,
+      "MS": 239.14953833332734
+    },
+    "total": {
+      "SS": 31115.94559827812,
+      "df": 75
+    }
+  },
+  "effectSizes": {
+    "architecture": {
+      "etaSq": 0.03416509201793609,
+      "partialEtaSq": 0.05814944235600241,
+      "cohenD": 0.6238712311493667
+    },
+    "recognition": {
+      "etaSq": 0.42177152590743666,
+      "partialEtaSq": 0.4325214597404903,
+      "cohenD": 1.7000785480386178
+    },
+    "interaction": {
+      "etaSq": 0.003989225708233428,
+      "partialEtaSq": 0.007157313011477686
+    }
+  },
+  "mainEffects": {
+    "architecture": 9.647832006702977,
+    "recognition": 26.290797540797534
+  },
+  "assumptions": {
+    "normality": true,
+    "homogeneity": false,
+    "leveneF": 3.3126010309809213,
+    "leveneP": 0.01
+  }
+}
+```