lex-agentic-self 0.1.0

data/spec/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine_spec.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+RSpec.describe Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::MonitoringEngine do
+  subject(:engine) { described_class.new }
+
+  describe '#initialize' do
+    it 'starts with empty judgments' do
+      expect(engine.judgments).to be_empty
+    end
+
+    it 'starts with a CalibrationTracker' do
+      expect(engine.calibration).to be_a(
+        Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::CalibrationTracker
+      )
+    end
+
+    it 'starts with no domain calibrations' do
+      expect(engine.domain_calibrations).to be_empty
+    end
+  end
+
+  describe '#record_judgment' do
+    it 'creates and stores a MonitoringJudgment' do
+      judgment = engine.record_judgment(type: :feeling_of_knowing, domain: :episodic)
+      expect(engine.judgments[judgment.id]).to eq(judgment)
+    end
+
+    it 'returns a MonitoringJudgment' do
+      result = engine.record_judgment(type: :confidence_rating, domain: :semantic)
+      expect(result).to be_a(Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::MonitoringJudgment)
+    end
+
+    it 'stores predicted_confidence on the judgment' do
+      j = engine.record_judgment(type: :effort_estimate, domain: :test, predicted_confidence: 0.8)
+      expect(j.predicted_confidence).to eq(0.8)
+    end
+
+    it 'stores effort on the judgment' do
+      j = engine.record_judgment(type: :effort_estimate, domain: :test, effort: 0.9)
+      expect(j.effort_level).to eq(0.9)
+    end
+  end
+
+  describe '#resolve_judgment' do
+    let!(:judgment) { engine.record_judgment(type: :feeling_of_knowing, domain: :episodic, predicted_confidence: 0.7) }
+
+    it 'resolves the judgment' do
+      engine.resolve_judgment(judgment_id: judgment.id, actual_outcome: 0.6)
+      expect(judgment.resolved).to be true
+    end
+
+    it 'updates overall calibration tracker' do
+      engine.resolve_judgment(judgment_id: judgment.id, actual_outcome: 0.6)
+      expect(engine.calibration.count).to eq(1)
+    end
+
+    it 'updates domain calibration tracker' do
+      engine.resolve_judgment(judgment_id: judgment.id, actual_outcome: 0.6)
+      expect(engine.domain_calibrations[:episodic]).not_to be_nil
+      expect(engine.domain_calibrations[:episodic].count).to eq(1)
+    end
+
+    it 'returns nil for unknown judgment_id' do
+      result = engine.resolve_judgment(judgment_id: 'nonexistent', actual_outcome: 0.5)
+      expect(result).to be_nil
+    end
+
+    it 'returns the resolved judgment' do
+      result = engine.resolve_judgment(judgment_id: judgment.id, actual_outcome: 0.5)
+      expect(result).to eq(judgment)
+    end
+  end
+
+  describe '#feeling_of_knowing' do
+    it 'returns a MonitoringJudgment with type :feeling_of_knowing' do
+      j = engine.feeling_of_knowing(domain: :semantic)
+      expect(j.judgment_type).to eq(:feeling_of_knowing)
+    end
+
+    it 'uses query richness to influence confidence' do
+      short  = engine.feeling_of_knowing(domain: :test, query: 'x')
+      long   = engine.feeling_of_knowing(domain: :test, query: 'one two three four five six seven')
+      expect(long.predicted_confidence).to be >= short.predicted_confidence
+    end
+
+    it 'records the judgment in the engine' do
+      j = engine.feeling_of_knowing(domain: :test)
+      expect(engine.judgments[j.id]).to eq(j)
+    end
+  end
+
+  describe '#judgment_of_learning' do
+    it 'returns a MonitoringJudgment with type :judgment_of_learning' do
+      j = engine.judgment_of_learning(domain: :procedural)
+      expect(j.judgment_type).to eq(:judgment_of_learning)
+    end
+
+    it 'uses content length to influence confidence' do
+      short_j = engine.judgment_of_learning(domain: :test, content: 'hi')
+      long_j  = engine.judgment_of_learning(domain: :test, content: 'x' * 200)
+      expect(long_j.predicted_confidence).to be >= short_j.predicted_confidence
+    end
+  end
+
+  describe '#detect_overconfidence' do
+    it 'returns empty when no resolved judgments' do
+      engine.record_judgment(type: :confidence_rating, domain: :test)
+      expect(engine.detect_overconfidence).to be_empty
+    end
+
+    it 'returns overconfident judgments' do
+      j = engine.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.9)
+      engine.resolve_judgment(judgment_id: j.id, actual_outcome: 0.1)
+      expect(engine.detect_overconfidence).to include(j)
+    end
+
+    it 'does not include well-calibrated judgments' do
+      j = engine.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.7)
+      engine.resolve_judgment(judgment_id: j.id, actual_outcome: 0.65)
+      expect(engine.detect_overconfidence).not_to include(j)
+    end
+  end
+
+  describe '#detect_underconfidence' do
+    it 'returns underconfident judgments' do
+      j = engine.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.2)
+      engine.resolve_judgment(judgment_id: j.id, actual_outcome: 0.9)
+      expect(engine.detect_underconfidence).to include(j)
+    end
+  end
+
+  describe '#average_effort' do
+    it 'returns 0.0 when no judgments' do
+      expect(engine.average_effort).to eq(0.0)
+    end
+
+    it 'computes mean effort across judgments' do
+      engine.record_judgment(type: :effort_estimate, domain: :test, effort: 0.4)
+      engine.record_judgment(type: :effort_estimate, domain: :test, effort: 0.6)
+      expect(engine.average_effort).to eq(0.5)
+    end
+  end
+
+  describe '#calibration_report' do
+    it 'includes overall calibration' do
+      report = engine.calibration_report
+      expect(report).to have_key(:overall)
+    end
+
+    it 'includes by_domain breakdown' do
+      j = engine.record_judgment(type: :feeling_of_knowing, domain: :episodic, predicted_confidence: 0.7)
+      engine.resolve_judgment(judgment_id: j.id, actual_outcome: 0.6)
+      report = engine.calibration_report
+      expect(report[:by_domain]).to have_key(:episodic)
+    end
+
+    it 'reports total_resolved count' do
+      j = engine.record_judgment(type: :feeling_of_knowing, domain: :test, predicted_confidence: 0.7)
+      engine.resolve_judgment(judgment_id: j.id, actual_outcome: 0.6)
+      expect(engine.calibration_report[:total_resolved]).to eq(1)
+    end
+  end
+
+  describe '#monitoring_report' do
+    before do
+      j1 = engine.record_judgment(type: :feeling_of_knowing, domain: :test, predicted_confidence: 0.7)
+      engine.resolve_judgment(judgment_id: j1.id, actual_outcome: 0.3)
+      engine.record_judgment(type: :confidence_rating, domain: :test)
+    end
+
+    it 'includes total_judgments' do
+      expect(engine.monitoring_report[:total_judgments]).to eq(2)
+    end
+
+    it 'includes resolved_count' do
+      expect(engine.monitoring_report[:resolved_count]).to eq(1)
+    end
+
+    it 'includes unresolved_count' do
+      expect(engine.monitoring_report[:unresolved_count]).to eq(1)
+    end
+
+    it 'includes overconfident_count' do
+      expect(engine.monitoring_report).to have_key(:overconfident_count)
+    end
+
+    it 'includes domain_count' do
+      expect(engine.monitoring_report[:domain_count]).to eq(1)
+    end
+  end
+
+  describe '#to_h' do
+    it 'returns a summary hash' do
+      h = engine.to_h
+      expect(h).to include(:judgment_count, :calibration, :domain_count, :average_effort)
+    end
+  end
+end

data/spec/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_judgment_spec.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+RSpec.describe Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::MonitoringJudgment do
+  let(:judgment) do
+    described_class.new(
+      judgment_type:        :feeling_of_knowing,
+      domain:               :episodic,
+      predicted_confidence: 0.7,
+      effort_level:         0.4
+    )
+  end
+
+  describe '#initialize' do
+    it 'assigns a UUID id' do
+      expect(judgment.id).to match(/\A[0-9a-f-]{36}\z/)
+    end
+
+    it 'stores judgment_type' do
+      expect(judgment.judgment_type).to eq(:feeling_of_knowing)
+    end
+
+    it 'stores domain' do
+      expect(judgment.domain).to eq(:episodic)
+    end
+
+    it 'stores predicted_confidence' do
+      expect(judgment.predicted_confidence).to eq(0.7)
+    end
+
+    it 'stores effort_level' do
+      expect(judgment.effort_level).to eq(0.4)
+    end
+
+    it 'starts unresolved' do
+      expect(judgment.resolved).to be false
+    end
+
+    it 'starts with nil actual_outcome' do
+      expect(judgment.actual_outcome).to be_nil
+    end
+
+    it 'clamps predicted_confidence above 1.0 to 1.0' do
+      j = described_class.new(judgment_type: :confidence_rating, domain: :test, predicted_confidence: 1.5)
+      expect(j.predicted_confidence).to eq(1.0)
+    end
+
+    it 'clamps predicted_confidence below 0.0 to 0.0' do
+      j = described_class.new(judgment_type: :confidence_rating, domain: :test, predicted_confidence: -0.3)
+      expect(j.predicted_confidence).to eq(0.0)
+    end
+
+    it 'clamps effort_level above 1.0 to 1.0' do
+      j = described_class.new(judgment_type: :effort_estimate, domain: :test, effort_level: 2.0)
+      expect(j.effort_level).to eq(1.0)
+    end
+  end
+
+  describe '#resolve!' do
+    it 'sets actual_outcome' do
+      judgment.resolve!(actual: 0.6)
+      expect(judgment.actual_outcome).to eq(0.6)
+    end
+
+    it 'marks as resolved' do
+      judgment.resolve!(actual: 0.6)
+      expect(judgment.resolved).to be true
+    end
+
+    it 'returns self' do
+      result = judgment.resolve!(actual: 0.6)
+      expect(result).to eq(judgment)
+    end
+
+    it 'clamps actual_outcome to 0.0..1.0' do
+      judgment.resolve!(actual: 1.5)
+      expect(judgment.actual_outcome).to eq(1.0)
+    end
+  end
+
+  describe '#calibration_error' do
+    it 'returns nil when unresolved' do
+      expect(judgment.calibration_error).to be_nil
+    end
+
+    it 'returns predicted - actual when resolved' do
+      judgment.resolve!(actual: 0.5)
+      expect(judgment.calibration_error).to eq(0.2)
+    end
+
+    it 'returns negative when underconfident' do
+      judgment.resolve!(actual: 0.9)
+      expect(judgment.calibration_error).to be < 0
+    end
+  end
+
+  describe '#overconfident?' do
+    it 'returns false when unresolved' do
+      expect(judgment.overconfident?).to be false
+    end
+
+    it 'returns true when calibration_error > OVERCONFIDENCE_THRESHOLD' do
+      judgment.resolve!(actual: 0.1)
+      expect(judgment.overconfident?).to be true
+    end
+
+    it 'returns false when calibration_error is within range' do
+      judgment.resolve!(actual: 0.65)
+      expect(judgment.overconfident?).to be false
+    end
+  end
+
+  describe '#underconfident?' do
+    it 'returns false when unresolved' do
+      expect(judgment.underconfident?).to be false
+    end
+
+    it 'returns true when actual >> predicted' do
+      j = described_class.new(judgment_type: :feeling_of_knowing, domain: :test, predicted_confidence: 0.2)
+      j.resolve!(actual: 0.9)
+      expect(j.underconfident?).to be true
+    end
+
+    it 'returns false for well-calibrated judgment' do
+      judgment.resolve!(actual: 0.65)
+      expect(judgment.underconfident?).to be false
+    end
+  end
+
+  describe '#confidence_label' do
+    it 'returns :very_high for 0.9' do
+      j = described_class.new(judgment_type: :confidence_rating, domain: :test, predicted_confidence: 0.9)
+      expect(j.confidence_label).to eq(:very_high)
+    end
+
+    it 'returns :moderate for 0.5' do
+      j = described_class.new(judgment_type: :confidence_rating, domain: :test, predicted_confidence: 0.5)
+      expect(j.confidence_label).to eq(:moderate)
+    end
+
+    it 'returns :very_low for 0.1' do
+      j = described_class.new(judgment_type: :confidence_rating, domain: :test, predicted_confidence: 0.1)
+      expect(j.confidence_label).to eq(:very_low)
+    end
+  end
+
+  describe '#effort_label' do
+    it 'returns :extreme for 0.9' do
+      j = described_class.new(judgment_type: :effort_estimate, domain: :test, effort_level: 0.9)
+      expect(j.effort_label).to eq(:extreme)
+    end
+
+    it 'returns :minimal for 0.1' do
+      j = described_class.new(judgment_type: :effort_estimate, domain: :test, effort_level: 0.1)
+      expect(j.effort_label).to eq(:minimal)
+    end
+  end
+
+  describe '#to_h' do
+    it 'includes all key fields' do
+      h = judgment.to_h
+      expect(h).to include(:id, :judgment_type, :domain, :predicted_confidence, :actual_outcome,
+                           :effort_level, :resolved, :calibration_error, :confidence_label, :effort_label, :created_at)
+    end
+
+    it 'reflects resolved state after resolution' do
+      judgment.resolve!(actual: 0.6)
+      h = judgment.to_h
+      expect(h[:resolved]).to be true
+      expect(h[:actual_outcome]).to eq(0.6)
+    end
+  end
+end

data/spec/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring_spec.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/agentic/self/metacognitive_monitoring/client'
+
+RSpec.describe Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Runners::MetacognitiveMonitoring do
+  let(:engine) { Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::MonitoringEngine.new }
+  let(:client) { Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Client.new(engine: engine) }
+
+  describe '#record_judgment' do
+    it 'succeeds with valid type' do
+      result = client.record_judgment(type: :feeling_of_knowing, domain: :episodic)
+      expect(result[:success]).to be true
+    end
+
+    it 'returns a judgment_id' do
+      result = client.record_judgment(type: :feeling_of_knowing, domain: :episodic)
+      expect(result[:judgment_id]).to match(/\A[0-9a-f-]{36}\z/)
+    end
+
+    it 'returns the judgment hash' do
+      result = client.record_judgment(type: :feeling_of_knowing, domain: :episodic)
+      expect(result[:judgment]).to include(:id, :judgment_type, :domain)
+    end
+
+    it 'rejects invalid judgment type' do
+      result = client.record_judgment(type: :nonexistent, domain: :test)
+      expect(result[:success]).to be false
+      expect(result[:error]).to eq(:invalid_judgment_type)
+    end
+
+    it 'includes valid_types in error response' do
+      result = client.record_judgment(type: :bad_type, domain: :test)
+      expect(result[:valid_types]).to include(:feeling_of_knowing)
+    end
+
+    it 'accepts string type and converts to symbol' do
+      result = client.record_judgment(type: 'confidence_rating', domain: :test)
+      expect(result[:success]).to be true
+    end
+
+    it 'records judgment in engine' do
+      result = client.record_judgment(type: :effort_estimate, domain: :test)
+      expect(engine.judgments[result[:judgment_id]]).not_to be_nil
+    end
+
+    it 'uses provided predicted_confidence' do
+      result = client.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.85)
+      expect(result[:judgment][:predicted_confidence]).to eq(0.85)
+    end
+  end
+
+  describe '#resolve_judgment' do
+    let!(:recorded) { client.record_judgment(type: :feeling_of_knowing, domain: :episodic, predicted_confidence: 0.7) }
+
+    it 'succeeds for existing judgment' do
+      result = client.resolve_judgment(judgment_id: recorded[:judgment_id], actual_outcome: 0.6)
+      expect(result[:success]).to be true
+    end
+
+    it 'returns the judgment_id' do
+      result = client.resolve_judgment(judgment_id: recorded[:judgment_id], actual_outcome: 0.6)
+      expect(result[:judgment_id]).to eq(recorded[:judgment_id])
+    end
+
+    it 'returns the resolved judgment hash' do
+      result = client.resolve_judgment(judgment_id: recorded[:judgment_id], actual_outcome: 0.6)
+      expect(result[:judgment][:resolved]).to be true
+    end
+
+    it 'returns failure for unknown id' do
+      result = client.resolve_judgment(judgment_id: 'no-such-id', actual_outcome: 0.5)
+      expect(result[:success]).to be false
+      expect(result[:error]).to eq(:not_found)
+    end
+  end
+
+  describe '#feeling_of_knowing' do
+    it 'succeeds' do
+      result = client.feeling_of_knowing(domain: :episodic)
+      expect(result[:success]).to be true
+    end
+
+    it 'returns judgment_id' do
+      result = client.feeling_of_knowing(domain: :episodic)
+      expect(result[:judgment_id]).to match(/\A[0-9a-f-]{36}\z/)
+    end
+
+    it 'returns predicted_confidence' do
+      result = client.feeling_of_knowing(domain: :episodic)
+      expect(result[:predicted_confidence]).to be_between(0.0, 1.0)
+    end
+
+    it 'returns a confidence_label' do
+      result = client.feeling_of_knowing(domain: :episodic)
+      expect(result[:confidence_label]).to be_a(Symbol)
+    end
+
+    it 'reflects query in domain' do
+      result = client.feeling_of_knowing(domain: :semantic, query: 'what is ruby')
+      expect(result[:domain]).to eq(:semantic)
+    end
+  end
+
+  describe '#judgment_of_learning' do
+    it 'succeeds' do
+      result = client.judgment_of_learning(domain: :procedural)
+      expect(result[:success]).to be true
+    end
+
+    it 'returns judgment_id' do
+      result = client.judgment_of_learning(domain: :procedural)
+      expect(result[:judgment_id]).to match(/\A[0-9a-f-]{36}\z/)
+    end
+
+    it 'returns confidence_label' do
+      result = client.judgment_of_learning(domain: :procedural, content: 'some content to assess')
+      expect(result[:confidence_label]).to be_a(Symbol)
+    end
+  end
+
+  describe '#detect_overconfidence' do
+    context 'with no overconfident judgments' do
+      it 'returns empty findings' do
+        result = client.detect_overconfidence
+        expect(result[:count]).to eq(0)
+        expect(result[:findings]).to be_empty
+      end
+    end
+
+    context 'with overconfident judgments' do
+      before do
+        r = client.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.95)
+        client.resolve_judgment(judgment_id: r[:judgment_id], actual_outcome: 0.1)
+      end
+
+      it 'detects overconfidence' do
+        result = client.detect_overconfidence
+        expect(result[:count]).to eq(1)
+      end
+
+      it 'includes finding details' do
+        result = client.detect_overconfidence
+        expect(result[:findings].first).to include(:id, :predicted_confidence, :actual_outcome)
+      end
+    end
+  end
+
+  describe '#detect_underconfidence' do
+    context 'with underconfident judgment' do
+      before do
+        r = client.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.1)
+        client.resolve_judgment(judgment_id: r[:judgment_id], actual_outcome: 0.9)
+      end
+
+      it 'detects underconfidence' do
+        result = client.detect_underconfidence
+        expect(result[:count]).to eq(1)
+        expect(result[:success]).to be true
+      end
+    end
+  end
+
+  describe '#calibration_report' do
+    it 'returns success' do
+      result = client.calibration_report
+      expect(result[:success]).to be true
+    end
+
+    it 'includes report' do
+      result = client.calibration_report
+      expect(result[:report]).to include(:overall, :by_domain, :total_resolved)
+    end
+
+    it 'populates by_domain after resolving a domain judgment' do
+      r = client.record_judgment(type: :feeling_of_knowing, domain: :episodic, predicted_confidence: 0.7)
+      client.resolve_judgment(judgment_id: r[:judgment_id], actual_outcome: 0.6)
+      report = client.calibration_report
+      expect(report[:report][:by_domain]).to have_key(:episodic)
+    end
+  end
+
+  describe '#monitoring_report' do
+    before do
+      r = client.record_judgment(type: :feeling_of_knowing, domain: :test, predicted_confidence: 0.7)
+      client.resolve_judgment(judgment_id: r[:judgment_id], actual_outcome: 0.5)
+      client.record_judgment(type: :effort_estimate, domain: :test)
+    end
+
+    it 'returns success' do
+      expect(client.monitoring_report[:success]).to be true
+    end
+
+    it 'includes total_judgments' do
+      expect(client.monitoring_report[:report][:total_judgments]).to eq(2)
+    end
+
+    it 'includes resolved and unresolved counts' do
+      report = client.monitoring_report[:report]
+      expect(report[:resolved_count]).to eq(1)
+      expect(report[:unresolved_count]).to eq(1)
+    end
+  end
+
+  describe '#average_effort' do
+    it 'returns success' do
+      result = client.average_effort
+      expect(result[:success]).to be true
+    end
+
+    it 'returns 0.0 when no judgments' do
+      result = client.average_effort
+      expect(result[:average_effort]).to eq(0.0)
+    end
+
+    it 'includes effort_label' do
+      client.record_judgment(type: :effort_estimate, domain: :test, effort: 0.9)
+      result = client.average_effort
+      expect(result[:effort_label]).to be_a(Symbol)
+    end
+
+    it 'reflects window parameter' do
+      result = client.average_effort(window: 10)
+      expect(result[:window]).to eq(10)
+    end
+  end
+
+  describe '#calibration_curve' do
+    it 'returns success' do
+      expect(client.calibration_curve[:success]).to be true
+    end
+
+    it 'returns requested number of bins' do
+      result = client.calibration_curve(bins: 5)
+      expect(result[:bins]).to eq(5)
+    end
+
+    it 'returns curve array' do
+      r = client.record_judgment(type: :confidence_rating, domain: :test, predicted_confidence: 0.7)
+      client.resolve_judgment(judgment_id: r[:judgment_id], actual_outcome: 0.6)
+      result = client.calibration_curve(bins: 5)
+      expect(result[:curve]).to be_an(Array)
+    end
+  end
+end

data/spec/legion/extensions/agentic/self/narrative_arc/client_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/agentic/self/narrative_arc/client'
+
+RSpec.describe Legion::Extensions::Agentic::Self::NarrativeArc::Client do
+  it 'responds to all narrative runner methods' do
+    client = described_class.new
+    expect(client).to respond_to(:create_arc)
+    expect(client).to respond_to(:add_beat)
+    expect(client).to respond_to(:get_arc)
+    expect(client).to respond_to(:active_arcs)
+    expect(client).to respond_to(:completed_arcs)
+    expect(client).to respond_to(:most_dramatic_arc)
+    expect(client).to respond_to(:arc_report)
+  end
+
+  it 'initializes with an arc engine' do
+    client = described_class.new
+    result = client.arc_report
+    expect(result[:report][:total_arcs]).to eq(0)
+  end
+end