lex-reward 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3289d5de024c3ca01934e873312a89a06e0ce8585c731ce66d484f843010fc59
+  data.tar.gz: 7bf0e6577cafb46e3740d9ec0d3e142111720c3fd5d84569c3f2ffb6969284e6
+SHA512:
+  metadata.gz: 71327dd3001af4e5bc4a417f69ebf1e62bfafb454234570b34e5bce982db454816eb1e3fbecb77065914f8b9306ea27575940feeaab80aac98d85835c36dafed
+  data.tar.gz: 9f4d6bf4b3a7b278b19383e50ee902a9ba23c26dfa718575d99d0775b45cbfba4bc66fbf099decb671113f8b00ae029c54566f4f63cd7cd9ce3aee1ab00677e5

data/lib/legion/extensions/reward/client.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/reward/helpers/constants'
+require 'legion/extensions/reward/helpers/reward_signal'
+require 'legion/extensions/reward/helpers/reward_store'
+require 'legion/extensions/reward/runners/reward'
+
+module Legion
+  module Extensions
+    module Reward
+      class Client
+        include Runners::Reward
+
+        attr_reader :reward_store
+
+        def initialize(reward_store: nil, **)
+          @reward_store = reward_store || Helpers::RewardStore.new
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/reward/helpers/constants.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Reward
+      module Helpers
+        module Constants
+          # Reward sources with weights (sum to 1.0)
+          # Each source contributes independently to the composite reward signal
+          REWARD_SOURCES = {
+            prediction_accuracy: { weight: 0.20, description: 'Correct predictions reinforced' },
+            curiosity_resolved:  { weight: 0.15, description: 'Wonder resolution satisfaction' },
+            goal_achieved:       { weight: 0.20, description: 'Intention completion reward' },
+            social_approval:     { weight: 0.10, description: 'Trust increase from peers' },
+            flow_state:          { weight: 0.10, description: 'Intrinsic flow motivation' },
+            error_avoidance:     { weight: 0.10, description: 'Low error rate maintenance' },
+            novelty_encounter:   { weight: 0.10, description: 'Novel experience exploration' },
+            homeostatic_balance: { weight: 0.05, description: 'System stability maintenance' }
+          }.freeze
+
+          # EMA alpha for running reward average
+          REWARD_ALPHA = 0.15
+
+          # EMA alpha for reward prediction (expected reward baseline)
+          PREDICTION_ALPHA = 0.1
+
+          # Minimum RPE magnitude to trigger learning signal
+          RPE_THRESHOLD = 0.05
+
+          # Reward signal range
+          REWARD_RANGE = { min: -1.0, max: 1.0 }.freeze
+
+          # RPE classification thresholds
+          RPE_LEVELS = {
+            large_positive: 0.3,   # "Way better than expected!" — strong reinforcement
+            positive:       0.1,   # "Better than expected" — moderate reinforcement
+            neutral:        0.05,  # "About as expected" — maintenance
+            negative:       -0.1,  # "Worse than expected" — mild suppression
+            large_negative: -0.3   # "Way worse than expected!" — strong suppression
+          }.freeze
+
+          # Temporal discount factor (per tick, for weighted history)
+          TEMPORAL_DISCOUNT = 0.95
+
+          # History cap
+          MAX_REWARD_HISTORY = 200
+
+          # Domain-specific reward history cap
+          MAX_DOMAIN_HISTORY = 50
+
+          # Anhedonia threshold — running average below this triggers concern
+          ANHEDONIA_THRESHOLD = -0.3
+
+          # Euphoria threshold — running average above this triggers concern
+          EUPHORIA_THRESHOLD = 0.7
+
+          # Reward momentum (how much prior reward influences next prediction)
+          MOMENTUM_WINDOW = 10
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/reward/helpers/reward_signal.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Reward
+      module Helpers
+        class RewardSignal
+          attr_reader :running_average, :predicted_reward, :last_rpe,
+                      :history, :domain_history, :tick_count
+
+          def initialize
+            @running_average = 0.0
+            @predicted_reward = 0.0
+            @last_rpe = 0.0
+            @history = []
+            @domain_history = {}
+            @tick_count = 0
+          end
+
+          def compute(source_signals)
+            @tick_count += 1
+            raw_reward = weighted_sum(source_signals)
+            reward = raw_reward.clamp(Constants::REWARD_RANGE[:min], Constants::REWARD_RANGE[:max])
+
+            @last_rpe = reward - @predicted_reward
+            @running_average = ema(@running_average, reward, Constants::REWARD_ALPHA)
+            @predicted_reward = ema(@predicted_reward, reward, Constants::PREDICTION_ALPHA)
+
+            record(reward, source_signals)
+
+            {
+              reward:           reward.round(4),
+              rpe:              @last_rpe.round(4),
+              rpe_class:        classify_rpe(@last_rpe),
+              running_average:  @running_average.round(4),
+              predicted_reward: @predicted_reward.round(4),
+              sources:          source_signals,
+              learning_signal:  learning_signal?
+            }
+          end
+
+          def record_domain_reward(domain, reward)
+            @domain_history[domain] ||= []
+            @domain_history[domain] << { reward: reward, at: Time.now.utc }
+            @domain_history[domain].shift while @domain_history[domain].size > Constants::MAX_DOMAIN_HISTORY
+          end
+
+          def domain_average(domain)
+            entries = @domain_history[domain]
+            return 0.0 if entries.nil? || entries.empty?
+
+            entries.sum { |e| e[:reward] } / entries.size.to_f
+          end
+
+          def domain_trend(domain)
+            entries = @domain_history[domain]
+            return :no_data if entries.nil? || entries.size < 5
+
+            recent = entries.last(10)
+            values = recent.map { |e| e[:reward] }
+            first_half = values[0...(values.size / 2)]
+            second_half = values[(values.size / 2)..]
+            diff = mean(second_half) - mean(first_half)
+
+            if diff > 0.05
+              :improving
+            elsif diff < -0.05
+              :declining
+            else
+              :stable
+            end
+          end
+
+          def anhedonic?
+            @running_average < Constants::ANHEDONIA_THRESHOLD
+          end
+
+          def euphoric?
+            @running_average > Constants::EUPHORIA_THRESHOLD
+          end
+
+          def learning_signal?
+            @last_rpe.abs >= Constants::RPE_THRESHOLD
+          end
+
+          def recent_rewards(limit = 20)
+            @history.last(limit)
+          end
+
+          def discounted_return(window = nil)
+            entries = window ? @history.last(window) : @history
+            return 0.0 if entries.empty?
+
+            total = 0.0
+            entries.reverse_each.with_index do |entry, idx|
+              total += entry[:reward] * (Constants::TEMPORAL_DISCOUNT**idx)
+            end
+            total
+          end
+
+          def reward_volatility
+            return 0.0 if @history.size < 3
+
+            recent = @history.last(Constants::MOMENTUM_WINDOW).map { |h| h[:reward] }
+            avg = mean(recent)
+            variance = recent.sum { |r| (r - avg)**2 } / recent.size.to_f
+            Math.sqrt(variance)
+          end
+
+          def to_h
+            {
+              running_average:  @running_average.round(4),
+              predicted_reward: @predicted_reward.round(4),
+              last_rpe:         @last_rpe.round(4),
+              rpe_class:        classify_rpe(@last_rpe),
+              tick_count:       @tick_count,
+              learning_signal:  learning_signal?,
+              anhedonic:        anhedonic?,
+              euphoric:         euphoric?,
+              volatility:       reward_volatility.round(4),
+              domains_tracked:  @domain_history.keys.size,
+              history_size:     @history.size
+            }
+          end
+
+          private
+
+          def weighted_sum(source_signals)
+            total = 0.0
+            Constants::REWARD_SOURCES.each do |source, config|
+              value = source_signals[source] || 0.0
+              total += value * config[:weight]
+            end
+            total
+          end
+
+          def classify_rpe(rpe)
+            if rpe >= Constants::RPE_LEVELS[:large_positive]
+              :large_positive
+            elsif rpe >= Constants::RPE_LEVELS[:positive]
+              :positive
+            elsif rpe >= -Constants::RPE_LEVELS[:neutral]
+              :neutral
+            elsif rpe >= Constants::RPE_LEVELS[:large_negative]
+              :negative
+            else
+              :large_negative
+            end
+          end
+
+          def ema(current, observed, alpha)
+            (current * (1.0 - alpha)) + (observed * alpha)
+          end
+
+          def mean(values)
+            return 0.0 if values.empty?
+
+            values.sum / values.size.to_f
+          end
+
+          def record(reward, sources)
+            @history << {
+              reward:  reward,
+              rpe:     @last_rpe,
+              sources: sources,
+              at:      Time.now.utc
+            }
+            @history.shift while @history.size > Constants::MAX_REWARD_HISTORY
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/reward/helpers/reward_store.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Reward
+      module Helpers
+        class RewardStore
+          attr_reader :signal
+
+          def initialize(signal: nil)
+            @signal = signal || RewardSignal.new
+          end
+
+          def process_tick(tick_results)
+            source_signals = extract_signals(tick_results)
+            result = @signal.compute(source_signals)
+
+            domain = extract_domain(tick_results)
+            @signal.record_domain_reward(domain, result[:reward]) if domain
+
+            result
+          end
+
+          def domain_report(domain)
+            {
+              domain:  domain,
+              average: @signal.domain_average(domain),
+              trend:   @signal.domain_trend(domain),
+              history: @signal.domain_history[domain]&.last(10) || []
+            }
+          end
+
+          def all_domain_averages
+            @signal.domain_history.keys.to_h do |domain|
+              [domain, @signal.domain_average(domain).round(4)]
+            end
+          end
+
+          def health_assessment
+            avg = @signal.running_average
+            vol = @signal.reward_volatility
+
+            if @signal.anhedonic?
+              { status: :anhedonic, description: 'Persistently low reward — possible disengagement', severity: :high }
+            elsif @signal.euphoric?
+              { status: :euphoric, description: 'Persistently high reward — possible overconfidence', severity: :moderate }
+            elsif vol > 0.4
+              { status: :volatile, description: 'Highly variable reward — unstable learning signals', severity: :moderate }
+            elsif avg.between?(-0.1, 0.1)
+              { status: :neutral, description: 'Low reward signal — minimal learning happening', severity: :low }
+            else
+              { status: :healthy, description: 'Balanced reward signal — healthy learning', severity: :none }
+            end
+          end
+
+          private
+
+          def extract_signals(tick_results)
+            {
+              prediction_accuracy: extract_prediction_reward(tick_results),
+              curiosity_resolved:  extract_curiosity_reward(tick_results),
+              goal_achieved:       extract_goal_reward(tick_results),
+              social_approval:     extract_social_reward(tick_results),
+              flow_state:          extract_flow_reward(tick_results),
+              error_avoidance:     extract_error_reward(tick_results),
+              novelty_encounter:   extract_novelty_reward(tick_results),
+              homeostatic_balance: extract_homeostatic_reward(tick_results)
+            }
+          end
+
+          def extract_prediction_reward(tick_results)
+            accuracy = tick_results.dig(:prediction_engine, :rolling_accuracy)
+            return 0.0 unless accuracy
+
+            (accuracy - 0.5) * 2.0
+          end
+
+          def extract_curiosity_reward(tick_results)
+            resolved = tick_results.dig(:curiosity, :resolved_count) || 0
+            intensity = tick_results.dig(:curiosity, :intensity) || 0.0
+
+            resolved_signal = [resolved * 0.3, 1.0].min
+            (resolved_signal + (intensity * 0.2)).clamp(-1.0, 1.0)
+          end
+
+          def extract_goal_reward(tick_results)
+            completed = tick_results.dig(:volition, :completed_count) || 0
+            failed = tick_results.dig(:volition, :failed_count) || 0
+
+            ((completed * 0.4) - (failed * 0.3)).clamp(-1.0, 1.0)
+          end
+
+          def extract_social_reward(tick_results)
+            trust_delta = tick_results.dig(:trust, :composite_delta) || 0.0
+            (trust_delta * 2.0).clamp(-1.0, 1.0)
+          end
+
+          def extract_flow_reward(tick_results)
+            in_flow = tick_results.dig(:flow, :in_flow)
+            score = tick_results.dig(:flow, :score) || 0.0
+
+            return score * 0.8 if in_flow
+
+            -0.1
+          end
+
+          def extract_error_reward(tick_results)
+            error_rate = tick_results.dig(:prediction_engine, :error_rate)
+            return 0.0 unless error_rate
+
+            (1.0 - (error_rate * 2.0)).clamp(-1.0, 1.0)
+          end
+
+          def extract_novelty_reward(tick_results)
+            novelty = tick_results.dig(:attention, :novelty_score) || 0.0
+            spotlight_count = tick_results.dig(:attention, :spotlight_count) || 0
+
+            ((novelty * 0.5) + [spotlight_count * 0.1, 0.5].min).clamp(-1.0, 1.0)
+          end
+
+          def extract_homeostatic_reward(tick_results)
+            deviation = tick_results.dig(:homeostasis, :worst_deviation) || 0.0
+            allostatic = tick_results.dig(:homeostasis, :allostatic_load) || 0.0
+
+            stability = 1.0 - [deviation, allostatic].max
+            (stability - 0.5).clamp(-1.0, 1.0)
+          end
+
+          def extract_domain(tick_results)
+            tick_results.dig(:volition, :current_domain) ||
+              tick_results.dig(:curiosity, :active_domain) ||
+              tick_results.dig(:attention, :focus_domain)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/reward/runners/reward.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Reward
+      module Runners
+        module Reward
+          include Legion::Extensions::Helpers::Lex if Legion::Extensions.const_defined?(:Helpers) &&
+                                                      Legion::Extensions::Helpers.const_defined?(:Lex)
+
+          def compute_reward(tick_results: {}, **)
+            result = reward_store.process_tick(tick_results)
+
+            Legion::Logging.debug "[reward] reward=#{result[:reward]} rpe=#{result[:rpe]} " \
+                                  "class=#{result[:rpe_class]} learning=#{result[:learning_signal]}"
+
+            result
+          end
+
+          def reward_status(**)
+            sig = reward_store.signal
+            health = reward_store.health_assessment
+
+            Legion::Logging.debug "[reward] status: avg=#{sig.running_average.round(3)} " \
+                                  "predicted=#{sig.predicted_reward.round(3)} health=#{health[:status]}"
+
+            sig.to_h.merge(health: health)
+          end
+
+          def reward_for(domain:, **)
+            report = reward_store.domain_report(domain)
+            Legion::Logging.debug "[reward] domain=#{domain} avg=#{report[:average].round(3)} trend=#{report[:trend]}"
+            report
+          end
+
+          def reward_history(limit: 20, **)
+            recent = reward_store.signal.recent_rewards(limit)
+            Legion::Logging.debug "[reward] history: #{recent.size} entries"
+
+            {
+              history:           recent,
+              total:             reward_store.signal.history.size,
+              discounted_return: reward_store.signal.discounted_return(limit).round(4)
+            }
+          end
+
+          def domain_rewards(**)
+            averages = reward_store.all_domain_averages
+            Legion::Logging.debug "[reward] domains: #{averages.size} tracked"
+
+            {
+              domains:      averages,
+              domain_count: averages.size,
+              best_domain:  averages.max_by { |_, v| v }&.first,
+              worst_domain: averages.min_by { |_, v| v }&.first
+            }
+          end
+
+          def reward_stats(**)
+            sig = reward_store.signal
+            health = reward_store.health_assessment
+
+            Legion::Logging.debug '[reward] stats'
+
+            {
+              running_average:   sig.running_average.round(4),
+              predicted_reward:  sig.predicted_reward.round(4),
+              volatility:        sig.reward_volatility.round(4),
+              tick_count:        sig.tick_count,
+              health:            health,
+              domains_tracked:   sig.domain_history.keys.size,
+              history_size:      sig.history.size,
+              discounted_return: sig.discounted_return.round(4),
+              anhedonic:         sig.anhedonic?,
+              euphoric:          sig.euphoric?
+            }
+          end
+
+          private
+
+          def reward_store
+            @reward_store ||= Helpers::RewardStore.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/reward/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Reward
+      VERSION = '0.1.0'
+    end
+  end
+end

data/lib/legion/extensions/reward.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/reward/version'
+require 'legion/extensions/reward/helpers/constants'
+require 'legion/extensions/reward/helpers/reward_signal'
+require 'legion/extensions/reward/helpers/reward_store'
+require 'legion/extensions/reward/runners/reward'
+require 'legion/extensions/reward/client'
+
+module Legion
+  module Extensions
+    module Reward
+      extend Legion::Extensions::Core if Legion::Extensions.const_defined?(:Core)
+    end
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: lex-reward
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matthew Iverson
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: legion-gaia
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Computes internal reward signals from cognitive outcomes, tracks reward
+  prediction error, and drives reinforcement learning
+email:
+- matt@legionIO.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/legion/extensions/reward.rb
+- lib/legion/extensions/reward/client.rb
+- lib/legion/extensions/reward/helpers/constants.rb
+- lib/legion/extensions/reward/helpers/reward_signal.rb
+- lib/legion/extensions/reward/helpers/reward_store.rb
+- lib/legion/extensions/reward/runners/reward.rb
+- lib/legion/extensions/reward/version.rb
+homepage: https://github.com/LegionIO/lex-reward
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Dopaminergic reward signal engine for LegionIO cognitive agents
+test_files: []