decision_agent 0.1.3 → 0.1.4

data/lib/decision_agent/monitoring/storage/memory_adapter.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require_relative "base_adapter"
+require "monitor"
+
+module DecisionAgent
+  module Monitoring
+    module Storage
+      # In-memory adapter for metrics storage (default, no dependencies)
+      class MemoryAdapter < BaseAdapter
+        include MonitorMixin
+
+        def initialize(window_size: 3600)
+          super()
+          @window_size = window_size
+          @metrics = {
+            decisions: [],
+            evaluations: [],
+            performance: [],
+            errors: []
+          }
+        end
+
+        def record_decision(decision, context, confidence: nil, evaluations_count: 0, duration_ms: nil, status: nil)
+          synchronize do
+            @metrics[:decisions] << {
+              decision: decision,
+              context: context,
+              confidence: confidence,
+              evaluations_count: evaluations_count,
+              duration_ms: duration_ms,
+              status: status,
+              timestamp: Time.now
+            }
+            cleanup_old_metrics
+          end
+        end
+
+        def record_evaluation(evaluator_name, score: nil, success: nil, duration_ms: nil, details: {})
+          synchronize do
+            @metrics[:evaluations] << {
+              evaluator_name: evaluator_name,
+              score: score,
+              success: success,
+              duration_ms: duration_ms,
+              details: details,
+              timestamp: Time.now
+            }
+            cleanup_old_metrics
+          end
+        end
+
+        def record_performance(operation, duration_ms: nil, status: nil, metadata: {})
+          synchronize do
+            @metrics[:performance] << {
+              operation: operation,
+              duration_ms: duration_ms,
+              status: status,
+              metadata: metadata,
+              timestamp: Time.now
+            }
+            cleanup_old_metrics
+          end
+        end
+
+        def record_error(error_type, message: nil, stack_trace: nil, severity: nil, context: {})
+          synchronize do
+            @metrics[:errors] << {
+              error_type: error_type,
+              message: message,
+              stack_trace: stack_trace,
+              severity: severity,
+              context: context,
+              timestamp: Time.now
+            }
+            cleanup_old_metrics
+          end
+        end
+
+        def statistics(time_range: 3600)
+          synchronize do
+            cutoff = Time.now - time_range
+            recent_decisions = @metrics[:decisions].select { |m| m[:timestamp] >= cutoff }
+            recent_evaluations = @metrics[:evaluations].select { |m| m[:timestamp] >= cutoff }
+            recent_performance = @metrics[:performance].select { |m| m[:timestamp] >= cutoff }
+            recent_errors = @metrics[:errors].select { |m| m[:timestamp] >= cutoff }
+
+            {
+              decisions: decision_statistics(recent_decisions),
+              evaluations: evaluation_statistics(recent_evaluations),
+              performance: performance_statistics(recent_performance),
+              errors: error_statistics(recent_errors)
+            }
+          end
+        end
+
+        def time_series(metric_type, bucket_size: 60, time_range: 3600)
+          synchronize do
+            cutoff = Time.now - time_range
+            metrics = @metrics[metric_type].select { |m| m[:timestamp] >= cutoff }
+
+            buckets = Hash.new(0)
+            metrics.each do |metric|
+              bucket = (metric[:timestamp].to_i / bucket_size) * bucket_size
+              buckets[bucket] += 1
+            end
+
+            timestamps = buckets.keys.sort
+            {
+              timestamps: timestamps.map { |ts| Time.at(ts).iso8601 },
+              data: timestamps.map { |ts| buckets[ts] }
+            }
+          end
+        end
+
+        def metrics_count
+          synchronize do
+            {
+              decisions: @metrics[:decisions].size,
+              evaluations: @metrics[:evaluations].size,
+              performance: @metrics[:performance].size,
+              errors: @metrics[:errors].size
+            }
+          end
+        end
+
+        def cleanup(older_than:)
+          synchronize do
+            cutoff = Time.now - older_than
+            count = 0
+
+            @metrics.each_value do |metric_array|
+              before_size = metric_array.size
+              metric_array.reject! { |m| m[:timestamp] < cutoff }
+              count += before_size - metric_array.size
+            end
+
+            count
+          end
+        end
+
+        def self.available?
+          true # Always available, no dependencies
+        end
+
+        private
+
+        def cleanup_old_metrics
+          cutoff = Time.now - @window_size
+          @metrics.each_value do |metric_array|
+            metric_array.reject! { |m| m[:timestamp] < cutoff }
+          end
+        end
+
+        def decision_statistics(decisions)
+          total = decisions.size
+          confidences = decisions.map { |d| d[:confidence] }.compact
+          statuses = decisions.map { |d| d[:status] }.compact
+
+          {
+            total: total,
+            by_decision: decisions.group_by { |d| d[:decision] }.transform_values(&:count),
+            average_confidence: confidences.empty? ? 0.0 : confidences.sum / confidences.size.to_f,
+            success_rate: calculate_success_rate(statuses)
+          }
+        end
+
+        def evaluation_statistics(evaluations)
+          total = evaluations.size
+          scores = evaluations.map { |e| e[:score] }.compact
+
+          {
+            total: total,
+            by_evaluator: evaluations.group_by { |e| e[:evaluator_name] }.transform_values(&:count),
+            average_score: scores.empty? ? 0.0 : scores.sum / scores.size.to_f,
+            success_rate_by_evaluator: evaluations.select { |e| e[:success] }
+                                                  .group_by { |e| e[:evaluator_name] }
+                                                  .transform_values(&:count)
+          }
+        end
+
+        def performance_statistics(performance_metrics)
+          total = performance_metrics.size
+          durations = performance_metrics.map { |p| p[:duration_ms] }.compact.sort
+          statuses = performance_metrics.map { |p| p[:status] }.compact
+
+          {
+            total: total,
+            average_duration_ms: durations.empty? ? 0.0 : durations.sum / durations.size.to_f,
+            p50: percentile(durations, 0.50),
+            p95: percentile(durations, 0.95),
+            p99: percentile(durations, 0.99),
+            success_rate: calculate_success_rate(statuses)
+          }
+        end
+
+        def error_statistics(errors)
+          {
+            total: errors.size,
+            by_type: errors.group_by { |e| e[:error_type] }.transform_values(&:count),
+            by_severity: errors.group_by { |e| e[:severity] }.transform_values(&:count),
+            critical_count: errors.count { |e| e[:severity] == "critical" }
+          }
+        end
+
+        def percentile(sorted_array, pct)
+          return 0.0 if sorted_array.empty?
+
+          index = ((sorted_array.length - 1) * pct).ceil
+          sorted_array[index].to_f
+        end
+
+        def calculate_success_rate(statuses)
+          return 0.0 if statuses.empty?
+
+          successful = statuses.count { |s| s == "success" }
+          successful.to_f / statuses.size
+        end
+      end
+    end
+  end
+end

data/lib/decision_agent/version.rb

@@ -1,3 +1,3 @@
 module DecisionAgent
-  VERSION = "0.1.3".freeze
+  VERSION = "0.1.4".freeze
 end

data/lib/decision_agent.rb

@@ -36,5 +36,12 @@ require_relative "decision_agent/monitoring/alert_manager"
 require_relative "decision_agent/monitoring/monitored_agent"
 # dashboard_server has additional dependencies (faye/websocket) - require it explicitly when needed
 
+require_relative "decision_agent/ab_testing/ab_test"
+require_relative "decision_agent/ab_testing/ab_test_assignment"
+require_relative "decision_agent/ab_testing/ab_test_manager"
+require_relative "decision_agent/ab_testing/ab_testing_agent"
+require_relative "decision_agent/ab_testing/storage/adapter"
+require_relative "decision_agent/ab_testing/storage/memory_adapter"
+
 module DecisionAgent
 end

data/lib/generators/decision_agent/install/install_generator.rb

@@ -10,6 +10,14 @@ module DecisionAgent
 
       desc "Installs DecisionAgent models and migrations for Rails"
 
+      class_option :monitoring, type: :boolean,
+                                default: false,
+                                desc: "Install monitoring tables and models for persistent metrics storage"
+
+      class_option :ab_testing, type: :boolean,
+                                default: false,
+                                desc: "Install A/B testing tables and models for variant testing"
+
       def self.next_migration_number(dirname)
         next_migration_number = current_migration_number(dirname) + 1
         ActiveRecord::Migration.next_migration_number(next_migration_number)
@@ -19,11 +27,40 @@ module DecisionAgent
         migration_template "migration.rb",
                            "db/migrate/create_decision_agent_tables.rb",
                            migration_version: migration_version
+
+        if options[:monitoring]
+          migration_template "monitoring_migration.rb",
+                             "db/migrate/create_decision_agent_monitoring_tables.rb",
+                             migration_version: migration_version
+        end
+
+        return unless options[:ab_testing]
+
+        migration_template "ab_testing_migration.rb",
+                           "db/migrate/create_decision_agent_ab_testing_tables.rb",
+                           migration_version: migration_version
       end
 
       def copy_models
         copy_file "rule.rb", "app/models/rule.rb"
         copy_file "rule_version.rb", "app/models/rule_version.rb"
+
+        if options[:monitoring]
+          copy_file "decision_log.rb", "app/models/decision_log.rb"
+          copy_file "evaluation_metric.rb", "app/models/evaluation_metric.rb"
+          copy_file "performance_metric.rb", "app/models/performance_metric.rb"
+          copy_file "error_metric.rb", "app/models/error_metric.rb"
+        end
+
+        return unless options[:ab_testing]
+
+        copy_file "ab_test_model.rb", "app/models/ab_test_model.rb"
+        copy_file "ab_test_assignment_model.rb", "app/models/ab_test_assignment_model.rb"
+      end
+
+      def copy_rake_tasks
+        copy_file "decision_agent_tasks.rake", "lib/tasks/decision_agent.rake" if options[:monitoring]
+        copy_file "ab_testing_tasks.rake", "lib/tasks/ab_testing.rake" if options[:ab_testing]
       end
 
       def show_readme

data/lib/generators/decision_agent/install/templates/ab_test_assignment_model.rb

@@ -0,0 +1,45 @@
+# A/B Test Assignment model for decision_agent gem
+# Tracks individual variant assignments and their results
+class ABTestAssignmentModel < ActiveRecord::Base
+  belongs_to :ab_test_model
+
+  validates :variant, presence: true, inclusion: { in: %w[champion challenger] }
+  validates :version_id, presence: true
+  validates :timestamp, presence: true
+
+  before_validation :set_defaults
+
+  # Scopes
+  scope :champion, -> { where(variant: "champion") }
+  scope :challenger, -> { where(variant: "challenger") }
+  scope :with_decisions, -> { where.not(decision_result: nil) }
+  scope :recent, -> { order(timestamp: :desc) }
+  scope :for_user, ->(user_id) { where(user_id: user_id) }
+
+  serialize :context, JSON
+
+  # Check if decision has been recorded
+  def decision_recorded?
+    decision_result.present?
+  end
+
+  # Get the test this assignment belongs to
+  def test
+    ab_test_model
+  end
+
+  # Record decision result
+  def record_decision!(decision, confidence_score)
+    update!(
+      decision_result: decision,
+      confidence: confidence_score
+    )
+  end
+
+  private
+
+  def set_defaults
+    self.timestamp ||= Time.current
+    self.context ||= {}
+  end
+end

data/lib/generators/decision_agent/install/templates/ab_test_model.rb

@@ -0,0 +1,54 @@
+# A/B Test model for decision_agent gem
+# Stores A/B test configurations
+class ABTestModel < ActiveRecord::Base
+  has_many :ab_test_assignment_models, dependent: :destroy
+
+  validates :name, presence: true
+  validates :champion_version_id, presence: true
+  validates :challenger_version_id, presence: true
+  validates :status, presence: true, inclusion: { in: %w[scheduled running completed cancelled] }
+
+  serialize :traffic_split, JSON
+
+  before_validation :set_defaults
+
+  # Scopes
+  scope :active, -> { where(status: "running") }
+  scope :scheduled, -> { where(status: "scheduled") }
+  scope :completed, -> { where(status: "completed") }
+  scope :running_or_scheduled, -> { where(status: %w[running scheduled]) }
+
+  # Check if test is currently running
+  def running?
+    status == "running" &&
+      (start_date.nil? || start_date <= Time.current) &&
+      (end_date.nil? || end_date > Time.current)
+  end
+
+  # Get statistics for this test
+  def statistics
+    {
+      total_assignments: ab_test_assignment_models.count,
+      champion_count: ab_test_assignment_models.where(variant: "champion").count,
+      challenger_count: ab_test_assignment_models.where(variant: "challenger").count,
+      with_decisions: ab_test_assignment_models.where.not(decision_result: nil).count,
+      avg_confidence: ab_test_assignment_models.where.not(confidence: nil).average(:confidence)&.to_f
+    }
+  end
+
+  # Get assignments by variant
+  def champion_assignments
+    ab_test_assignment_models.where(variant: "champion")
+  end
+
+  def challenger_assignments
+    ab_test_assignment_models.where(variant: "challenger")
+  end
+
+  private
+
+  def set_defaults
+    self.traffic_split ||= { champion: 90, challenger: 10 }
+    self.status ||= "scheduled"
+  end
+end

data/lib/generators/decision_agent/install/templates/ab_testing_migration.rb

@@ -0,0 +1,43 @@
+class CreateDecisionAgentABTestingTables < ActiveRecord::Migration[7.0]
+  def change
+    # A/B Tests table
+    create_table :ab_test_models do |t|
+      t.string :name, null: false
+      t.string :champion_version_id, null: false
+      t.string :challenger_version_id, null: false
+      t.text :traffic_split, null: false # JSON: { champion: 90, challenger: 10 }
+      t.datetime :start_date
+      t.datetime :end_date
+      t.string :status, null: false, default: "scheduled" # scheduled, running, completed, cancelled
+      t.timestamps
+    end
+
+    add_index :ab_test_models, :status
+    add_index :ab_test_models, :start_date
+    add_index :ab_test_models, %i[status start_date], name: "index_ab_tests_on_status_and_start_date"
+
+    # A/B Test Assignments table
+    create_table :ab_test_assignment_models do |t|
+      t.references :ab_test_model, null: false, foreign_key: true, index: true
+      t.string :user_id # Optional: for consistent assignment to same users
+      t.string :variant, null: false # "champion" or "challenger"
+      t.string :version_id, null: false
+      t.datetime :timestamp, null: false, default: -> { "CURRENT_TIMESTAMP" }
+
+      # Decision results (populated after decision is made)
+      t.string :decision_result
+      t.float :confidence
+      t.text :context # JSON: additional context
+
+      t.timestamps
+    end
+
+    add_index :ab_test_assignment_models, :user_id
+    add_index :ab_test_assignment_models, :variant
+    add_index :ab_test_assignment_models, :timestamp
+    add_index :ab_test_assignment_models, %i[ab_test_model_id variant], name: "index_assignments_on_test_and_variant"
+
+    # Optional: Index for querying assignments with decisions
+    add_index :ab_test_assignment_models, :decision_result, where: "decision_result IS NOT NULL", name: "index_assignments_with_decisions"
+  end
+end

data/lib/generators/decision_agent/install/templates/ab_testing_tasks.rake

@@ -0,0 +1,189 @@
+# rubocop:disable Metrics/BlockLength
+namespace :decision_agent do
+  namespace :ab_testing do
+    desc "List all A/B tests"
+    task list: :environment do
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      tests = manager.list_tests
+      puts "\n📊 A/B Tests (Total: #{tests.size})\n"
+      puts "=" * 80
+
+      if tests.empty?
+        puts "No A/B tests found."
+      else
+        tests.each do |test|
+          puts "\nID: #{test.id}"
+          puts "Name: #{test.name}"
+          puts "Status: #{test.status}"
+          puts "Champion Version: #{test.champion_version_id}"
+          puts "Challenger Version: #{test.challenger_version_id}"
+          puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
+          puts "Start Date: #{test.start_date}"
+          puts "End Date: #{test.end_date || 'N/A'}"
+          puts "-" * 80
+        end
+      end
+    end
+
+    desc "Show A/B test results - Usage: rake decision_agent:ab_testing:results[test_id]"
+    task :results, [:test_id] => :environment do |_t, args|
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      test_id = args[:test_id]
+      raise "Test ID is required. Usage: rake decision_agent:ab_testing:results[123]" unless test_id
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      results = manager.get_results(test_id)
+
+      puts "\n📊 A/B Test Results"
+      puts "=" * 80
+      puts "\nTest: #{results[:test][:name]}"
+      puts "Status: #{results[:test][:status]}"
+      puts "Total Assignments: #{results[:total_assignments]}"
+      puts ""
+
+      puts "🏆 Champion (Version #{results[:test][:champion_version_id]})"
+      puts "  Assignments: #{results[:champion][:total_assignments]}"
+      puts "  Decisions Recorded: #{results[:champion][:decisions_recorded]}"
+      if results[:champion][:avg_confidence]
+        puts "  Avg Confidence: #{results[:champion][:avg_confidence]}"
+        puts "  Min/Max Confidence: #{results[:champion][:min_confidence]} / #{results[:champion][:max_confidence]}"
+      end
+      puts ""
+
+      puts "🆕 Challenger (Version #{results[:test][:challenger_version_id]})"
+      puts "  Assignments: #{results[:challenger][:total_assignments]}"
+      puts "  Decisions Recorded: #{results[:challenger][:decisions_recorded]}"
+      if results[:challenger][:avg_confidence]
+        puts "  Avg Confidence: #{results[:challenger][:avg_confidence]}"
+        puts "  Min/Max Confidence: #{results[:challenger][:min_confidence]} / #{results[:challenger][:max_confidence]}"
+      end
+      puts ""
+
+      if results[:comparison][:statistical_significance] == "insufficient_data"
+        puts "⚠️  Insufficient data for statistical comparison"
+      else
+        puts "📈 Statistical Comparison"
+        puts "  Improvement: #{results[:comparison][:improvement_percentage]}%"
+        puts "  Winner: #{results[:comparison][:winner]}"
+        puts "  Statistical Significance: #{results[:comparison][:statistical_significance]}"
+        puts "  Confidence Level: #{(results[:comparison][:confidence_level] * 100).round(0)}%"
+        puts ""
+        puts "💡 Recommendation: #{results[:comparison][:recommendation]}"
+      end
+
+      puts "=" * 80
+    end
+
+    desc "Start an A/B test - Usage: rake decision_agent:ab_testing:start[test_id]"
+    task :start, [:test_id] => :environment do |_t, args|
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      test_id = args[:test_id]
+      raise "Test ID is required. Usage: rake decision_agent:ab_testing:start[123]" unless test_id
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      manager.start_test(test_id)
+      puts "✅ A/B test #{test_id} started successfully!"
+    end
+
+    desc "Complete an A/B test - Usage: rake decision_agent:ab_testing:complete[test_id]"
+    task :complete, [:test_id] => :environment do |_t, args|
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      test_id = args[:test_id]
+      raise "Test ID is required. Usage: rake decision_agent:ab_testing:complete[123]" unless test_id
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      manager.complete_test(test_id)
+      puts "✅ A/B test #{test_id} completed successfully!"
+    end
+
+    desc "Cancel an A/B test - Usage: rake decision_agent:ab_testing:cancel[test_id]"
+    task :cancel, [:test_id] => :environment do |_t, args|
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      test_id = args[:test_id]
+      raise "Test ID is required. Usage: rake decision_agent:ab_testing:cancel[123]" unless test_id
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      manager.cancel_test(test_id)
+      puts "✅ A/B test #{test_id} cancelled successfully!"
+    end
+
+    desc "Create a new A/B test - Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
+    task :create, %i[name champion_id challenger_id split] => :environment do |_t, args|
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      name = args[:name]
+      champion_id = args[:champion_id]
+      challenger_id = args[:challenger_id]
+      split = args[:split] || "90,10"
+
+      unless name && champion_id && challenger_id
+        raise "Missing arguments. Usage: rake decision_agent:ab_testing:create[name,champion_id,challenger_id,split]"
+      end
+
+      champion_pct, challenger_pct = split.split(",").map(&:to_i)
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      test = manager.create_test(
+        name: name,
+        champion_version_id: champion_id,
+        challenger_version_id: challenger_id,
+        traffic_split: { champion: champion_pct, challenger: challenger_pct }
+      )
+
+      puts "✅ A/B test created successfully!"
+      puts "   ID: #{test.id}"
+      puts "   Name: #{test.name}"
+      puts "   Status: #{test.status}"
+    end
+
+    desc "Show active A/B tests"
+    task active: :environment do
+      require "decision_agent/ab_testing/ab_test_manager"
+
+      manager = DecisionAgent::ABTesting::ABTestManager.new(
+        storage_adapter: DecisionAgent::ABTesting::Storage::ActiveRecordAdapter.new
+      )
+
+      tests = manager.active_tests
+      puts "\n🔄 Active A/B Tests (Total: #{tests.size})\n"
+      puts "=" * 80
+
+      if tests.empty?
+        puts "No active A/B tests found."
+      else
+        tests.each do |test|
+          puts "\nID: #{test.id}"
+          puts "Name: #{test.name}"
+          puts "Champion vs Challenger: #{test.champion_version_id} vs #{test.challenger_version_id}"
+          puts "Traffic Split: #{test.traffic_split[:champion]}% / #{test.traffic_split[:challenger]}%"
+          puts "-" * 80
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/BlockLength