decision_agent 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c8a831b21d5bb62dbb4948827cc0b4bc9bcf021b52f04a2d9141d1339287274
-  data.tar.gz: 97d80cab6513385dc9e9b4bddb7555da1e8629488ce358319aa0842e74cf41e1
+  metadata.gz: fac7efbaf0c8afd76053d21e611eca64f5880d5b6434720fd90ae34549f6244d
+  data.tar.gz: ecc22d0a9bb19053fd103b06f2c1be768db432b6d9ed715eac8adddcbd6f68dc
 SHA512:
-  metadata.gz: dcfa8f3cc0a5931a1a163b97caf7586c17eddfee38850b52b832fa1a94d720342868c6823158597d0dafa14bf7df82b9d9b394fdffac3c74d06a556eca2b5c18
-  data.tar.gz: f183fc1b8c021e589571d0d9a97c3b2679fec3b529d2e73a76a4b7752a9792dcba77f4914de876c5448711416179dcbd0ba35b515fe1c4f2a74bcab27c1a873e
+  metadata.gz: b7a0be6ad95512697b6e49a8145668cde64256051db57a39cf925edbd93886d7772cb93026a9af7a9eaea2e9968756ec628ca94d0ebc5a0d1aef0c409eeae97d
+  data.tar.gz: 06777a3c4155b1adff552f712a05a0d4bd2694c6f8af631e36611528c050cf1ea711268230dba6d1a64522e127c0c312b7221fd5d36bdc56f69009b0a98c17e4

data/lib/decision_agent/ab_testing/ab_test.rb

@@ -0,0 +1,197 @@
+module DecisionAgent
+  module ABTesting
+    # Represents an A/B test configuration for comparing rule versions
+    class ABTest
+      attr_reader :id, :name, :champion_version_id, :challenger_version_id,
+                  :traffic_split, :start_date, :end_date, :status
+
+      # @param name [String] Name of the A/B test
+      # @param champion_version_id [String, Integer] ID of the current/champion version
+      # @param challenger_version_id [String, Integer] ID of the new/challenger version
+      # @param options [Hash] Optional configuration
+      # @option options [Hash] :traffic_split Traffic distribution (default: { champion: 90, challenger: 10 })
+      # @option options [Time] :start_date When the test starts (defaults to now)
+      # @option options [Time] :end_date When the test ends (optional)
+      # @option options [String] :status Test status: running, completed, cancelled, scheduled
+      # @option options [String, Integer] :id Optional ID (for persistence)
+      def initialize(
+        name:,
+        champion_version_id:,
+        challenger_version_id:,
+        **options
+      )
+        @id = options[:id]
+        @name = name
+        @champion_version_id = champion_version_id
+        @challenger_version_id = challenger_version_id
+        @traffic_split = normalize_traffic_split(options[:traffic_split] || { champion: 90, challenger: 10 })
+        @start_date = options[:start_date] || Time.now.utc
+        @end_date = options[:end_date]
+        @status = options[:status] || "scheduled"
+
+        validate!
+      end
+
+      # Assign a variant based on traffic split
+      # Uses consistent hashing to ensure same user gets same variant
+      # @param user_id [String, nil] Optional user identifier for consistent assignment
+      # @return [Symbol] :champion or :challenger
+      def assign_variant(user_id: nil)
+        raise TestNotRunningError, "Test '#{@name}' is not running (status: #{@status})" unless running?
+
+        if user_id
+          # Consistent hashing: same user always gets same variant
+          hash_value = Digest::SHA256.hexdigest("#{@id}:#{user_id}").to_i(16)
+          percentage = hash_value % 100
+        else
+          # Random assignment
+          percentage = rand(100)
+        end
+
+        percentage < @traffic_split[:champion] ? :champion : :challenger
+      end
+
+      # Get the version ID for the assigned variant
+      # @param variant [Symbol] :champion or :challenger
+      # @return [String, Integer] The version ID
+      def version_for_variant(variant)
+        case variant
+        when :champion
+          @champion_version_id
+        when :challenger
+          @challenger_version_id
+        else
+          raise ArgumentError, "Invalid variant: #{variant}. Must be :champion or :challenger"
+        end
+      end
+
+      # Check if test is currently running
+      # @return [Boolean]
+      def running?
+        return false unless @status == "running"
+        return false if @start_date && Time.now.utc < @start_date
+        return false if @end_date && Time.now.utc > @end_date
+
+        true
+      end
+
+      # Check if test is scheduled to start
+      # @return [Boolean]
+      def scheduled?
+        @status == "scheduled" && @start_date && Time.now.utc < @start_date
+      end
+
+      # Check if test is completed
+      # @return [Boolean]
+      def completed?
+        @status == "completed" || (@end_date && Time.now.utc > @end_date)
+      end
+
+      # Start the test
+      def start!
+        raise InvalidStatusTransitionError, "Cannot start test from status: #{@status}" unless can_start?
+
+        @status = "running"
+        @start_date = Time.now.utc if @start_date.nil? || @start_date > Time.now.utc
+      end
+
+      # Complete the test
+      def complete!
+        raise InvalidStatusTransitionError, "Cannot complete test from status: #{@status}" unless can_complete?
+
+        @status = "completed"
+        @end_date = Time.now.utc
+      end
+
+      # Cancel the test
+      def cancel!
+        raise InvalidStatusTransitionError, "Cannot cancel test from status: #{@status}" if @status == "completed"
+
+        @status = "cancelled"
+      end
+
+      # Convert to hash representation
+      # @return [Hash]
+      def to_h
+        {
+          id: @id,
+          name: @name,
+          champion_version_id: @champion_version_id,
+          challenger_version_id: @challenger_version_id,
+          traffic_split: @traffic_split,
+          start_date: @start_date,
+          end_date: @end_date,
+          status: @status
+        }
+      end
+
+      private
+
+      def validate!
+        raise ValidationError, "Test name is required" if @name.nil? || @name.strip.empty?
+        raise ValidationError, "Champion version ID is required" if @champion_version_id.nil?
+        raise ValidationError, "Challenger version ID is required" if @challenger_version_id.nil?
+        raise ValidationError, "Champion and challenger must be different versions" if @champion_version_id == @challenger_version_id
+
+        validate_traffic_split!
+        validate_dates!
+        validate_status!
+      end
+
+      def validate_traffic_split!
+        raise ValidationError, "Traffic split must be a Hash" unless @traffic_split.is_a?(Hash)
+
+        unless @traffic_split.key?(:champion) && @traffic_split.key?(:challenger)
+          raise ValidationError,
+                "Traffic split must have :champion and :challenger keys"
+        end
+
+        total = @traffic_split[:champion] + @traffic_split[:challenger]
+        raise ValidationError, "Traffic split must sum to 100, got #{total}" unless total == 100
+
+        raise ValidationError, "Traffic percentages must be non-negative" if @traffic_split.values.any?(&:negative?)
+      end
+
+      def validate_dates!
+        return unless @start_date && @end_date
+
+        raise ValidationError, "End date must be after start date" if @end_date <= @start_date
+      end
+
+      def validate_status!
+        valid_statuses = %w[scheduled running completed cancelled]
+        return if valid_statuses.include?(@status)
+
+        raise ValidationError, "Invalid status: #{@status}. Must be one of: #{valid_statuses.join(', ')}"
+      end
+
+      def normalize_traffic_split(split)
+        case split
+        when Hash
+          # Handle both string and symbol keys
+          {
+            champion: (split[:champion] || split["champion"] || 50).to_i,
+            challenger: (split[:challenger] || split["challenger"] || 50).to_i
+          }
+        when Array
+          # Handle array format [90, 10]
+          { champion: split[0].to_i, challenger: split[1].to_i }
+        else
+          raise ValidationError, "Traffic split must be a Hash or Array"
+        end
+      end
+
+      def can_start?
+        %w[scheduled].include?(@status)
+      end
+
+      def can_complete?
+        %w[running].include?(@status)
+      end
+    end
+
+    # Custom errors
+    class TestNotRunningError < StandardError; end
+    class InvalidStatusTransitionError < StandardError; end
+  end
+end

data/lib/decision_agent/ab_testing/ab_test_assignment.rb

@@ -0,0 +1,76 @@
+module DecisionAgent
+  module ABTesting
+    # Tracks individual assignments of users/requests to A/B test variants
+    class ABTestAssignment
+      attr_reader :id, :ab_test_id, :user_id, :variant, :version_id,
+                  :timestamp, :decision_result, :confidence, :context
+
+      # @param ab_test_id [String, Integer] The A/B test ID
+      # @param variant [Symbol] :champion or :challenger
+      # @param version_id [String, Integer] The rule version ID that was used
+      # @param options [Hash] Optional configuration
+      # @option options [String] :user_id User identifier (optional)
+      # @option options [Time] :timestamp When the assignment occurred
+      # @option options [String] :decision_result The decision outcome
+      # @option options [Float] :confidence Confidence score of the decision
+      # @option options [Hash] :context Additional context for the decision
+      # @option options [String, Integer] :id Optional ID (for persistence)
+      def initialize(
+        ab_test_id:,
+        variant:,
+        version_id:,
+        **options
+      )
+        @id = options[:id]
+        @ab_test_id = ab_test_id
+        @user_id = options[:user_id]
+        @variant = variant
+        @version_id = version_id
+        @timestamp = options[:timestamp] || Time.now.utc
+        @decision_result = options[:decision_result]
+        @confidence = options[:confidence]
+        @context = options[:context] || {}
+
+        validate!
+      end
+
+      # Update the assignment with decision results
+      # @param decision [String] The decision result
+      # @param confidence [Float] The confidence score
+      def record_decision(decision, confidence)
+        @decision_result = decision
+        @confidence = confidence
+      end
+
+      # Convert to hash representation
+      # @return [Hash]
+      def to_h
+        {
+          id: @id,
+          ab_test_id: @ab_test_id,
+          user_id: @user_id,
+          variant: @variant,
+          version_id: @version_id,
+          timestamp: @timestamp,
+          decision_result: @decision_result,
+          confidence: @confidence,
+          context: @context
+        }
+      end
+
+      private
+
+      def validate!
+        raise ValidationError, "AB test ID is required" if @ab_test_id.nil?
+        raise ValidationError, "Variant is required" if @variant.nil?
+        raise ValidationError, "Version ID is required" if @version_id.nil?
+
+        raise ValidationError, "Variant must be :champion or :challenger, got: #{@variant}" unless %i[champion challenger].include?(@variant)
+
+        return unless @confidence && (@confidence.negative? || @confidence > 1)
+
+        raise ValidationError, "Confidence must be between 0 and 1, got: #{@confidence}"
+      end
+    end
+  end
+end

data/lib/decision_agent/ab_testing/ab_test_manager.rb

@@ -0,0 +1,317 @@
+require "monitor"
+
+module DecisionAgent
+  module ABTesting
+    # Manages A/B tests and provides high-level orchestration
+    class ABTestManager
+      include MonitorMixin
+
+      attr_reader :storage_adapter, :version_manager
+
+      # @param storage_adapter [Storage::Adapter] Storage adapter for persistence
+      # @param version_manager [Versioning::VersionManager] Version manager for rule versions
+      def initialize(storage_adapter: nil, version_manager: nil)
+        super()
+        @storage_adapter = storage_adapter || default_storage_adapter
+        @version_manager = version_manager || Versioning::VersionManager.new
+        @active_tests_cache = {}
+      end
+
+      # Create a new A/B test
+      # @param name [String] Name of the test
+      # @param champion_version_id [String, Integer] ID of the champion version
+      # @param challenger_version_id [String, Integer] ID of the challenger version
+      # @param traffic_split [Hash] Traffic distribution
+      # @param start_date [Time, nil] When to start the test
+      # @param end_date [Time, nil] When to end the test
+      # @return [ABTest] The created test
+      def create_test(name:, champion_version_id:, challenger_version_id:, traffic_split: { champion: 90, challenger: 10 }, start_date: nil,
+                      end_date: nil)
+        synchronize do
+          # Validate that both versions exist
+          validate_version_exists!(champion_version_id, "champion")
+          validate_version_exists!(challenger_version_id, "challenger")
+
+          test = ABTest.new(
+            name: name,
+            champion_version_id: champion_version_id,
+            challenger_version_id: challenger_version_id,
+            traffic_split: traffic_split,
+            start_date: start_date || Time.now.utc,
+            end_date: end_date,
+            status: start_date && start_date > Time.now.utc ? "scheduled" : "running"
+          )
+
+          saved_test = @storage_adapter.save_test(test)
+          invalidate_cache!
+          saved_test
+        end
+      end
+
+      # Get an A/B test by ID
+      # @param test_id [String, Integer] The test ID
+      # @return [ABTest, nil] The test or nil if not found
+      def get_test(test_id)
+        @storage_adapter.get_test(test_id)
+      end
+
+      # Get all active A/B tests
+      # @return [Array<ABTest>] Array of active tests
+      def active_tests
+        synchronize do
+          return @active_tests_cache[:tests] if cache_valid?
+
+          tests = @storage_adapter.list_tests(status: "running")
+          @active_tests_cache = { tests: tests, timestamp: Time.now.utc }
+          tests
+        end
+      end
+
+      # Assign a variant for a request
+      # @param test_id [String, Integer] The A/B test ID
+      # @param user_id [String, nil] Optional user identifier for consistent assignment
+      # @return [Hash] Assignment details { test_id:, variant:, version_id: }
+      def assign_variant(test_id:, user_id: nil)
+        test = get_test(test_id)
+        raise TestNotFoundError, "Test not found: #{test_id}" unless test
+
+        variant = test.assign_variant(user_id: user_id)
+        version_id = test.version_for_variant(variant)
+
+        assignment = ABTestAssignment.new(
+          ab_test_id: test_id,
+          user_id: user_id,
+          variant: variant,
+          version_id: version_id
+        )
+
+        saved_assignment = @storage_adapter.save_assignment(assignment)
+
+        {
+          test_id: test_id,
+          variant: variant,
+          version_id: version_id,
+          assignment_id: saved_assignment.id
+        }
+      end
+
+      # Record the decision result for an assignment
+      # @param assignment_id [String, Integer] The assignment ID
+      # @param decision [String] The decision result
+      # @param confidence [Float] The confidence score
+      def record_decision(assignment_id:, decision:, confidence:)
+        @storage_adapter.update_assignment(assignment_id, decision_result: decision, confidence: confidence)
+      end
+
+      # Get results comparison for an A/B test
+      # @param test_id [String, Integer] The test ID
+      # @return [Hash] Comparison statistics
+      def get_results(test_id)
+        test = get_test(test_id)
+        raise TestNotFoundError, "Test not found: #{test_id}" unless test
+
+        assignments = @storage_adapter.get_assignments(test_id)
+
+        champion_assignments = assignments.select { |a| a.variant == :champion }
+        challenger_assignments = assignments.select { |a| a.variant == :challenger }
+
+        {
+          test: test.to_h,
+          champion: calculate_variant_stats(champion_assignments, "Champion"),
+          challenger: calculate_variant_stats(challenger_assignments, "Challenger"),
+          comparison: compare_variants(champion_assignments, challenger_assignments),
+          total_assignments: assignments.size,
+          timestamp: Time.now.utc
+        }
+      end
+
+      # Start a scheduled test
+      # @param test_id [String, Integer] The test ID
+      def start_test(test_id)
+        synchronize do
+          test = get_test(test_id)
+          raise TestNotFoundError, "Test not found: #{test_id}" unless test
+
+          test.start!
+          @storage_adapter.update_test(test_id, status: "running", start_date: test.start_date)
+          invalidate_cache!
+        end
+      end
+
+      # Complete a running test
+      # @param test_id [String, Integer] The test ID
+      def complete_test(test_id)
+        synchronize do
+          test = get_test(test_id)
+          raise TestNotFoundError, "Test not found: #{test_id}" unless test
+
+          test.complete!
+          @storage_adapter.update_test(test_id, status: "completed", end_date: test.end_date)
+          invalidate_cache!
+        end
+      end
+
+      # Cancel a test
+      # @param test_id [String, Integer] The test ID
+      def cancel_test(test_id)
+        synchronize do
+          test = get_test(test_id)
+          raise TestNotFoundError, "Test not found: #{test_id}" unless test
+
+          test.cancel!
+          @storage_adapter.update_test(test_id, status: "cancelled")
+          invalidate_cache!
+        end
+      end
+
+      # List all tests with optional filtering
+      # @param status [String, nil] Filter by status
+      # @param limit [Integer, nil] Limit results
+      # @return [Array<ABTest>] Array of tests
+      def list_tests(status: nil, limit: nil)
+        @storage_adapter.list_tests(status: status, limit: limit)
+      end
+
+      private
+
+      def default_storage_adapter
+        # Use in-memory adapter by default
+        require_relative "storage/memory_adapter"
+        Storage::MemoryAdapter.new
+      end
+
+      def validate_version_exists!(version_id, label)
+        version = @version_manager.get_version(version_id: version_id)
+        return if version
+
+        raise VersionNotFoundError, "#{label.capitalize} version not found: #{version_id}"
+      end
+
+      def cache_valid?
+        return false unless @active_tests_cache[:timestamp]
+
+        # Cache is valid for 60 seconds
+        Time.now.utc - @active_tests_cache[:timestamp] < 60
+      end
+
+      def invalidate_cache!
+        @active_tests_cache = {}
+      end
+
+      def calculate_variant_stats(assignments, label)
+        with_decisions = assignments.select(&:decision_result)
+
+        if with_decisions.empty?
+          return {
+            label: label,
+            total_assignments: assignments.size,
+            decisions_recorded: 0,
+            avg_confidence: nil,
+            decision_distribution: {}
+          }
+        end
+
+        confidences = with_decisions.map(&:confidence)
+        decision_counts = with_decisions.group_by(&:decision_result).transform_values(&:size)
+
+        {
+          label: label,
+          total_assignments: assignments.size,
+          decisions_recorded: with_decisions.size,
+          avg_confidence: (confidences.sum / confidences.size.to_f).round(4),
+          min_confidence: confidences.min&.round(4),
+          max_confidence: confidences.max&.round(4),
+          decision_distribution: decision_counts
+        }
+      end
+
+      def compare_variants(champion_assignments, challenger_assignments)
+        champion_with_decisions = champion_assignments.select(&:decision_result)
+        challenger_with_decisions = challenger_assignments.select(&:decision_result)
+
+        return { statistical_significance: "insufficient_data" } if champion_with_decisions.empty? || challenger_with_decisions.empty?
+
+        champion_confidences = champion_with_decisions.map(&:confidence)
+        challenger_confidences = challenger_with_decisions.map(&:confidence)
+
+        champion_avg = champion_confidences.sum / champion_confidences.size.to_f
+        challenger_avg = challenger_confidences.sum / challenger_confidences.size.to_f
+
+        improvement = ((challenger_avg - champion_avg) / champion_avg * 100).round(2)
+
+        # Calculate statistical significance using Welch's t-test approximation
+        sig_result = calculate_statistical_significance(champion_confidences, challenger_confidences)
+
+        {
+          champion_avg_confidence: champion_avg.round(4),
+          challenger_avg_confidence: challenger_avg.round(4),
+          improvement_percentage: improvement,
+          winner: determine_winner(champion_avg, challenger_avg, sig_result[:significant]),
+          statistical_significance: sig_result[:significant] ? "significant" : "not_significant",
+          confidence_level: sig_result[:confidence_level],
+          recommendation: generate_recommendation(improvement, sig_result[:significant])
+        }
+      end
+
+      def calculate_statistical_significance(sample1, sample2)
+        n1 = sample1.size
+        n2 = sample2.size
+
+        return { significant: false, confidence_level: 0 } if n1 < 30 || n2 < 30
+
+        mean1 = sample1.sum / n1.to_f
+        mean2 = sample2.sum / n2.to_f
+
+        var1 = sample1.map { |x| (x - mean1)**2 }.sum / (n1 - 1).to_f
+        var2 = sample2.map { |x| (x - mean2)**2 }.sum / (n2 - 1).to_f
+
+        # Welch's t-statistic
+        t_stat = (mean1 - mean2) / Math.sqrt((var1 / n1) + (var2 / n2))
+
+        # Simplified p-value approximation (for demonstration)
+        # In production, use a proper statistical library
+        t_stat_abs = t_stat.abs
+
+        confidence_level = if t_stat_abs > 2.576
+                             0.99 # 99% confidence
+                           elsif t_stat_abs > 1.96
+                             0.95 # 95% confidence
+                           elsif t_stat_abs > 1.645
+                             0.90 # 90% confidence
+                           else
+                             0.0
+                           end
+
+        {
+          significant: confidence_level >= 0.95,
+          confidence_level: confidence_level,
+          t_statistic: t_stat.round(4)
+        }
+      end
+
+      def determine_winner(champion_avg, challenger_avg, significant)
+        return "inconclusive" unless significant
+
+        challenger_avg > champion_avg ? "challenger" : "champion"
+      end
+
+      def generate_recommendation(improvement, significant)
+        if !significant
+          "Continue testing - not enough data for statistical significance"
+        elsif improvement > 5
+          "Strong evidence to promote challenger"
+        elsif improvement.positive?
+          "Moderate evidence to promote challenger"
+        elsif improvement > -5
+          "Results are similar - consider other factors"
+        else
+          "Keep champion - challenger performs worse"
+        end
+      end
+    end
+
+    # Custom errors
+    class TestNotFoundError < StandardError; end
+    class VersionNotFoundError < StandardError; end
+  end
+end