decision_agent 0.1.3 → 0.1.6

data/lib/decision_agent/testing/batch_test_runner.rb

@@ -0,0 +1,244 @@
+require "json"
+
+module DecisionAgent
+  module Testing
+    # Result of a single test scenario execution
+    class TestResult
+      attr_reader :scenario_id, :decision, :confidence, :execution_time_ms, :error, :evaluations
+
+      def initialize(scenario_id:, decision: nil, confidence: nil, execution_time_ms: 0, error: nil, evaluations: [])
+        @scenario_id = scenario_id.to_s.freeze
+        @decision = decision&.to_s&.freeze
+        @confidence = confidence&.to_f
+        @execution_time_ms = execution_time_ms.to_f
+        @error = error
+        @evaluations = evaluations.freeze
+
+        freeze
+      end
+
+      def success?
+        @error.nil?
+      end
+
+      def to_h
+        {
+          scenario_id: @scenario_id,
+          decision: @decision,
+          confidence: @confidence,
+          execution_time_ms: @execution_time_ms,
+          error: @error&.message,
+          success: success?,
+          evaluations: @evaluations.map { |e| e.respond_to?(:to_h) ? e.to_h : e }
+        }
+      end
+    end
+
+    # Executes batch tests against an agent
+    class BatchTestRunner
+      attr_reader :agent, :results
+
+      def initialize(agent)
+        @agent = agent
+        @results = []
+        @checkpoint_file = nil
+      end
+
+      # Run batch tests against scenarios
+      # @param scenarios [Array<TestScenario>] Test scenarios to execute
+      # @param options [Hash] Execution options
+      #   - :parallel [Boolean] Use parallel execution (default: true)
+      #   - :thread_count [Integer] Number of threads for parallel execution (default: 4)
+      #   - :progress_callback [Proc] Callback for progress updates (called with { completed: N, total: M, percentage: X })
+      #   - :feedback [Hash] Optional feedback to pass to agent
+      #   - :checkpoint_file [String] Path to checkpoint file for resume capability (optional)
+      # @return [Array<TestResult>] Array of test results
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      def run(scenarios, options = {})
+        @results = []
+        @checkpoint_file = options[:checkpoint_file]
+        options = {
+          parallel: true,
+          thread_count: 4,
+          progress_callback: nil,
+          feedback: {},
+          checkpoint_file: nil
+        }.merge(options)
+
+        total = scenarios.size
+        completed = 0
+        mutex = Mutex.new
+
+        # Load checkpoint if exists
+        completed_scenario_ids = load_checkpoint if @checkpoint_file && File.exist?(@checkpoint_file)
+
+        # Filter out already completed scenarios
+        remaining_scenarios = if completed_scenario_ids&.any?
+                                scenarios.reject { |s| completed_scenario_ids.include?(s.id) }
+                              else
+                                scenarios
+                              end
+
+        if options[:parallel] && remaining_scenarios.size > 1
+          run_parallel(remaining_scenarios, options, mutex) do |result|
+            completed += 1
+            save_checkpoint(result.scenario_id) if @checkpoint_file
+            options[:progress_callback]&.call(
+              completed: completed + (completed_scenario_ids&.size || 0),
+              total: total,
+              percentage: ((completed + (completed_scenario_ids&.size || 0)).to_f / total * 100).round(2)
+            )
+          end
+        else
+          remaining_scenarios.each_with_index do |scenario, index|
+            result = execute_scenario(scenario, options[:feedback])
+            @results << result
+            save_checkpoint(result.scenario_id) if @checkpoint_file
+            completed = index + 1
+            options[:progress_callback]&.call(
+              completed: completed + (completed_scenario_ids&.size || 0),
+              total: total,
+              percentage: ((completed + (completed_scenario_ids&.size || 0)).to_f / total * 100).round(2)
+            )
+          end
+        end
+
+        # Clean up checkpoint file on successful completion
+        delete_checkpoint if @checkpoint_file && File.exist?(@checkpoint_file)
+
+        @results
+      end
+
+      # Resume batch test execution from a checkpoint
+      # @param scenarios [Array<TestScenario>] All test scenarios (including already completed ones)
+      # @param checkpoint_file [String] Path to checkpoint file
+      # @param options [Hash] Same as run method
+      # @return [Array<TestResult>] Array of test results (only newly executed ones)
+      def resume(scenarios, checkpoint_file, options = {})
+        options[:checkpoint_file] = checkpoint_file
+        run(scenarios, options)
+      end
+
+      # Get execution statistics
+      # @return [Hash] Statistics about the batch test run
+      def statistics
+        return {} if @results.empty?
+
+        successful = @results.count(&:success?)
+        failed = @results.size - successful
+        execution_times = @results.map(&:execution_time_ms).compact
+
+        {
+          total: @results.size,
+          successful: successful,
+          failed: failed,
+          success_rate: successful.to_f / @results.size,
+          avg_execution_time_ms: execution_times.any? ? execution_times.sum / execution_times.size : 0,
+          min_execution_time_ms: execution_times.min || 0,
+          max_execution_time_ms: execution_times.max || 0,
+          total_execution_time_ms: execution_times.sum
+        }
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      private
+
+      def run_parallel(scenarios, options, mutex)
+        thread_count = [options[:thread_count], scenarios.size].min
+        queue = Queue.new
+        scenarios.each { |s| queue << s }
+
+        threads = Array.new(thread_count) do
+          Thread.new do
+            loop do
+              scenario = begin
+                queue.pop(true)
+              rescue StandardError
+                nil
+              end
+              break unless scenario
+
+              result = execute_scenario(scenario, options[:feedback])
+              mutex.synchronize do
+                @results << result
+                yield result
+              end
+            end
+          end
+        end
+
+        threads.each(&:join)
+      end
+
+      def execute_scenario(scenario, feedback)
+        start_time = Time.now
+
+        begin
+          decision = @agent.decide(context: scenario.context, feedback: feedback)
+
+          execution_time_ms = ((Time.now - start_time) * 1000).round(2)
+
+          TestResult.new(
+            scenario_id: scenario.id,
+            decision: decision.decision,
+            confidence: decision.confidence,
+            execution_time_ms: execution_time_ms,
+            evaluations: decision.evaluations
+          )
+        rescue StandardError => e
+          execution_time_ms = ((Time.now - start_time) * 1000).round(2)
+
+          TestResult.new(
+            scenario_id: scenario.id,
+            execution_time_ms: execution_time_ms,
+            error: e
+          )
+        end
+      end
+
+      def save_checkpoint(scenario_id)
+        return unless @checkpoint_file
+
+        checkpoint_data = load_checkpoint_data
+        checkpoint_data[:completed_scenario_ids] << scenario_id.to_s unless checkpoint_data[:completed_scenario_ids].include?(scenario_id.to_s)
+        checkpoint_data[:last_updated] = Time.now.to_i
+
+        File.write(@checkpoint_file, JSON.pretty_generate(checkpoint_data))
+      rescue StandardError => e
+        # Silently fail checkpoint saving to not interrupt test execution
+        warn "Failed to save checkpoint: #{e.message}" if $VERBOSE
+      end
+
+      def load_checkpoint
+        return [] unless @checkpoint_file && File.exist?(@checkpoint_file)
+
+        checkpoint_data = load_checkpoint_data
+        checkpoint_data[:completed_scenario_ids] || []
+      rescue StandardError => e
+        warn "Failed to load checkpoint: #{e.message}" if $VERBOSE
+        []
+      end
+
+      def load_checkpoint_data
+        return { completed_scenario_ids: [], last_updated: nil } unless @checkpoint_file && File.exist?(@checkpoint_file)
+
+        content = File.read(@checkpoint_file)
+        data = JSON.parse(content, symbolize_names: true)
+        data[:completed_scenario_ids] ||= []
+        data
+      rescue JSON::ParserError
+        { completed_scenario_ids: [], last_updated: nil }
+      rescue StandardError
+        { completed_scenario_ids: [], last_updated: nil }
+      end
+
+      def delete_checkpoint
+        return unless @checkpoint_file && File.exist?(@checkpoint_file)
+
+        File.delete(@checkpoint_file)
+      rescue StandardError => e
+        warn "Failed to delete checkpoint: #{e.message}" if $VERBOSE
+      end
+    end
+  end
+end

data/lib/decision_agent/testing/test_coverage_analyzer.rb

@@ -0,0 +1,191 @@
+require "set"
+
+module DecisionAgent
+  module Testing
+    # Coverage report for test scenarios
+    class CoverageReport
+      attr_reader :total_rules, :covered_rules, :untested_rules, :coverage_percentage, :rule_coverage, :condition_coverage
+
+      def initialize(total_rules:, covered_rules:, untested_rules:, coverage_percentage:, rule_coverage:, condition_coverage:)
+        @total_rules = total_rules
+        @covered_rules = covered_rules
+        @untested_rules = untested_rules.freeze
+        @coverage_percentage = coverage_percentage
+        @rule_coverage = rule_coverage.freeze
+        @condition_coverage = condition_coverage.freeze
+
+        freeze
+      end
+
+      def to_h
+        {
+          total_rules: @total_rules,
+          covered_rules: @covered_rules,
+          untested_rules: @untested_rules,
+          coverage_percentage: @coverage_percentage,
+          rule_coverage: @rule_coverage,
+          condition_coverage: @condition_coverage
+        }
+      end
+    end
+
+    # Analyzes test coverage of rules and conditions
+    class TestCoverageAnalyzer
+      def initialize
+        @executed_rules = Set.new
+        @executed_conditions = Set.new
+        @rule_evaluation_count = {}
+        @condition_evaluation_count = {}
+      end
+
+      # Analyze coverage from test results
+      # @param results [Array<TestResult>] Test results from batch execution
+      # @param agent [Agent] The agent used for testing (to get all available rules)
+      # @return [CoverageReport] Coverage report
+      def analyze(results, agent = nil)
+        reset
+
+        # Track which rules and conditions were executed
+        results.each do |result|
+          next unless result.success?
+
+          result.evaluations.each do |evaluation|
+            track_evaluation(evaluation)
+          end
+        end
+
+        # Get all available rules from agent if provided
+        all_rules = agent ? extract_rules_from_agent(agent) : []
+        all_conditions = agent ? extract_conditions_from_agent(agent) : []
+
+        generate_report(all_rules, all_conditions)
+      end
+
+      # Get coverage percentage
+      # @return [Float] Coverage percentage (0.0 to 1.0)
+      def coverage_percentage
+        return 0.0 if @executed_rules.empty?
+
+        total = @rule_evaluation_count.size
+        return 0.0 if total.zero?
+
+        @executed_rules.size.to_f / total
+      end
+
+      private
+
+      def reset
+        @executed_rules = Set.new
+        @executed_conditions = Set.new
+        @rule_evaluation_count = {}
+        @condition_evaluation_count = {}
+      end
+
+      def track_evaluation(evaluation)
+        # Extract rule identifier from evaluation
+        rule_id = extract_rule_id(evaluation)
+        condition_id = extract_condition_id(evaluation)
+
+        if rule_id
+          @executed_rules << rule_id
+          @rule_evaluation_count[rule_id] = (@rule_evaluation_count[rule_id] || 0) + 1
+        end
+
+        return unless condition_id
+
+        @executed_conditions << condition_id
+        @condition_evaluation_count[condition_id] = (@condition_evaluation_count[condition_id] || 0) + 1
+      end
+
+      def extract_rule_id(evaluation)
+        # Try to get rule_id from metadata
+        return evaluation.metadata[:rule_id] if evaluation.respond_to?(:metadata) && evaluation.metadata.is_a?(Hash)
+
+        # Fallback to evaluator_name as rule identifier
+        return evaluation.evaluator_name if evaluation.respond_to?(:evaluator_name)
+
+        nil
+      end
+
+      def extract_condition_id(evaluation)
+        # Try to get condition_id from metadata
+        return evaluation.metadata[:condition_id] if evaluation.respond_to?(:metadata) && evaluation.metadata.is_a?(Hash)
+
+        nil
+      end
+
+      def extract_rules_from_agent(agent)
+        rules = []
+
+        agent.evaluators.each do |evaluator|
+          # Try to extract rule information from evaluator
+          if evaluator.respond_to?(:rules)
+            rules.concat(Array(evaluator.rules))
+          elsif evaluator.respond_to?(:rule_id)
+            rules << evaluator.rule_id
+          else
+            # Use evaluator class name as rule identifier
+            rules << evaluator.class.name
+          end
+        end
+
+        rules.uniq
+      end
+
+      def extract_conditions_from_agent(agent)
+        conditions = []
+
+        agent.evaluators.each do |evaluator|
+          # Try to extract condition information from evaluator
+          if evaluator.respond_to?(:conditions)
+            conditions.concat(Array(evaluator.conditions))
+          elsif evaluator.respond_to?(:condition_id)
+            conditions << evaluator.condition_id
+          end
+        end
+
+        conditions.uniq
+      end
+
+      def generate_report(all_rules, all_conditions)
+        total_rules = all_rules.any? ? all_rules.size : @executed_rules.size
+        covered_rules = @executed_rules.size
+        untested_rules = all_rules.any? ? (all_rules - @executed_rules.to_a) : []
+
+        # Cap coverage at 1.0 (100%)
+        coverage_percentage = if total_rules.positive?
+                                [(covered_rules.to_f / total_rules), 1.0].min
+                              else
+                                0.0
+                              end
+
+        # Build rule coverage details
+        rule_coverage = all_rules.map do |rule|
+          {
+            rule_id: rule,
+            covered: @executed_rules.include?(rule),
+            execution_count: @rule_evaluation_count[rule] || 0
+          }
+        end
+
+        # Build condition coverage details
+        condition_coverage = all_conditions.map do |condition|
+          {
+            condition_id: condition,
+            covered: @executed_conditions.include?(condition),
+            execution_count: @condition_evaluation_count[condition] || 0
+          }
+        end
+
+        CoverageReport.new(
+          total_rules: total_rules,
+          covered_rules: covered_rules,
+          untested_rules: untested_rules,
+          coverage_percentage: coverage_percentage,
+          rule_coverage: rule_coverage,
+          condition_coverage: condition_coverage
+        )
+      end
+    end
+  end
+end

data/lib/decision_agent/testing/test_result_comparator.rb

@@ -0,0 +1,235 @@
+module DecisionAgent
+  module Testing
+    # Comparison result for a single test scenario
+    class ComparisonResult
+      attr_reader :scenario_id, :match, :decision_match, :confidence_match, :differences, :actual, :expected
+
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(scenario_id:, match:, decision_match:, confidence_match:, differences:, actual:, expected:)
+        @scenario_id = scenario_id.to_s.freeze
+        @match = match
+        @decision_match = decision_match
+        @confidence_match = confidence_match
+        @differences = differences.freeze
+        @actual = actual
+        @expected = expected
+
+        freeze
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def to_h
+        {
+          scenario_id: @scenario_id,
+          match: @match,
+          decision_match: @decision_match,
+          confidence_match: @confidence_match,
+          differences: @differences,
+          actual: {
+            decision: @actual[:decision],
+            confidence: @actual[:confidence]
+          },
+          expected: {
+            decision: @expected[:decision],
+            confidence: @expected[:confidence]
+          }
+        }
+      end
+    end
+
+    # Compares test results with expected outcomes
+    class TestResultComparator
+      attr_reader :comparison_results
+
+      def initialize(options = {})
+        @options = {
+          confidence_tolerance: 0.01, # 1% tolerance for confidence comparison
+          fuzzy_match: false # Whether to do fuzzy matching on decisions
+        }.merge(options)
+        @comparison_results = []
+      end
+
+      # Compare test results with expected results from scenarios
+      # @param results [Array<TestResult>] Actual test results
+      # @param scenarios [Array<TestScenario>] Test scenarios with expected results
+      # @return [Hash] Comparison summary with accuracy metrics
+      def compare(results, scenarios)
+        @comparison_results = []
+
+        # Create a map of scenario_id -> scenario for quick lookup
+        scenarios.each_with_object({}) do |scenario, map|
+          map[scenario.id] = scenario
+        end
+
+        # Create a map of scenario_id -> result for quick lookup
+        result_map = results.each_with_object({}) do |result, map|
+          map[result.scenario_id] = result
+        end
+
+        # Compare each scenario with its result
+        scenarios.each do |scenario|
+          next unless scenario.expected_result?
+
+          result = result_map[scenario.id]
+          # Only compare if we have a result (skip if result is missing)
+          next unless result
+
+          comparison = compare_single(scenario, result)
+          @comparison_results << comparison
+        end
+
+        generate_summary
+      end
+
+      # Generate a summary report
+      # @return [Hash] Summary with accuracy metrics and mismatches
+      def generate_summary
+        return empty_summary if @comparison_results.empty?
+
+        total = @comparison_results.size
+        matches = @comparison_results.count(&:match)
+        mismatches = total - matches
+
+        {
+          total: total,
+          matches: matches,
+          mismatches: mismatches,
+          accuracy_rate: matches.to_f / total,
+          decision_accuracy: @comparison_results.count(&:decision_match).to_f / total,
+          confidence_accuracy: @comparison_results.count(&:confidence_match).to_f / total,
+          mismatches_detail: @comparison_results.reject(&:match).map(&:to_h)
+        }
+      end
+
+      # Export comparison results to CSV
+      # @param file_path [String] Path to output CSV file
+      def export_csv(file_path)
+        require "csv"
+
+        CSV.open(file_path, "w") do |csv|
+          csv << %w[scenario_id match decision_match confidence_match expected_decision actual_decision expected_confidence
+                    actual_confidence differences]
+          @comparison_results.each do |result|
+            csv << [
+              result.scenario_id,
+              result.match,
+              result.decision_match,
+              result.confidence_match,
+              result.expected[:decision],
+              result.actual[:decision],
+              result.expected[:confidence],
+              result.actual[:confidence],
+              result.differences.join("; ")
+            ]
+          end
+        end
+      end
+
+      # Export comparison results to JSON
+      # @param file_path [String] Path to output JSON file
+      def export_json(file_path)
+        require "json"
+
+        File.write(file_path, JSON.pretty_generate({
+                                                     summary: generate_summary,
+                                                     results: @comparison_results.map(&:to_h)
+                                                   }))
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
+      def compare_single(scenario, result)
+        differences = []
+        confidence_match = false
+
+        if result.nil? || !result.success?
+          differences << "Test execution failed: #{result&.error&.message || 'No result'}"
+          return ComparisonResult.new(
+            scenario_id: scenario.id,
+            match: false,
+            decision_match: false,
+            confidence_match: false,
+            differences: differences,
+            actual: { decision: nil, confidence: nil },
+            expected: {
+              decision: scenario.expected_decision,
+              confidence: scenario.expected_confidence
+            }
+          )
+        end
+
+        # Compare decision
+        expected_decision = scenario.expected_decision&.to_s
+        actual_decision = result.decision&.to_s
+
+        decision_match = if expected_decision.nil?
+                           true # No expectation, so it matches
+                         elsif @options[:fuzzy_match]
+                           fuzzy_decision_match?(expected_decision, actual_decision)
+                         else
+                           expected_decision == actual_decision
+                         end
+
+        differences << "Decision mismatch: expected '#{expected_decision}', got '#{actual_decision}'" unless decision_match
+
+        # Compare confidence
+        expected_confidence = scenario.expected_confidence
+        actual_confidence = result.confidence
+
+        if expected_confidence.nil?
+          confidence_match = true # No expectation, so it matches
+        elsif actual_confidence.nil?
+          confidence_match = false
+          differences << "Confidence missing in actual result"
+        else
+          tolerance = @options[:confidence_tolerance]
+          confidence_match = (expected_confidence - actual_confidence).abs <= tolerance
+          unless confidence_match
+            diff = (expected_confidence - actual_confidence).abs.round(4)
+            differences << "Confidence mismatch: expected #{expected_confidence}, got #{actual_confidence} (diff: #{diff})"
+          end
+        end
+
+        match = decision_match && confidence_match
+
+        ComparisonResult.new(
+          scenario_id: scenario.id,
+          match: match,
+          decision_match: decision_match,
+          confidence_match: confidence_match,
+          differences: differences,
+          actual: {
+            decision: actual_decision,
+            confidence: actual_confidence
+          },
+          expected: {
+            decision: expected_decision,
+            confidence: expected_confidence
+          }
+        )
+      end
+      # rubocop:enable Metrics/MethodLength, Metrics/PerceivedComplexity
+
+      def fuzzy_decision_match?(expected, actual)
+        return true if expected == actual
+        return true if expected&.downcase == actual&.downcase
+        return true if expected&.strip == actual&.strip
+
+        false
+      end
+
+      def empty_summary
+        {
+          total: 0,
+          matches: 0,
+          mismatches: 0,
+          accuracy_rate: 0.0,
+          decision_accuracy: 0.0,
+          confidence_accuracy: 0.0,
+          mismatches_detail: []
+        }
+      end
+    end
+  end
+end