decision_agent 0.3.0 → 1.1.0

data/lib/decision_agent/simulation/scenario_engine.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module DecisionAgent
+  module Simulation
+    # Engine for managing and executing test scenarios
+    class ScenarioEngine
+      attr_reader :agent, :version_manager
+
+      def initialize(agent:, version_manager: nil)
+        @agent = agent
+        @version_manager = version_manager || Versioning::VersionManager.new
+      end
+
+      # Execute a single scenario
+      # @param scenario [Hash] Scenario definition with context and optional metadata
+      # @param rule_version [String, Integer, Hash, nil] Optional rule version to use
+      # @return [Hash] Scenario execution result
+      def execute(scenario:, rule_version: nil)
+        context = scenario[:context] || scenario["context"] || scenario
+        metadata = scenario[:metadata] || scenario["metadata"] || {}
+
+        analysis_agent = build_agent_from_version(rule_version) if rule_version
+        analysis_agent ||= @agent
+
+        ctx = context.is_a?(Context) ? context : Context.new(context)
+        decision = analysis_agent.decide(context: ctx)
+
+        {
+          scenario_id: scenario[:id] || scenario["id"] || generate_scenario_id,
+          context: ctx.to_h,
+          decision: decision.decision,
+          confidence: decision.confidence,
+          explanations: decision.explanations,
+          metadata: metadata,
+          executed_at: Time.now.utc.iso8601
+        }
+      end
+
+      # Execute multiple scenarios
+      # @param scenarios [Array<Hash>] Array of scenario definitions
+      # @param rule_version [String, Integer, Hash, nil] Optional rule version
+      # @param options [Hash] Execution options
+      #   - :parallel [Boolean] Use parallel execution (default: true)
+      #   - :thread_count [Integer] Number of threads (default: 4)
+      #   - :progress_callback [Proc] Progress callback
+      # @return [Hash] Batch execution results
+      def execute_batch(scenarios:, rule_version: nil, options: {})
+        options = {
+          parallel: true,
+          thread_count: 4,
+          progress_callback: nil
+        }.merge(options)
+
+        analysis_agent = build_agent_from_version(rule_version) if rule_version
+        analysis_agent ||= @agent
+
+        results = execute_scenarios_with_progress(
+          scenarios, analysis_agent, rule_version, options
+        )
+
+        build_batch_report(results)
+      end
+
+      def execute_scenarios_with_progress(scenarios, analysis_agent, rule_version, options)
+        results = []
+        mutex = Mutex.new
+        progress_tracker = ProgressTracker.new(scenarios.size, options[:progress_callback])
+
+        if options[:parallel] && scenarios.size > 1
+          execute_parallel(scenarios, analysis_agent, options, mutex) do |result|
+            mutex.synchronize do
+              results << result
+              progress_tracker.increment
+            end
+          end
+        else
+          execute_sequential(scenarios, rule_version, results, progress_tracker)
+        end
+
+        results
+      end
+
+      def execute_sequential(scenarios, rule_version, results, progress_tracker)
+        scenarios.each_with_index do |scenario, _index|
+          result = execute(scenario: scenario, rule_version: rule_version)
+          results << result
+          progress_tracker.increment
+        end
+      end
+
+      # Helper class for tracking progress
+      class ProgressTracker
+        def initialize(total, callback)
+          @total = total
+          @callback = callback
+          @completed = 0
+        end
+
+        def increment
+          @completed += 1
+          return unless @callback
+
+          @callback.call(
+            completed: @completed,
+            total: @total,
+            percentage: (@completed.to_f / @total * 100).round(2)
+          )
+        end
+      end
+
+      # Compare scenarios across different rule versions
+      # @param scenarios [Array<Hash>] Scenarios to test
+      # @param versions [Array<String, Integer, Hash>] Rule versions to compare
+      # @param options [Hash] Execution options
+      # @return [Hash] Comparison results
+      def compare_versions(scenarios:, versions:, options: {})
+        options = {
+          parallel: true,
+          thread_count: 4
+        }.merge(options)
+
+        version_results = {}
+        versions.each do |version|
+          results = execute_batch(scenarios: scenarios, rule_version: version, options: options)
+          version_id = version.is_a?(Hash) ? (version[:id] || version["id"]) : version
+          version_results[version_id.to_s] = results
+        end
+
+        {
+          scenarios: scenarios,
+          versions: versions.map { |v| v.is_a?(Hash) ? (v[:id] || v["id"]) : v },
+          results_by_version: version_results,
+          comparison: build_version_comparison(version_results)
+        }
+      end
+
+      private
+
+      def build_agent_from_version(version)
+        version_hash = resolve_version(version)
+        evaluators = build_evaluators_from_version(version_hash)
+        Agent.new(
+          evaluators: evaluators,
+          scoring_strategy: @agent.scoring_strategy,
+          audit_adapter: Audit::NullAdapter.new
+        )
+      end
+
+      def resolve_version(version)
+        case version
+        when String, Integer
+          version_data = @version_manager.get_version(version_id: version)
+          raise VersionComparisonError, "Version not found: #{version}" unless version_data
+
+          version_data
+        when Hash
+          version
+        else
+          raise VersionComparisonError, "Invalid version format: #{version.class}"
+        end
+      end
+
+      def build_evaluators_from_version(version)
+        content = version[:content] || version["content"]
+        return @agent.evaluators unless content
+
+        if content.is_a?(Hash) && content[:evaluators]
+          build_evaluators_from_config(content[:evaluators])
+        elsif content.is_a?(Hash) && (content[:rules] || content["rules"])
+          [Evaluators::JsonRuleEvaluator.new(rules_json: content)]
+        else
+          @agent.evaluators
+        end
+      end
+
+      def build_evaluators_from_config(configs)
+        Array(configs).map do |config|
+          case config[:type] || config["type"]
+          when "json_rule"
+            Evaluators::JsonRuleEvaluator.new(rules_json: config[:rules] || config["rules"])
+          when "dmn"
+            model = config[:model] || config["model"]
+            decision_id = config[:decision_id] || config["decision_id"]
+            Evaluators::DmnEvaluator.new(model: model, decision_id: decision_id)
+          else
+            raise VersionComparisonError, "Unknown evaluator type: #{config[:type]}"
+          end
+        end
+      end
+
+      def execute_parallel(scenarios, analysis_agent, options, _mutex, &block)
+        thread_count = [options[:thread_count], scenarios.size].min
+        queue = Queue.new
+        scenarios.each { |s| queue << s }
+
+        threads = Array.new(thread_count) do
+          Thread.new do
+            process_scenarios_from_queue(queue, analysis_agent, &block)
+          end
+        end
+
+        threads.each(&:join)
+      end
+
+      def process_scenarios_from_queue(queue, analysis_agent)
+        loop do
+          scenario = dequeue_scenario(queue)
+          break unless scenario
+
+          result = process_scenario(scenario, analysis_agent)
+          yield result
+        end
+      end
+
+      def dequeue_scenario(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def process_scenario(scenario, analysis_agent)
+        context = extract_scenario_context(scenario)
+        metadata = extract_scenario_metadata(scenario)
+        ctx = context.is_a?(Context) ? context : Context.new(context)
+
+        begin
+          decision = analysis_agent.decide(context: ctx)
+          build_scenario_result(scenario, ctx, decision, metadata)
+        rescue NoEvaluationsError
+          build_error_result(scenario, ctx, metadata)
+        end
+      end
+
+      def extract_scenario_context(scenario)
+        scenario[:context] || scenario["context"] || scenario
+      end
+
+      def extract_scenario_metadata(scenario)
+        scenario[:metadata] || scenario["metadata"] || {}
+      end
+
+      def build_scenario_result(scenario, context, decision, metadata)
+        {
+          scenario_id: extract_scenario_id(scenario),
+          context: context.to_h,
+          decision: decision.decision,
+          confidence: decision.confidence,
+          explanations: decision.explanations,
+          metadata: metadata,
+          executed_at: Time.now.utc.iso8601
+        }
+      end
+
+      def build_error_result(scenario, context, metadata)
+        {
+          scenario_id: extract_scenario_id(scenario),
+          context: context.to_h,
+          decision: nil,
+          confidence: 0.0,
+          explanations: [],
+          metadata: metadata,
+          executed_at: Time.now.utc.iso8601,
+          error: "No evaluators returned a decision"
+        }
+      end
+
+      def extract_scenario_id(scenario)
+        scenario[:id] || scenario["id"] || generate_scenario_id
+      end
+
+      def build_batch_report(results)
+        {
+          total_scenarios: results.size,
+          decision_distribution: results.group_by { |r| r[:decision] }.transform_values(&:count),
+          average_confidence: calculate_average_confidence(results),
+          min_confidence: results.map { |r| r[:confidence] }.min || 0,
+          max_confidence: results.map { |r| r[:confidence] }.max || 0,
+          results: results
+        }
+      end
+
+      def build_version_comparison(version_results)
+        comparison = {}
+        version_ids = version_results.keys
+
+        # Compare decision distributions
+        decision_comparison = {}
+        version_ids.each do |version_id|
+          results = version_results[version_id][:results] || []
+          decision_comparison[version_id] = results.group_by { |r| r[:decision] }.transform_values(&:count)
+        end
+
+        comparison[:decision_distributions] = decision_comparison
+
+        # Compare average confidence
+        confidence_comparison = {}
+        version_ids.each do |version_id|
+          results = version_results[version_id][:results] || []
+          confidences = results.map { |r| r[:confidence] }.compact
+          confidence_comparison[version_id] = confidences.any? ? confidences.sum / confidences.size : 0
+        end
+
+        comparison[:average_confidence] = confidence_comparison
+
+        comparison
+      end
+
+      def calculate_average_confidence(results)
+        confidences = results.map { |r| r[:confidence] }.compact
+        confidences.any? ? confidences.sum / confidences.size : 0
+      end
+
+      def generate_scenario_id
+        "scenario_#{Time.now.to_f}_#{rand(1000)}"
+      end
+    end
+  end
+end

data/lib/decision_agent/simulation/scenario_library.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module DecisionAgent
+  module Simulation
+    # Library of pre-defined test scenario templates
+    class ScenarioLibrary
+      # Get scenario template by name
+      # @param template_name [String, Symbol] Template name
+      # @return [Hash, nil] Scenario template or nil if not found
+      def self.get_template(template_name)
+        templates[template_name.to_sym] || templates[template_name.to_s]
+      end
+
+      # List all available templates
+      # @return [Array<String>] Array of template names
+      def self.list_templates
+        templates.keys.map(&:to_s)
+      end
+
+      # Create scenario from template
+      # @param template_name [String, Symbol] Template name
+      # @param overrides [Hash] Values to override in template
+      # @return [Hash] Scenario definition
+      def self.create_scenario(template_name, overrides: {})
+        template = get_template(template_name)
+        raise DecisionAgent::Simulation::ScenarioExecutionError, "Template not found: #{template_name}" unless template
+
+        scenario = template.dup
+        merge_overrides(scenario, overrides)
+        scenario
+      end
+
+      # Get edge case scenarios for a given context structure
+      # @param base_context [Hash] Base context to generate edge cases from
+      # @return [Array<Hash>] Array of edge case scenarios
+      def self.generate_edge_cases(base_context)
+        scenarios = []
+
+        # Generate scenarios with nil values
+        base_context.each_key do |key|
+          edge_scenario = base_context.dup
+          edge_scenario[key] = nil
+          scenarios << { context: edge_scenario, metadata: { type: "edge_case", field: key, value: "nil" } }
+        end
+
+        # Generate scenarios with extreme numeric values and empty strings
+        base_context.each do |key, value|
+          if value.is_a?(Numeric)
+            # Zero value
+            zero_scenario = base_context.dup
+            zero_scenario[key] = 0
+            scenarios << { context: zero_scenario, metadata: { type: "edge_case", field: key, value: "zero" } }
+
+            # Negative value (if positive)
+            if value.positive?
+              neg_scenario = base_context.dup
+              neg_scenario[key] = -value
+              scenarios << { context: neg_scenario, metadata: { type: "edge_case", field: key, value: "negative" } }
+            end
+
+            # Very large value
+            large_scenario = base_context.dup
+            large_scenario[key] = value * 1000
+            scenarios << { context: large_scenario, metadata: { type: "edge_case", field: key, value: "large" } }
+          elsif value.is_a?(String)
+            # Empty string
+            empty_scenario = base_context.dup
+            empty_scenario[key] = ""
+            scenarios << { context: empty_scenario, metadata: { type: "edge_case", field: key, value: "empty_string" } }
+          end
+        end
+
+        scenarios
+      end
+
+      def self.templates
+        loan_approval_templates
+          .merge(fraud_detection_templates)
+          .merge(pricing_templates)
+      end
+
+      def self.loan_approval_templates
+        {
+          loan_approval_high_risk: build_loan_scenario(100_000, 550, 30_000, "unemployed", "high_risk"),
+          loan_approval_low_risk: build_loan_scenario(50_000, 800, 100_000, "employed", "low_risk"),
+          loan_approval_medium_risk: build_loan_scenario(75_000, 650, 60_000, "employed", "medium_risk")
+        }
+      end
+
+      def self.build_loan_scenario(amount, credit_score, income, employment_status, risk_level)
+        {
+          context: {
+            amount: amount,
+            credit_score: credit_score,
+            income: income,
+            employment_status: employment_status
+          },
+          metadata: {
+            type: "loan_approval",
+            category: risk_level,
+            description: "#{risk_level.capitalize.tr('_', ' ')} loan application scenario"
+          }
+        }
+      end
+
+      def self.fraud_detection_templates
+        {
+          fraud_detection_suspicious: build_fraud_scenario(10_000, 5, 50, "unusual", "suspicious"),
+          fraud_detection_normal: build_fraud_scenario(100, 365, 3, "usual", "normal")
+        }
+      end
+
+      def self.build_fraud_scenario(amount, account_age, transaction_count, location, category)
+        {
+          context: {
+            transaction_amount: amount,
+            account_age_days: account_age,
+            transaction_count_24h: transaction_count,
+            location: location
+          },
+          metadata: {
+            type: "fraud_detection",
+            category: category,
+            description: "#{category.capitalize} transaction scenario"
+          }
+        }
+      end
+
+      def self.pricing_templates
+        {
+          pricing_high_value: build_pricing_scenario("premium", 5_000, 10_000, "high", "high_value"),
+          pricing_standard: build_pricing_scenario("standard", 100, 500, "medium", "standard")
+        }
+      end
+
+      def self.build_pricing_scenario(tier, order_value, loyalty_points, frequency, category)
+        {
+          context: {
+            customer_tier: tier,
+            order_value: order_value,
+            loyalty_points: loyalty_points,
+            purchase_frequency: frequency
+          },
+          metadata: {
+            type: "pricing",
+            category: category,
+            description: "#{category.capitalize.tr('_', ' ')} customer pricing scenario"
+          }
+        }
+      end
+
+      def self.merge_overrides(scenario, overrides)
+        scenario[:context] = scenario[:context].merge(overrides[:context]) if overrides[:context]
+
+        scenario[:metadata] = (scenario[:metadata] || {}).merge(overrides[:metadata]) if overrides[:metadata]
+
+        overrides.each do |key, value|
+          next if %i[context metadata].include?(key)
+
+          scenario[key] = value
+        end
+      end
+    end
+  end
+end