decision_agent 0.1.3 → 0.1.6

data/spec/testing/batch_test_runner_spec.rb

@@ -0,0 +1,307 @@
+require "spec_helper"
+require "tempfile"
+
+RSpec.describe DecisionAgent::Testing::BatchTestRunner do
+  let(:evaluator) { DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0) }
+  let(:agent) { DecisionAgent::Agent.new(evaluators: [evaluator]) }
+  let(:runner) { DecisionAgent::Testing::BatchTestRunner.new(agent) }
+
+  describe "#initialize" do
+    it "creates a batch test runner with an agent" do
+      expect(runner.agent).to eq(agent)
+      expect(runner.results).to eq([])
+    end
+  end
+
+  describe "#run" do
+    let(:scenarios) do
+      [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 }
+        ),
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_2",
+          context: { user_id: 456 }
+        )
+      ]
+    end
+
+    it "executes test scenarios sequentially" do
+      results = runner.run(scenarios, parallel: false)
+
+      expect(results.size).to eq(2)
+      expect(results.all?(&:success?)).to be true
+      expect(results[0].decision).to eq("approve")
+      expect(results[1].decision).to eq("approve")
+    end
+
+    it "executes test scenarios in parallel when enabled" do
+      results = runner.run(scenarios, parallel: true, thread_count: 2)
+
+      expect(results.size).to eq(2)
+      expect(results.all?(&:success?)).to be true
+    end
+
+    it "executes single scenario sequentially even with parallel enabled" do
+      single_scenario = [scenarios[0]]
+      results = runner.run(single_scenario, parallel: true)
+
+      expect(results.size).to eq(1)
+      expect(results.all?(&:success?)).to be true
+    end
+
+    it "handles empty scenarios array" do
+      results = runner.run([])
+      expect(results).to eq([])
+    end
+
+    it "calls progress callback during execution" do
+      progress_updates = []
+
+      runner.run(scenarios, progress_callback: lambda { |progress|
+        progress_updates << progress
+      })
+
+      expect(progress_updates.size).to be >= 2
+      expect(progress_updates.last[:completed]).to eq(2)
+      expect(progress_updates.last[:total]).to eq(2)
+      expect(progress_updates.last[:percentage]).to eq(100.0)
+    end
+
+    it "tracks execution time for each scenario" do
+      results = runner.run(scenarios)
+
+      results.each do |result|
+        expect(result.execution_time_ms).to be >= 0
+        expect(result.execution_time_ms).to be_a(Numeric)
+      end
+    end
+
+    it "handles errors gracefully" do
+      # Create an agent that will raise an error
+      error_evaluator = Class.new do
+        def evaluate(_context, _feedback: {})
+          raise StandardError, "Test error"
+        end
+      end.new
+
+      error_agent = DecisionAgent::Agent.new(evaluators: [error_evaluator])
+      error_runner = DecisionAgent::Testing::BatchTestRunner.new(error_agent)
+
+      results = error_runner.run(scenarios)
+
+      expect(results.size).to eq(2)
+      expect(results.none?(&:success?)).to be true
+      expect(results[0].error).to be_a(StandardError)
+    end
+
+    it "passes feedback to agent" do
+      feedback = { source: "batch_test" }
+      results = runner.run(scenarios, feedback: feedback)
+
+      expect(results.size).to eq(2)
+      expect(results.all?(&:success?)).to be true
+    end
+  end
+
+  describe "#statistics" do
+    it "returns empty hash when no results" do
+      expect(runner.statistics).to eq({})
+    end
+
+    it "calculates statistics from results" do
+      scenarios = [
+        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
+        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
+      ]
+
+      runner.run(scenarios)
+      stats = runner.statistics
+
+      expect(stats[:total]).to eq(2)
+      expect(stats[:successful]).to eq(2)
+      expect(stats[:failed]).to eq(0)
+      expect(stats[:success_rate]).to eq(1.0)
+      expect(stats[:avg_execution_time_ms]).to be >= 0
+      expect(stats[:min_execution_time_ms]).to be >= 0
+      expect(stats[:max_execution_time_ms]).to be >= 0
+    end
+
+    it "handles nil execution times in statistics" do
+      scenarios = [
+        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 })
+      ]
+      runner.run(scenarios)
+
+      # Manually add a result with nil execution time
+      runner.instance_variable_get(:@results) << DecisionAgent::Testing::TestResult.new(
+        scenario_id: "test_2",
+        execution_time_ms: nil
+      )
+
+      stats = runner.statistics
+      expect(stats[:total]).to eq(2)
+    end
+  end
+
+  describe "#resume" do
+    let(:scenarios) do
+      [
+        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
+        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
+      ]
+    end
+
+    it "resumes from checkpoint file" do
+      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
+      checkpoint_file.write(JSON.pretty_generate({ completed_scenario_ids: ["test_1"], last_updated: Time.now.to_i }))
+      checkpoint_file.close
+
+      results = runner.resume(scenarios, checkpoint_file.path)
+
+      # Should only run test_2 since test_1 is already completed
+      expect(results.size).to eq(1)
+      expect(results[0].scenario_id).to eq("test_2")
+
+      checkpoint_file.unlink
+    end
+  end
+
+  describe "checkpoint functionality" do
+    let(:scenarios) do
+      [
+        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
+        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
+      ]
+    end
+
+    it "saves checkpoints during execution" do
+      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
+      checkpoint_file.close
+      File.delete(checkpoint_file.path) # Start with no file
+
+      runner.run(scenarios, checkpoint_file: checkpoint_file.path)
+
+      # Checkpoint file should exist and contain completed scenario IDs
+      expect(File.exist?(checkpoint_file.path)).to be false # Should be cleaned up after completion
+    end
+
+    it "handles checkpoint file errors gracefully" do
+      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
+      checkpoint_file.close
+
+      # Make file read-only to cause write errors
+      File.chmod(0o444, checkpoint_file.path)
+
+      expect do
+        runner.run(scenarios, checkpoint_file: checkpoint_file.path)
+      end.not_to raise_error
+
+      # Clean up - file might have been deleted, so check first
+      if File.exist?(checkpoint_file.path)
+        File.chmod(0o644, checkpoint_file.path)
+        checkpoint_file.unlink
+      end
+    end
+
+    it "loads checkpoint data correctly" do
+      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
+      checkpoint_data = {
+        completed_scenario_ids: ["test_1"],
+        last_updated: Time.now.to_i
+      }
+      checkpoint_file.write(JSON.pretty_generate(checkpoint_data))
+      checkpoint_file.close
+
+      results = runner.run(scenarios, checkpoint_file: checkpoint_file.path)
+
+      # Should only execute test_2
+      expect(results.size).to eq(1)
+      expect(results[0].scenario_id).to eq("test_2")
+
+      checkpoint_file.unlink
+    end
+
+    it "handles invalid JSON in checkpoint file" do
+      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
+      checkpoint_file.write("invalid json")
+      checkpoint_file.close
+
+      # Should handle gracefully and start fresh
+      results = runner.run(scenarios, checkpoint_file: checkpoint_file.path)
+      expect(results.size).to eq(2)
+
+      checkpoint_file.unlink
+    end
+  end
+
+  describe "TestResult" do
+    let(:result) do
+      DecisionAgent::Testing::TestResult.new(
+        scenario_id: "test_1",
+        decision: "approve",
+        confidence: 0.95,
+        execution_time_ms: 10.5
+      )
+    end
+
+    it "creates a successful test result" do
+      expect(result.success?).to be true
+      expect(result.scenario_id).to eq("test_1")
+      expect(result.decision).to eq("approve")
+      expect(result.confidence).to eq(0.95)
+      expect(result.execution_time_ms).to eq(10.5)
+    end
+
+    it "creates a failed test result" do
+      error = StandardError.new("Test error")
+      failed_result = DecisionAgent::Testing::TestResult.new(
+        scenario_id: "test_1",
+        error: error
+      )
+
+      expect(failed_result.success?).to be false
+      expect(failed_result.error).to eq(error)
+    end
+
+    it "converts to hash" do
+      hash = result.to_h
+
+      expect(hash[:scenario_id]).to eq("test_1")
+      expect(hash[:decision]).to eq("approve")
+      expect(hash[:confidence]).to eq(0.95)
+      expect(hash[:execution_time_ms]).to eq(10.5)
+      expect(hash[:success]).to be true
+    end
+
+    it "includes evaluations in hash" do
+      evaluation = DecisionAgent::Evaluation.new(
+        decision: "approve",
+        weight: 1.0,
+        reason: "Test",
+        evaluator_name: "TestEvaluator"
+      )
+      result_with_eval = DecisionAgent::Testing::TestResult.new(
+        scenario_id: "test_1",
+        decision: "approve",
+        evaluations: [evaluation]
+      )
+
+      hash = result_with_eval.to_h
+      expect(hash[:evaluations]).to be_an(Array)
+      expect(hash[:evaluations].first).to respond_to(:to_h)
+    end
+
+    it "handles nil decision and confidence" do
+      result = DecisionAgent::Testing::TestResult.new(
+        scenario_id: "test_1",
+        decision: nil,
+        confidence: nil
+      )
+
+      expect(result.decision).to be_nil
+      expect(result.confidence).to be_nil
+    end
+  end
+end

data/spec/testing/test_coverage_analyzer_spec.rb

@@ -0,0 +1,292 @@
+require "spec_helper"
+
+RSpec.describe DecisionAgent::Testing::TestCoverageAnalyzer do
+  let(:analyzer) { DecisionAgent::Testing::TestCoverageAnalyzer.new }
+
+  describe "#analyze" do
+    let(:evaluator) do
+      evaluator = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
+      # Add metadata to simulate rule_id
+      allow(evaluator).to receive(:evaluate).and_wrap_original do |method, context, **kwargs|
+        evaluation = method.call(context, **kwargs)
+        # Create a new evaluation with metadata
+        DecisionAgent::Evaluation.new(
+          decision: evaluation.decision,
+          weight: evaluation.weight,
+          reason: evaluation.reason,
+          evaluator_name: evaluation.evaluator_name,
+          metadata: { rule_id: "rule_1", condition_id: "condition_1" }
+        )
+      end
+      evaluator
+    end
+
+    let(:agent) { DecisionAgent::Agent.new(evaluators: [evaluator]) }
+
+    let(:results) do
+      [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95,
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: "TestEvaluator",
+              metadata: { rule_id: "rule_1", condition_id: "condition_1" }
+            )
+          ]
+        ),
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_2",
+          decision: "approve",
+          confidence: 0.90,
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: "TestEvaluator",
+              metadata: { rule_id: "rule_2", condition_id: "condition_2" }
+            )
+          ]
+        )
+      ]
+    end
+
+    it "analyzes coverage from test results" do
+      report = analyzer.analyze(results, agent)
+
+      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
+      expect(report.covered_rules).to be >= 0
+      # Coverage percentage should be capped at 1.0
+      expect(report.coverage_percentage).to be <= 1.0
+      expect(report.coverage_percentage).to be >= 0.0
+    end
+
+    it "tracks executed rules" do
+      report = analyzer.analyze(results, agent)
+
+      # Should have tracked at least some rules
+      expect(report.rule_coverage).to be_an(Array)
+    end
+
+    it "identifies untested rules when agent is provided" do
+      # Create agent with multiple evaluators
+      evaluator1 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
+      evaluator2 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "reject", weight: 1.0)
+
+      multi_agent = DecisionAgent::Agent.new(evaluators: [evaluator1, evaluator2])
+
+      # Results only exercise one rule
+      single_result = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95,
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: evaluator1.class.name,
+              metadata: { rule_id: "rule_1" }
+            )
+          ]
+        )
+      ]
+
+      report = analyzer.analyze(single_result, multi_agent)
+
+      expect(report.total_rules).to be >= 1
+      expect(report.coverage_percentage).to be <= 1.0
+    end
+
+    it "handles agent with evaluators that have rules method" do
+      evaluator_with_rules = double("EvaluatorWithRules",
+                                    evaluate: nil,
+                                    rules: %w[rule_1 rule_2])
+      agent_with_rules = DecisionAgent::Agent.new(evaluators: [evaluator_with_rules])
+
+      report = analyzer.analyze([], agent_with_rules)
+      expect(report.total_rules).to be >= 2
+    end
+
+    it "handles agent with evaluators that have rule_id method" do
+      evaluator_with_rule_id = double("EvaluatorWithRuleId",
+                                      evaluate: nil,
+                                      rule_id: "my_rule_1")
+      agent_with_rule_id = DecisionAgent::Agent.new(evaluators: [evaluator_with_rule_id])
+
+      report = analyzer.analyze([], agent_with_rule_id)
+      expect(report.total_rules).to be >= 1
+    end
+
+    it "handles agent with evaluators that have conditions method" do
+      evaluator_with_conditions = double("EvaluatorWithConditions",
+                                         evaluate: nil,
+                                         conditions: %w[condition_1 condition_2])
+      agent_with_conditions = DecisionAgent::Agent.new(evaluators: [evaluator_with_conditions])
+
+      report = analyzer.analyze([], agent_with_conditions)
+      expect(report.condition_coverage).to be_an(Array)
+    end
+
+    it "handles agent with evaluators that have condition_id method" do
+      evaluator_with_condition_id = double("EvaluatorWithConditionId",
+                                           evaluate: nil,
+                                           condition_id: "my_condition_1")
+      agent_with_condition_id = DecisionAgent::Agent.new(evaluators: [evaluator_with_condition_id])
+
+      report = analyzer.analyze([], agent_with_condition_id)
+      expect(report.condition_coverage).to be_an(Array)
+    end
+
+    it "caps coverage percentage at 1.0" do
+      # Create a scenario where coverage could exceed 1.0
+      evaluator1 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
+      single_agent = DecisionAgent::Agent.new(evaluators: [evaluator1])
+
+      results_with_metadata = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: evaluator1.class.name,
+              metadata: { rule_id: "rule_1" }
+            )
+          ]
+        )
+      ]
+
+      report = analyzer.analyze(results_with_metadata, single_agent)
+      expect(report.coverage_percentage).to be <= 1.0
+    end
+
+    it "handles results without agent" do
+      report = analyzer.analyze(results, nil)
+
+      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
+      expect(report.covered_rules).to be >= 0
+    end
+
+    it "handles empty results" do
+      report = analyzer.analyze([], agent)
+
+      expect(report.covered_rules).to eq(0)
+      expect(report.coverage_percentage).to eq(0.0)
+    end
+
+    it "handles failed test results" do
+      failed_results = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          error: StandardError.new("Test failed")
+        )
+      ]
+
+      report = analyzer.analyze(failed_results, agent)
+
+      expect(report.covered_rules).to eq(0)
+    end
+
+    it "handles evaluations without metadata" do
+      results_no_metadata = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: "TestEvaluator"
+              # No metadata
+            )
+          ]
+        )
+      ]
+
+      report = analyzer.analyze(results_no_metadata, agent)
+      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
+    end
+
+    it "handles evaluations with evaluator_name as rule identifier" do
+      results_eval_name = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          evaluations: [
+            DecisionAgent::Evaluation.new(
+              decision: "approve",
+              weight: 1.0,
+              reason: "Test",
+              evaluator_name: "MyEvaluator"
+              # No metadata, should use evaluator_name
+            )
+          ]
+        )
+      ]
+
+      report = analyzer.analyze(results_eval_name, agent)
+      expect(report.covered_rules).to be >= 0
+    end
+  end
+
+  describe "#coverage_percentage" do
+    it "returns 0.0 when no rules executed" do
+      expect(analyzer.coverage_percentage).to eq(0.0)
+    end
+
+    it "returns 0.0 when rule_evaluation_count is empty" do
+      analyzer.instance_variable_set(:@executed_rules, Set.new(["rule_1"]))
+      analyzer.instance_variable_set(:@rule_evaluation_count, {})
+
+      expect(analyzer.coverage_percentage).to eq(0.0)
+    end
+  end
+
+  describe "CoverageReport" do
+    let(:report) do
+      DecisionAgent::Testing::CoverageReport.new(
+        total_rules: 10,
+        covered_rules: 7,
+        untested_rules: %w[rule_8 rule_9 rule_10],
+        coverage_percentage: 0.7,
+        rule_coverage: [
+          { rule_id: "rule_1", covered: true, execution_count: 5 }
+        ],
+        condition_coverage: [
+          { condition_id: "condition_1", covered: true, execution_count: 3 }
+        ]
+      )
+    end
+
+    it "creates a coverage report" do
+      expect(report.total_rules).to eq(10)
+      expect(report.covered_rules).to eq(7)
+      expect(report.untested_rules).to eq(%w[rule_8 rule_9 rule_10])
+      expect(report.coverage_percentage).to eq(0.7)
+    end
+
+    it "converts to hash" do
+      hash = report.to_h
+
+      expect(hash[:total_rules]).to eq(10)
+      expect(hash[:covered_rules]).to eq(7)
+      expect(hash[:coverage_percentage]).to eq(0.7)
+      expect(hash[:rule_coverage]).to be_an(Array)
+      expect(hash[:condition_coverage]).to be_an(Array)
+    end
+
+    it "freezes the report" do
+      expect(report.frozen?).to be true
+    end
+  end
+end