decision_agent 0.3.0 → 1.0.1

data/spec/testing/batch_test_runner_spec.rb

@@ -1,307 +0,0 @@
-require "spec_helper"
-require "tempfile"
-
-RSpec.describe DecisionAgent::Testing::BatchTestRunner do
-  let(:evaluator) { DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0) }
-  let(:agent) { DecisionAgent::Agent.new(evaluators: [evaluator]) }
-  let(:runner) { DecisionAgent::Testing::BatchTestRunner.new(agent) }
-
-  describe "#initialize" do
-    it "creates a batch test runner with an agent" do
-      expect(runner.agent).to eq(agent)
-      expect(runner.results).to eq([])
-    end
-  end
-
-  describe "#run" do
-    let(:scenarios) do
-      [
-        DecisionAgent::Testing::TestScenario.new(
-          id: "test_1",
-          context: { user_id: 123 }
-        ),
-        DecisionAgent::Testing::TestScenario.new(
-          id: "test_2",
-          context: { user_id: 456 }
-        )
-      ]
-    end
-
-    it "executes test scenarios sequentially" do
-      results = runner.run(scenarios, parallel: false)
-
-      expect(results.size).to eq(2)
-      expect(results.all?(&:success?)).to be true
-      expect(results[0].decision).to eq("approve")
-      expect(results[1].decision).to eq("approve")
-    end
-
-    it "executes test scenarios in parallel when enabled" do
-      results = runner.run(scenarios, parallel: true, thread_count: 2)
-
-      expect(results.size).to eq(2)
-      expect(results.all?(&:success?)).to be true
-    end
-
-    it "executes single scenario sequentially even with parallel enabled" do
-      single_scenario = [scenarios[0]]
-      results = runner.run(single_scenario, parallel: true)
-
-      expect(results.size).to eq(1)
-      expect(results.all?(&:success?)).to be true
-    end
-
-    it "handles empty scenarios array" do
-      results = runner.run([])
-      expect(results).to eq([])
-    end
-
-    it "calls progress callback during execution" do
-      progress_updates = []
-
-      runner.run(scenarios, progress_callback: lambda { |progress|
-        progress_updates << progress
-      })
-
-      expect(progress_updates.size).to be >= 2
-      expect(progress_updates.last[:completed]).to eq(2)
-      expect(progress_updates.last[:total]).to eq(2)
-      expect(progress_updates.last[:percentage]).to eq(100.0)
-    end
-
-    it "tracks execution time for each scenario" do
-      results = runner.run(scenarios)
-
-      results.each do |result|
-        expect(result.execution_time_ms).to be >= 0
-        expect(result.execution_time_ms).to be_a(Numeric)
-      end
-    end
-
-    it "handles errors gracefully" do
-      # Create an agent that will raise an error
-      error_evaluator = Class.new do
-        def evaluate(_context, _feedback: {})
-          raise StandardError, "Test error"
-        end
-      end.new
-
-      error_agent = DecisionAgent::Agent.new(evaluators: [error_evaluator])
-      error_runner = DecisionAgent::Testing::BatchTestRunner.new(error_agent)
-
-      results = error_runner.run(scenarios)
-
-      expect(results.size).to eq(2)
-      expect(results.none?(&:success?)).to be true
-      expect(results[0].error).to be_a(StandardError)
-    end
-
-    it "passes feedback to agent" do
-      feedback = { source: "batch_test" }
-      results = runner.run(scenarios, feedback: feedback)
-
-      expect(results.size).to eq(2)
-      expect(results.all?(&:success?)).to be true
-    end
-  end
-
-  describe "#statistics" do
-    it "returns empty hash when no results" do
-      expect(runner.statistics).to eq({})
-    end
-
-    it "calculates statistics from results" do
-      scenarios = [
-        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
-        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
-      ]
-
-      runner.run(scenarios)
-      stats = runner.statistics
-
-      expect(stats[:total]).to eq(2)
-      expect(stats[:successful]).to eq(2)
-      expect(stats[:failed]).to eq(0)
-      expect(stats[:success_rate]).to eq(1.0)
-      expect(stats[:avg_execution_time_ms]).to be >= 0
-      expect(stats[:min_execution_time_ms]).to be >= 0
-      expect(stats[:max_execution_time_ms]).to be >= 0
-    end
-
-    it "handles nil execution times in statistics" do
-      scenarios = [
-        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 })
-      ]
-      runner.run(scenarios)
-
-      # Manually add a result with nil execution time
-      runner.instance_variable_get(:@results) << DecisionAgent::Testing::TestResult.new(
-        scenario_id: "test_2",
-        execution_time_ms: nil
-      )
-
-      stats = runner.statistics
-      expect(stats[:total]).to eq(2)
-    end
-  end
-
-  describe "#resume" do
-    let(:scenarios) do
-      [
-        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
-        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
-      ]
-    end
-
-    it "resumes from checkpoint file" do
-      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
-      checkpoint_file.write(JSON.pretty_generate({ completed_scenario_ids: ["test_1"], last_updated: Time.now.to_i }))
-      checkpoint_file.close
-
-      results = runner.resume(scenarios, checkpoint_file.path)
-
-      # Should only run test_2 since test_1 is already completed
-      expect(results.size).to eq(1)
-      expect(results[0].scenario_id).to eq("test_2")
-
-      checkpoint_file.unlink
-    end
-  end
-
-  describe "checkpoint functionality" do
-    let(:scenarios) do
-      [
-        DecisionAgent::Testing::TestScenario.new(id: "test_1", context: { user_id: 123 }),
-        DecisionAgent::Testing::TestScenario.new(id: "test_2", context: { user_id: 456 })
-      ]
-    end
-
-    it "saves checkpoints during execution" do
-      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
-      checkpoint_file.close
-      File.delete(checkpoint_file.path) # Start with no file
-
-      runner.run(scenarios, checkpoint_file: checkpoint_file.path)
-
-      # Checkpoint file should exist and contain completed scenario IDs
-      expect(File.exist?(checkpoint_file.path)).to be false # Should be cleaned up after completion
-    end
-
-    it "handles checkpoint file errors gracefully" do
-      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
-      checkpoint_file.close
-
-      # Make file read-only to cause write errors
-      File.chmod(0o444, checkpoint_file.path)
-
-      expect do
-        runner.run(scenarios, checkpoint_file: checkpoint_file.path)
-      end.not_to raise_error
-
-      # Clean up - file might have been deleted, so check first
-      if File.exist?(checkpoint_file.path)
-        File.chmod(0o644, checkpoint_file.path)
-        checkpoint_file.unlink
-      end
-    end
-
-    it "loads checkpoint data correctly" do
-      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
-      checkpoint_data = {
-        completed_scenario_ids: ["test_1"],
-        last_updated: Time.now.to_i
-      }
-      checkpoint_file.write(JSON.pretty_generate(checkpoint_data))
-      checkpoint_file.close
-
-      results = runner.run(scenarios, checkpoint_file: checkpoint_file.path)
-
-      # Should only execute test_2
-      expect(results.size).to eq(1)
-      expect(results[0].scenario_id).to eq("test_2")
-
-      checkpoint_file.unlink
-    end
-
-    it "handles invalid JSON in checkpoint file" do
-      checkpoint_file = Tempfile.new(["checkpoint", ".json"])
-      checkpoint_file.write("invalid json")
-      checkpoint_file.close
-
-      # Should handle gracefully and start fresh
-      results = runner.run(scenarios, checkpoint_file: checkpoint_file.path)
-      expect(results.size).to eq(2)
-
-      checkpoint_file.unlink
-    end
-  end
-
-  describe "TestResult" do
-    let(:result) do
-      DecisionAgent::Testing::TestResult.new(
-        scenario_id: "test_1",
-        decision: "approve",
-        confidence: 0.95,
-        execution_time_ms: 10.5
-      )
-    end
-
-    it "creates a successful test result" do
-      expect(result.success?).to be true
-      expect(result.scenario_id).to eq("test_1")
-      expect(result.decision).to eq("approve")
-      expect(result.confidence).to eq(0.95)
-      expect(result.execution_time_ms).to eq(10.5)
-    end
-
-    it "creates a failed test result" do
-      error = StandardError.new("Test error")
-      failed_result = DecisionAgent::Testing::TestResult.new(
-        scenario_id: "test_1",
-        error: error
-      )
-
-      expect(failed_result.success?).to be false
-      expect(failed_result.error).to eq(error)
-    end
-
-    it "converts to hash" do
-      hash = result.to_h
-
-      expect(hash[:scenario_id]).to eq("test_1")
-      expect(hash[:decision]).to eq("approve")
-      expect(hash[:confidence]).to eq(0.95)
-      expect(hash[:execution_time_ms]).to eq(10.5)
-      expect(hash[:success]).to be true
-    end
-
-    it "includes evaluations in hash" do
-      evaluation = DecisionAgent::Evaluation.new(
-        decision: "approve",
-        weight: 1.0,
-        reason: "Test",
-        evaluator_name: "TestEvaluator"
-      )
-      result_with_eval = DecisionAgent::Testing::TestResult.new(
-        scenario_id: "test_1",
-        decision: "approve",
-        evaluations: [evaluation]
-      )
-
-      hash = result_with_eval.to_h
-      expect(hash[:evaluations]).to be_an(Array)
-      expect(hash[:evaluations].first).to respond_to(:to_h)
-    end
-
-    it "handles nil decision and confidence" do
-      result = DecisionAgent::Testing::TestResult.new(
-        scenario_id: "test_1",
-        decision: nil,
-        confidence: nil
-      )
-
-      expect(result.decision).to be_nil
-      expect(result.confidence).to be_nil
-    end
-  end
-end

data/spec/testing/test_coverage_analyzer_spec.rb

@@ -1,292 +0,0 @@
-require "spec_helper"
-
-RSpec.describe DecisionAgent::Testing::TestCoverageAnalyzer do
-  let(:analyzer) { DecisionAgent::Testing::TestCoverageAnalyzer.new }
-
-  describe "#analyze" do
-    let(:evaluator) do
-      evaluator = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
-      # Add metadata to simulate rule_id
-      allow(evaluator).to receive(:evaluate).and_wrap_original do |method, context, **kwargs|
-        evaluation = method.call(context, **kwargs)
-        # Create a new evaluation with metadata
-        DecisionAgent::Evaluation.new(
-          decision: evaluation.decision,
-          weight: evaluation.weight,
-          reason: evaluation.reason,
-          evaluator_name: evaluation.evaluator_name,
-          metadata: { rule_id: "rule_1", condition_id: "condition_1" }
-        )
-      end
-      evaluator
-    end
-
-    let(:agent) { DecisionAgent::Agent.new(evaluators: [evaluator]) }
-
-    let(:results) do
-      [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          decision: "approve",
-          confidence: 0.95,
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: "TestEvaluator",
-              metadata: { rule_id: "rule_1", condition_id: "condition_1" }
-            )
-          ]
-        ),
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_2",
-          decision: "approve",
-          confidence: 0.90,
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: "TestEvaluator",
-              metadata: { rule_id: "rule_2", condition_id: "condition_2" }
-            )
-          ]
-        )
-      ]
-    end
-
-    it "analyzes coverage from test results" do
-      report = analyzer.analyze(results, agent)
-
-      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
-      expect(report.covered_rules).to be >= 0
-      # Coverage percentage should be capped at 1.0
-      expect(report.coverage_percentage).to be <= 1.0
-      expect(report.coverage_percentage).to be >= 0.0
-    end
-
-    it "tracks executed rules" do
-      report = analyzer.analyze(results, agent)
-
-      # Should have tracked at least some rules
-      expect(report.rule_coverage).to be_an(Array)
-    end
-
-    it "identifies untested rules when agent is provided" do
-      # Create agent with multiple evaluators
-      evaluator1 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
-      evaluator2 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "reject", weight: 1.0)
-
-      multi_agent = DecisionAgent::Agent.new(evaluators: [evaluator1, evaluator2])
-
-      # Results only exercise one rule
-      single_result = [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          decision: "approve",
-          confidence: 0.95,
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: evaluator1.class.name,
-              metadata: { rule_id: "rule_1" }
-            )
-          ]
-        )
-      ]
-
-      report = analyzer.analyze(single_result, multi_agent)
-
-      expect(report.total_rules).to be >= 1
-      expect(report.coverage_percentage).to be <= 1.0
-    end
-
-    it "handles agent with evaluators that have rules method" do
-      evaluator_with_rules = double("EvaluatorWithRules",
-                                    evaluate: nil,
-                                    rules: %w[rule_1 rule_2])
-      agent_with_rules = DecisionAgent::Agent.new(evaluators: [evaluator_with_rules])
-
-      report = analyzer.analyze([], agent_with_rules)
-      expect(report.total_rules).to be >= 2
-    end
-
-    it "handles agent with evaluators that have rule_id method" do
-      evaluator_with_rule_id = double("EvaluatorWithRuleId",
-                                      evaluate: nil,
-                                      rule_id: "my_rule_1")
-      agent_with_rule_id = DecisionAgent::Agent.new(evaluators: [evaluator_with_rule_id])
-
-      report = analyzer.analyze([], agent_with_rule_id)
-      expect(report.total_rules).to be >= 1
-    end
-
-    it "handles agent with evaluators that have conditions method" do
-      evaluator_with_conditions = double("EvaluatorWithConditions",
-                                         evaluate: nil,
-                                         conditions: %w[condition_1 condition_2])
-      agent_with_conditions = DecisionAgent::Agent.new(evaluators: [evaluator_with_conditions])
-
-      report = analyzer.analyze([], agent_with_conditions)
-      expect(report.condition_coverage).to be_an(Array)
-    end
-
-    it "handles agent with evaluators that have condition_id method" do
-      evaluator_with_condition_id = double("EvaluatorWithConditionId",
-                                           evaluate: nil,
-                                           condition_id: "my_condition_1")
-      agent_with_condition_id = DecisionAgent::Agent.new(evaluators: [evaluator_with_condition_id])
-
-      report = analyzer.analyze([], agent_with_condition_id)
-      expect(report.condition_coverage).to be_an(Array)
-    end
-
-    it "caps coverage percentage at 1.0" do
-      # Create a scenario where coverage could exceed 1.0
-      evaluator1 = DecisionAgent::Evaluators::StaticEvaluator.new(decision: "approve", weight: 1.0)
-      single_agent = DecisionAgent::Agent.new(evaluators: [evaluator1])
-
-      results_with_metadata = [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          decision: "approve",
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: evaluator1.class.name,
-              metadata: { rule_id: "rule_1" }
-            )
-          ]
-        )
-      ]
-
-      report = analyzer.analyze(results_with_metadata, single_agent)
-      expect(report.coverage_percentage).to be <= 1.0
-    end
-
-    it "handles results without agent" do
-      report = analyzer.analyze(results, nil)
-
-      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
-      expect(report.covered_rules).to be >= 0
-    end
-
-    it "handles empty results" do
-      report = analyzer.analyze([], agent)
-
-      expect(report.covered_rules).to eq(0)
-      expect(report.coverage_percentage).to eq(0.0)
-    end
-
-    it "handles failed test results" do
-      failed_results = [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          error: StandardError.new("Test failed")
-        )
-      ]
-
-      report = analyzer.analyze(failed_results, agent)
-
-      expect(report.covered_rules).to eq(0)
-    end
-
-    it "handles evaluations without metadata" do
-      results_no_metadata = [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          decision: "approve",
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: "TestEvaluator"
-              # No metadata
-            )
-          ]
-        )
-      ]
-
-      report = analyzer.analyze(results_no_metadata, agent)
-      expect(report).to be_a(DecisionAgent::Testing::CoverageReport)
-    end
-
-    it "handles evaluations with evaluator_name as rule identifier" do
-      results_eval_name = [
-        DecisionAgent::Testing::TestResult.new(
-          scenario_id: "test_1",
-          decision: "approve",
-          evaluations: [
-            DecisionAgent::Evaluation.new(
-              decision: "approve",
-              weight: 1.0,
-              reason: "Test",
-              evaluator_name: "MyEvaluator"
-              # No metadata, should use evaluator_name
-            )
-          ]
-        )
-      ]
-
-      report = analyzer.analyze(results_eval_name, agent)
-      expect(report.covered_rules).to be >= 0
-    end
-  end
-
-  describe "#coverage_percentage" do
-    it "returns 0.0 when no rules executed" do
-      expect(analyzer.coverage_percentage).to eq(0.0)
-    end
-
-    it "returns 0.0 when rule_evaluation_count is empty" do
-      analyzer.instance_variable_set(:@executed_rules, Set.new(["rule_1"]))
-      analyzer.instance_variable_set(:@rule_evaluation_count, {})
-
-      expect(analyzer.coverage_percentage).to eq(0.0)
-    end
-  end
-
-  describe "CoverageReport" do
-    let(:report) do
-      DecisionAgent::Testing::CoverageReport.new(
-        total_rules: 10,
-        covered_rules: 7,
-        untested_rules: %w[rule_8 rule_9 rule_10],
-        coverage_percentage: 0.7,
-        rule_coverage: [
-          { rule_id: "rule_1", covered: true, execution_count: 5 }
-        ],
-        condition_coverage: [
-          { condition_id: "condition_1", covered: true, execution_count: 3 }
-        ]
-      )
-    end
-
-    it "creates a coverage report" do
-      expect(report.total_rules).to eq(10)
-      expect(report.covered_rules).to eq(7)
-      expect(report.untested_rules).to eq(%w[rule_8 rule_9 rule_10])
-      expect(report.coverage_percentage).to eq(0.7)
-    end
-
-    it "converts to hash" do
-      hash = report.to_h
-
-      expect(hash[:total_rules]).to eq(10)
-      expect(hash[:covered_rules]).to eq(7)
-      expect(hash[:coverage_percentage]).to eq(0.7)
-      expect(hash[:rule_coverage]).to be_an(Array)
-      expect(hash[:condition_coverage]).to be_an(Array)
-    end
-
-    it "freezes the report" do
-      expect(report.frozen?).to be true
-    end
-  end
-end