decision_agent 0.1.3 → 0.1.6

data/spec/testing/test_result_comparator_spec.rb

@@ -0,0 +1,392 @@
+require "spec_helper"
+require "tempfile"
+
+RSpec.describe DecisionAgent::Testing::TestResultComparator do
+  let(:comparator) { DecisionAgent::Testing::TestResultComparator.new }
+
+  describe "#compare" do
+    let(:scenarios) do
+      [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        ),
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_2",
+          context: { user_id: 456 },
+          expected_decision: "reject",
+          expected_confidence: 0.80
+        )
+      ]
+    end
+
+    let(:results) do
+      [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95
+        ),
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_2",
+          decision: "reject",
+          confidence: 0.80
+        )
+      ]
+    end
+
+    it "compares results with expected outcomes" do
+      summary = comparator.compare(results, scenarios)
+
+      expect(summary[:total]).to eq(2)
+      expect(summary[:matches]).to eq(2)
+      expect(summary[:mismatches]).to eq(0)
+      expect(summary[:accuracy_rate]).to eq(1.0)
+    end
+
+    it "identifies mismatches" do
+      mismatched_results = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "reject", # Wrong decision
+          confidence: 0.95
+        ),
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_2",
+          decision: "reject",
+          confidence: 0.50 # Wrong confidence
+        )
+      ]
+
+      summary = comparator.compare(mismatched_results, scenarios)
+
+      expect(summary[:matches]).to eq(0)
+      expect(summary[:mismatches]).to eq(2)
+      expect(summary[:accuracy_rate]).to eq(0.0)
+      expect(summary[:mismatches_detail].size).to eq(2)
+    end
+
+    it "handles confidence tolerance" do
+      comparator_with_tolerance = DecisionAgent::Testing::TestResultComparator.new(
+        confidence_tolerance: 0.1
+      )
+
+      results_with_tolerance = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.96 # Within 0.1 tolerance of 0.95
+        )
+      ]
+
+      scenarios_single = [scenarios[0]]
+      summary = comparator_with_tolerance.compare(results_with_tolerance, scenarios_single)
+
+      expect(summary[:matches]).to eq(1)
+      expect(summary[:confidence_accuracy]).to eq(1.0)
+    end
+
+    it "handles fuzzy matching" do
+      comparator_fuzzy = DecisionAgent::Testing::TestResultComparator.new(fuzzy_match: true)
+
+      scenarios_fuzzy = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "APPROVE", # Uppercase
+          expected_confidence: 0.95
+        )
+      ]
+
+      results_fuzzy = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve", # Lowercase - should match with fuzzy
+          confidence: 0.95
+        )
+      ]
+
+      summary = comparator_fuzzy.compare(results_fuzzy, scenarios_fuzzy)
+      expect(summary[:matches]).to eq(1)
+    end
+
+    it "handles fuzzy matching with whitespace" do
+      comparator_fuzzy = DecisionAgent::Testing::TestResultComparator.new(fuzzy_match: true)
+
+      scenarios_fuzzy = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: " approve ", # With spaces
+          expected_confidence: 0.95
+        )
+      ]
+
+      results_fuzzy = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve", # Without spaces - should match with fuzzy
+          confidence: 0.95
+        )
+      ]
+
+      summary = comparator_fuzzy.compare(results_fuzzy, scenarios_fuzzy)
+      expect(summary[:matches]).to eq(1)
+    end
+
+    it "handles nil expected confidence" do
+      scenarios_nil_conf = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: nil
+        )
+      ]
+
+      results_nil_conf = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95
+        )
+      ]
+
+      summary = comparator.compare(results_nil_conf, scenarios_nil_conf)
+      expect(summary[:matches]).to eq(1)
+    end
+
+    it "handles nil actual confidence when expected is present" do
+      scenarios_with_conf = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        )
+      ]
+
+      results_no_conf = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: nil
+        )
+      ]
+
+      summary = comparator.compare(results_no_conf, scenarios_with_conf)
+      expect(summary[:matches]).to eq(0)
+      expect(summary[:mismatches]).to eq(1)
+    end
+
+    it "handles missing results for scenarios" do
+      scenarios_missing = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        ),
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_2",
+          context: { user_id: 456 },
+          expected_decision: "reject",
+          expected_confidence: 0.80
+        )
+      ]
+
+      # Only provide result for test_1
+      results_missing = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95
+        )
+      ]
+
+      summary = comparator.compare(results_missing, scenarios_missing)
+      # Should only compare test_1 since test_2 has no result
+      expect(summary[:total]).to eq(1)
+    end
+
+    it "handles confidence outside tolerance" do
+      comparator_strict = DecisionAgent::Testing::TestResultComparator.new(
+        confidence_tolerance: 0.01
+      )
+
+      scenarios_strict = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        )
+      ]
+
+      results_outside = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.98 # Outside 0.01 tolerance
+        )
+      ]
+
+      summary = comparator_strict.compare(results_outside, scenarios_strict)
+      expect(summary[:matches]).to eq(0)
+      expect(summary[:confidence_accuracy]).to eq(0.0)
+    end
+
+    it "handles missing expected results" do
+      scenarios_no_expected = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 }
+          # No expected_decision
+        )
+      ]
+
+      summary = comparator.compare(results, scenarios_no_expected)
+
+      # Should not compare scenarios without expected results
+      expect(summary[:total]).to eq(0)
+    end
+
+    it "handles failed test results" do
+      failed_results = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          error: StandardError.new("Test failed")
+        )
+      ]
+
+      # Only compare scenarios that have expected results
+      scenarios_with_expected = scenarios.select(&:expected_result?)
+      summary = comparator.compare(failed_results, scenarios_with_expected)
+
+      expect(summary[:mismatches]).to eq(1)
+      expect(comparator.comparison_results[0].match).to be false
+    end
+  end
+
+  describe "#generate_summary" do
+    it "returns empty summary when no comparisons" do
+      summary = comparator.generate_summary
+
+      expect(summary[:total]).to eq(0)
+      expect(summary[:matches]).to eq(0)
+      expect(summary[:accuracy_rate]).to eq(0.0)
+    end
+  end
+
+  describe "#export_csv" do
+    it "exports comparison results to CSV" do
+      scenarios = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        )
+      ]
+
+      results = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95
+        )
+      ]
+
+      comparator.compare(results, scenarios)
+
+      file = Tempfile.new(["comparison", ".csv"])
+      comparator.export_csv(file.path)
+
+      content = File.read(file.path)
+      expect(content).to include("scenario_id")
+      expect(content).to include("test_1")
+      expect(content).to include("true") # match
+
+      file.unlink
+    end
+  end
+
+  describe "#export_json" do
+    it "exports comparison results to JSON" do
+      scenarios = [
+        DecisionAgent::Testing::TestScenario.new(
+          id: "test_1",
+          context: { user_id: 123 },
+          expected_decision: "approve",
+          expected_confidence: 0.95
+        )
+      ]
+
+      results = [
+        DecisionAgent::Testing::TestResult.new(
+          scenario_id: "test_1",
+          decision: "approve",
+          confidence: 0.95
+        )
+      ]
+
+      comparator.compare(results, scenarios)
+
+      file = Tempfile.new(["comparison", ".json"])
+      comparator.export_json(file.path)
+
+      content = JSON.parse(File.read(file.path))
+      expect(content).to have_key("summary")
+      expect(content).to have_key("results")
+      expect(content["summary"]["total"]).to eq(1)
+
+      file.unlink
+    end
+
+    it "handles empty comparison results" do
+      file = Tempfile.new(["comparison", ".csv"])
+      comparator.export_csv(file.path)
+
+      content = File.read(file.path)
+      expect(content).to include("scenario_id")
+
+      file.unlink
+    end
+  end
+
+  describe "ComparisonResult" do
+    let(:comparison_result) do
+      DecisionAgent::Testing::ComparisonResult.new(
+        scenario_id: "test_1",
+        match: true,
+        decision_match: true,
+        confidence_match: true,
+        differences: [],
+        actual: { decision: "approve", confidence: 0.95 },
+        expected: { decision: "approve", confidence: 0.95 }
+      )
+    end
+
+    it "creates a comparison result" do
+      expect(comparison_result.scenario_id).to eq("test_1")
+      expect(comparison_result.match).to be true
+      expect(comparison_result.decision_match).to be true
+      expect(comparison_result.confidence_match).to be true
+    end
+
+    it "converts to hash" do
+      hash = comparison_result.to_h
+
+      expect(hash[:scenario_id]).to eq("test_1")
+      expect(hash[:match]).to be true
+      expect(hash[:actual][:decision]).to eq("approve")
+      expect(hash[:expected][:decision]).to eq("approve")
+    end
+
+    it "freezes the comparison result" do
+      expect(comparison_result.frozen?).to be true
+    end
+  end
+end

data/spec/testing/test_scenario_spec.rb

@@ -0,0 +1,113 @@
+require "spec_helper"
+
+RSpec.describe DecisionAgent::Testing::TestScenario do
+  describe "#initialize" do
+    it "creates a test scenario with required fields" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { user_id: 123, amount: 1000 }
+      )
+
+      expect(scenario.id).to eq("test_1")
+      expect(scenario.context).to eq({ user_id: 123, amount: 1000 })
+      expect(scenario.expected_decision).to be_nil
+      expect(scenario.expected_confidence).to be_nil
+    end
+
+    it "creates a test scenario with expected results" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_2",
+        context: { user_id: 456 },
+        expected_decision: "approve",
+        expected_confidence: 0.95
+      )
+
+      expect(scenario.expected_decision).to eq("approve")
+      expect(scenario.expected_confidence).to eq(0.95)
+    end
+
+    it "freezes the scenario for immutability" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_3",
+        context: { key: "value" }
+      )
+
+      expect(scenario.frozen?).to be true
+    end
+  end
+
+  describe "#expected_result?" do
+    it "returns true when expected_decision is set" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { key: "value" },
+        expected_decision: "approve"
+      )
+
+      expect(scenario.expected_result?).to be true
+    end
+
+    it "returns false when expected_decision is nil" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { key: "value" }
+      )
+
+      expect(scenario.expected_result?).to be false
+    end
+  end
+
+  describe "#to_h" do
+    it "converts scenario to hash" do
+      scenario = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { user_id: 123 },
+        expected_decision: "approve",
+        expected_confidence: 0.9,
+        metadata: { source: "csv" }
+      )
+
+      hash = scenario.to_h
+
+      expect(hash).to eq({
+                           id: "test_1",
+                           context: { user_id: 123 },
+                           expected_decision: "approve",
+                           expected_confidence: 0.9,
+                           metadata: { source: "csv" }
+                         })
+    end
+  end
+
+  describe "#==" do
+    it "returns true for equal scenarios" do
+      scenario1 = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { user_id: 123 },
+        expected_decision: "approve"
+      )
+
+      scenario2 = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { user_id: 123 },
+        expected_decision: "approve"
+      )
+
+      expect(scenario1).to eq(scenario2)
+    end
+
+    it "returns false for different scenarios" do
+      scenario1 = DecisionAgent::Testing::TestScenario.new(
+        id: "test_1",
+        context: { user_id: 123 }
+      )
+
+      scenario2 = DecisionAgent::Testing::TestScenario.new(
+        id: "test_2",
+        context: { user_id: 123 }
+      )
+
+      expect(scenario1).not_to eq(scenario2)
+    end
+  end
+end

data/spec/versioning/adapter_spec.rb

@@ -0,0 +1,156 @@
+require "spec_helper"
+require "fileutils"
+require "tempfile"
+require "decision_agent/versioning/adapter"
+require "decision_agent/versioning/file_storage_adapter"
+
+RSpec.describe DecisionAgent::Versioning::Adapter do
+  let(:temp_dir) { Dir.mktmpdir }
+  let(:adapter) { DecisionAgent::Versioning::FileStorageAdapter.new(storage_path: temp_dir) }
+  let(:rule_id) { "test_rule_001" }
+  let(:rule_content) do
+    {
+      version: "1.0",
+      ruleset: "test_ruleset",
+      rules: [
+        {
+          id: "rule_1",
+          if: { field: "amount", op: "gt", value: 100 },
+          then: { decision: "approve", weight: 0.8, reason: "High value" }
+        }
+      ]
+    }
+  end
+
+  after do
+    FileUtils.rm_rf(temp_dir)
+  end
+
+  describe "#compare_versions" do
+    it "returns nil when first version doesn't exist" do
+      v2 = adapter.create_version(rule_id: rule_id, content: rule_content)
+
+      comparison = adapter.compare_versions(version_id_1: "nonexistent", version_id_2: v2[:id])
+      expect(comparison).to be_nil
+    end
+
+    it "returns nil when second version doesn't exist" do
+      v1 = adapter.create_version(rule_id: rule_id, content: rule_content)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: "nonexistent")
+      expect(comparison).to be_nil
+    end
+
+    it "calculates differences correctly with added keys" do
+      content1 = { key1: "value1", key2: "value2" }
+      content2 = { key1: "value1", key2: "value2", key3: "value3" }
+
+      v1 = adapter.create_version(rule_id: rule_id, content: content1)
+      v2 = adapter.create_version(rule_id: "rule_2", content: content2)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: v2[:id])
+
+      expect(comparison[:differences][:added]).not_to be_empty
+    end
+
+    it "calculates differences correctly with removed keys" do
+      content1 = { key1: "value1", key2: "value2", key3: "value3" }
+      content2 = { key1: "value1", key2: "value2" }
+
+      v1 = adapter.create_version(rule_id: rule_id, content: content1)
+      v2 = adapter.create_version(rule_id: "rule_2", content: content2)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: v2[:id])
+
+      expect(comparison[:differences][:removed]).not_to be_empty
+    end
+
+    it "calculates differences correctly with changed values" do
+      content1 = { key1: "value1", key2: "value2" }
+      content2 = { key1: "value1", key2: "value2_changed" }
+
+      v1 = adapter.create_version(rule_id: rule_id, content: content1)
+      v2 = adapter.create_version(rule_id: "rule_2", content: content2)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: v2[:id])
+
+      expect(comparison[:differences][:changed]).to have_key(:key2)
+      expect(comparison[:differences][:changed][:key2][:old]).to eq("value2")
+      expect(comparison[:differences][:changed][:key2][:new]).to eq("value2_changed")
+    end
+
+    it "does not include nil values in changed differences" do
+      content1 = { key1: "value1", key2: "value2" }
+      content2 = { key1: "value1", key2: nil }
+
+      v1 = adapter.create_version(rule_id: rule_id, content: content1)
+      v2 = adapter.create_version(rule_id: "rule_2", content: content2)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: v2[:id])
+
+      # key2 should not be in changed since new value is nil
+      expect(comparison[:differences][:changed]).not_to have_key(:key2)
+    end
+
+    it "returns identical versions with empty differences" do
+      v1 = adapter.create_version(rule_id: rule_id, content: rule_content)
+      v2 = adapter.create_version(rule_id: "rule_2", content: rule_content)
+
+      comparison = adapter.compare_versions(version_id_1: v1[:id], version_id_2: v2[:id])
+
+      expect(comparison[:differences][:added]).to be_empty
+      expect(comparison[:differences][:removed]).to be_empty
+      expect(comparison[:differences][:changed]).to be_empty
+    end
+  end
+
+  describe "abstract methods" do
+    # Test that abstract methods raise NotImplementedError when called on base class
+    # We use a minimal test adapter class for this
+    let(:abstract_adapter) do
+      Class.new(DecisionAgent::Versioning::Adapter).new
+    end
+
+    it "raises NotImplementedError for create_version" do
+      expect do
+        abstract_adapter.create_version(rule_id: "test", content: {})
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for list_versions" do
+      expect do
+        abstract_adapter.list_versions(rule_id: "test")
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for get_version" do
+      expect do
+        abstract_adapter.get_version(version_id: "test")
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for get_version_by_number" do
+      expect do
+        abstract_adapter.get_version_by_number(rule_id: "test", version_number: 1)
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for get_active_version" do
+      expect do
+        abstract_adapter.get_active_version(rule_id: "test")
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for activate_version" do
+      expect do
+        abstract_adapter.activate_version(version_id: "test")
+      end.to raise_error(NotImplementedError)
+    end
+
+    it "raises NotImplementedError for delete_version" do
+      expect do
+        abstract_adapter.delete_version(version_id: "test")
+      end.to raise_error(NotImplementedError)
+    end
+  end
+end