decision_agent 0.1.3 → 0.1.4

data/spec/ab_testing/ab_test_manager_spec.rb

@@ -0,0 +1,330 @@
+require "spec_helper"
+require "decision_agent/ab_testing/ab_test_manager"
+require "decision_agent/ab_testing/storage/memory_adapter"
+require "decision_agent/versioning/file_storage_adapter"
+
+RSpec.describe DecisionAgent::ABTesting::ABTestManager do
+  let(:version_manager) do
+    DecisionAgent::Versioning::VersionManager.new(
+      adapter: DecisionAgent::Versioning::FileStorageAdapter.new(storage_path: "/tmp/spec_ab_test_versions")
+    )
+  end
+
+  let(:storage_adapter) { DecisionAgent::ABTesting::Storage::MemoryAdapter.new }
+  let(:manager) { described_class.new(storage_adapter: storage_adapter, version_manager: version_manager) }
+
+  before do
+    # Create test versions
+    @champion = version_manager.save_version(
+      rule_id: "test_rule",
+      rule_content: { rules: [{ decision: "approve", weight: 1.0 }] },
+      created_by: "spec"
+    )
+
+    @challenger = version_manager.save_version(
+      rule_id: "test_rule",
+      rule_content: { rules: [{ decision: "reject", weight: 1.0 }] },
+      created_by: "spec"
+    )
+  end
+
+  after do
+    FileUtils.rm_rf("/tmp/spec_ab_test_versions")
+  end
+
+  describe "#create_test" do
+    it "creates a new A/B test" do
+      test = manager.create_test(
+        name: "Test A vs B",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id]
+      )
+
+      expect(test).to be_a(DecisionAgent::ABTesting::ABTest)
+      expect(test.name).to eq("Test A vs B")
+      expect(test.id).not_to be_nil
+    end
+
+    it "validates that champion version exists" do
+      expect do
+        manager.create_test(
+          name: "Test",
+          champion_version_id: "nonexistent",
+          challenger_version_id: @challenger[:id]
+        )
+      end.to raise_error(DecisionAgent::ABTesting::VersionNotFoundError, /Champion/)
+    end
+
+    it "validates that challenger version exists" do
+      expect do
+        manager.create_test(
+          name: "Test",
+          champion_version_id: @champion[:id],
+          challenger_version_id: "nonexistent"
+        )
+      end.to raise_error(DecisionAgent::ABTesting::VersionNotFoundError, /Challenger/)
+    end
+
+    it "accepts custom traffic split" do
+      test = manager.create_test(
+        name: "Custom Split",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        traffic_split: { champion: 70, challenger: 30 }
+      )
+
+      expect(test.traffic_split).to eq({ champion: 70, challenger: 30 })
+    end
+  end
+
+  describe "#get_test" do
+    it "retrieves a test by ID" do
+      created_test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id]
+      )
+
+      retrieved_test = manager.get_test(created_test.id)
+
+      expect(retrieved_test).not_to be_nil
+      expect(retrieved_test.id).to eq(created_test.id)
+      expect(retrieved_test.name).to eq("Test")
+    end
+
+    it "returns nil for nonexistent test" do
+      test = manager.get_test(99_999)
+      expect(test).to be_nil
+    end
+  end
+
+  describe "#assign_variant" do
+    let(:test) do
+      manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+    end
+
+    before do
+      manager.start_test(test.id)
+    end
+
+    it "assigns a variant and returns assignment details" do
+      assignment = manager.assign_variant(test_id: test.id, user_id: "user_123")
+
+      expect(assignment[:test_id]).to eq(test.id)
+      expect(%i[champion challenger]).to include(assignment[:variant])
+      expect([@champion[:id], @challenger[:id]]).to include(assignment[:version_id])
+      expect(assignment[:assignment_id]).not_to be_nil
+    end
+
+    it "assigns same variant to same user" do
+      user_id = "consistent_user"
+      assignment1 = manager.assign_variant(test_id: test.id, user_id: user_id)
+      assignment2 = manager.assign_variant(test_id: test.id, user_id: user_id)
+
+      expect(assignment1[:variant]).to eq(assignment2[:variant])
+    end
+
+    it "raises error for nonexistent test" do
+      expect do
+        manager.assign_variant(test_id: 99_999)
+      end.to raise_error(DecisionAgent::ABTesting::TestNotFoundError)
+    end
+  end
+
+  describe "#record_decision" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "records decision result for an assignment" do
+      assignment = manager.assign_variant(test_id: test.id)
+
+      expect do
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.95
+        )
+      end.not_to raise_error
+    end
+  end
+
+  describe "#get_results" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "returns results with statistics" do
+      # Create some assignments and record decisions
+      10.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.8 + (rand * 0.2)
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      expect(results[:test]).to be_a(Hash)
+      expect(results[:champion]).to be_a(Hash)
+      expect(results[:challenger]).to be_a(Hash)
+      expect(results[:comparison]).to be_a(Hash)
+      expect(results[:total_assignments]).to eq(10)
+    end
+
+    it "handles tests with no assignments" do
+      results = manager.get_results(test.id)
+
+      expect(results[:total_assignments]).to eq(0)
+      expect(results[:champion][:decisions_recorded]).to eq(0)
+      expect(results[:challenger][:decisions_recorded]).to eq(0)
+    end
+  end
+
+  describe "#active_tests" do
+    it "returns only running tests" do
+      test1 = manager.create_test(
+        name: "Running Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test1.id)
+
+      manager.create_test(
+        name: "Scheduled Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+
+      active = manager.active_tests
+
+      expect(active.size).to eq(1)
+      expect(active.first.id).to eq(test1.id)
+    end
+
+    it "caches active tests for performance" do
+      test = manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+
+      # First call
+      manager.active_tests
+
+      # Expect storage adapter not to be called again (cached)
+      expect(storage_adapter).not_to receive(:list_tests)
+      manager.active_tests
+    end
+  end
+
+  describe "test lifecycle" do
+    let(:test) do
+      manager.create_test(
+        name: "Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        start_date: Time.now.utc + 3600
+      )
+    end
+
+    it "starts a scheduled test" do
+      manager.start_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("running")
+    end
+
+    it "completes a running test" do
+      manager.start_test(test.id)
+      manager.complete_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("completed")
+    end
+
+    it "cancels a test" do
+      manager.cancel_test(test.id)
+      updated_test = manager.get_test(test.id)
+
+      expect(updated_test.status).to eq("cancelled")
+    end
+  end
+
+  describe "statistical analysis" do
+    let(:test) do
+      test = manager.create_test(
+        name: "Statistical Test",
+        champion_version_id: @champion[:id],
+        challenger_version_id: @challenger[:id],
+        traffic_split: { champion: 50, challenger: 50 },
+        start_date: Time.now.utc + 3600
+      )
+      manager.start_test(test.id)
+      test
+    end
+
+    it "calculates improvement percentage" do
+      # Create assignments with different confidence levels
+      50.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+
+        # Champion: avg 0.7, Challenger: avg 0.9
+        confidence = assignment[:variant] == :champion ? 0.7 : 0.9
+
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: confidence
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      # Challenger should have higher avg confidence (0.9 vs 0.7)
+      expect(results[:comparison][:improvement_percentage]).to be > 0
+      expect(%w[champion challenger inconclusive]).to include(results[:comparison][:winner])
+    end
+
+    it "indicates insufficient data when sample is too small" do
+      # Create only a few assignments
+      5.times do |i|
+        assignment = manager.assign_variant(test_id: test.id, user_id: "user_#{i}")
+        manager.record_decision(
+          assignment_id: assignment[:assignment_id],
+          decision: "approve",
+          confidence: 0.8
+        )
+      end
+
+      results = manager.get_results(test.id)
+
+      expect(results[:comparison][:statistical_significance]).to eq("not_significant")
+    end
+  end
+end

data/spec/ab_testing/ab_test_spec.rb

@@ -0,0 +1,270 @@
+require "spec_helper"
+require "decision_agent/ab_testing/ab_test"
+
+RSpec.describe DecisionAgent::ABTesting::ABTest do
+  describe "#initialize" do
+    it "creates a valid A/B test with default values" do
+      test = described_class.new(
+        name: "Test A vs B",
+        champion_version_id: "v1",
+        challenger_version_id: "v2"
+      )
+
+      expect(test.name).to eq("Test A vs B")
+      expect(test.champion_version_id).to eq("v1")
+      expect(test.challenger_version_id).to eq("v2")
+      expect(test.traffic_split).to eq({ champion: 90, challenger: 10 })
+      expect(test.status).to eq("scheduled")
+    end
+
+    it "accepts custom traffic split as hash" do
+      test = described_class.new(
+        name: "Custom Split",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        traffic_split: { champion: 70, challenger: 30 }
+      )
+
+      expect(test.traffic_split).to eq({ champion: 70, challenger: 30 })
+    end
+
+    it "accepts custom traffic split as array" do
+      test = described_class.new(
+        name: "Array Split",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        traffic_split: [80, 20]
+      )
+
+      expect(test.traffic_split).to eq({ champion: 80, challenger: 20 })
+    end
+
+    it "raises error if traffic split doesn't sum to 100" do
+      expect do
+        described_class.new(
+          name: "Bad Split",
+          champion_version_id: "v1",
+          challenger_version_id: "v2",
+          traffic_split: { champion: 60, challenger: 30 }
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /must sum to 100/)
+    end
+
+    it "raises error if champion and challenger are the same" do
+      expect do
+        described_class.new(
+          name: "Same Versions",
+          champion_version_id: "v1",
+          challenger_version_id: "v1"
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /must be different/)
+    end
+
+    it "raises error if name is empty" do
+      expect do
+        described_class.new(
+          name: "",
+          champion_version_id: "v1",
+          challenger_version_id: "v2"
+        )
+      end.to raise_error(DecisionAgent::ValidationError, /name is required/)
+    end
+  end
+
+  describe "#assign_variant" do
+    let(:test) do
+      described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        traffic_split: { champion: 90, challenger: 10 },
+        status: "running",
+        id: 123
+      )
+    end
+
+    it "assigns champion or challenger based on traffic split" do
+      assignments = 1000.times.map { test.assign_variant }
+      champion_count = assignments.count { |v| v == :champion }
+      challenger_count = assignments.count { |v| v == :challenger }
+
+      # With 90/10 split, expect roughly 900/100
+      expect(champion_count).to be_between(850, 950)
+      expect(challenger_count).to be_between(50, 150)
+    end
+
+    it "assigns same variant to same user consistently" do
+      user_id = "user_123"
+      variants = 10.times.map { test.assign_variant(user_id: user_id) }
+
+      expect(variants.uniq.size).to eq(1)
+    end
+
+    it "assigns different users to different variants based on split" do
+      assignments = 1000.times.map { |i| test.assign_variant(user_id: "user_#{i}") }
+      champion_count = assignments.count { |v| v == :champion }
+      challenger_count = assignments.count { |v| v == :challenger }
+
+      expect(champion_count).to be_between(850, 950)
+      expect(challenger_count).to be_between(50, 150)
+    end
+
+    it "raises error if test is not running" do
+      test = described_class.new(
+        name: "Not Running",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "completed"
+      )
+
+      expect do
+        test.assign_variant
+      end.to raise_error(DecisionAgent::ABTesting::TestNotRunningError)
+    end
+  end
+
+  describe "#version_for_variant" do
+    let(:test) do
+      described_class.new(
+        name: "Test",
+        champion_version_id: "champion_v1",
+        challenger_version_id: "challenger_v2"
+      )
+    end
+
+    it "returns champion version ID for :champion variant" do
+      expect(test.version_for_variant(:champion)).to eq("champion_v1")
+    end
+
+    it "returns challenger version ID for :challenger variant" do
+      expect(test.version_for_variant(:challenger)).to eq("challenger_v2")
+    end
+
+    it "raises error for invalid variant" do
+      expect do
+        test.version_for_variant(:invalid)
+      end.to raise_error(ArgumentError, /Invalid variant/)
+    end
+  end
+
+  describe "#running?" do
+    it "returns true when status is running and within date range" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "running",
+        start_date: Time.now.utc - 3600,
+        end_date: Time.now.utc + 3600
+      )
+
+      expect(test.running?).to be true
+    end
+
+    it "returns false when status is not running" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "completed"
+      )
+
+      expect(test.running?).to be false
+    end
+
+    it "returns false when start date is in future" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "running",
+        start_date: Time.now.utc + 3600
+      )
+
+      expect(test.running?).to be false
+    end
+
+    it "returns false when end date has passed" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "running",
+        start_date: Time.now.utc - 7200,
+        end_date: Time.now.utc - 3600
+      )
+
+      expect(test.running?).to be false
+    end
+  end
+
+  describe "status transitions" do
+    it "can start a scheduled test" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "scheduled"
+      )
+
+      expect { test.start! }.not_to raise_error
+      expect(test.status).to eq("running")
+    end
+
+    it "can complete a running test" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "running"
+      )
+
+      expect { test.complete! }.not_to raise_error
+      expect(test.status).to eq("completed")
+    end
+
+    it "can cancel a test" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "running"
+      )
+
+      expect { test.cancel! }.not_to raise_error
+      expect(test.status).to eq("cancelled")
+    end
+
+    it "raises error when trying invalid status transition" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        status: "completed"
+      )
+
+      expect do
+        test.start!
+      end.to raise_error(DecisionAgent::ABTesting::InvalidStatusTransitionError)
+    end
+  end
+
+  describe "#to_h" do
+    it "returns hash representation" do
+      test = described_class.new(
+        name: "Test",
+        champion_version_id: "v1",
+        challenger_version_id: "v2",
+        id: 123
+      )
+
+      hash = test.to_h
+
+      expect(hash[:id]).to eq(123)
+      expect(hash[:name]).to eq("Test")
+      expect(hash[:champion_version_id]).to eq("v1")
+      expect(hash[:challenger_version_id]).to eq("v2")
+      expect(hash[:status]).to eq("scheduled")
+    end
+  end
+end