ab-split 1.0.0

data/spec/algorithms/block_randomization_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+describe Split::Algorithms::BlockRandomization do
+
+  let(:experiment) { Split::Experiment.new 'experiment' }
+  let(:alternative_A) { Split::Alternative.new 'A', 'experiment' }
+  let(:alternative_B) { Split::Alternative.new 'B', 'experiment' }
+  let(:alternative_C) { Split::Alternative.new 'C', 'experiment' }
+
+  before :each do
+    allow(experiment).to receive(:alternatives) { [alternative_A, alternative_B, alternative_C] }
+  end
+
+  it "should return an alternative" do
+    expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment).class).to eq(Split::Alternative)
+  end
+
+  it "should always return the minimum participation option" do
+    allow(alternative_A).to receive(:participant_count) { 1 }
+    allow(alternative_B).to receive(:participant_count) { 1 }
+    allow(alternative_C).to receive(:participant_count) { 0 }
+    expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment)).to eq(alternative_C)
+  end
+
+  it "should return one of the minimum participation options when multiple" do
+    allow(alternative_A).to receive(:participant_count) { 0 }
+    allow(alternative_B).to receive(:participant_count) { 0 }
+    allow(alternative_C).to receive(:participant_count) { 0 }
+    alternative = Split::Algorithms::BlockRandomization.choose_alternative(experiment)
+    expect([alternative_A, alternative_B, alternative_C].include?(alternative)).to be(true)
+  end
+end

data/spec/algorithms/weighted_sample_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+describe Split::Algorithms::WeightedSample do
+  it "should return an alternative" do
+    experiment = Split::ExperimentCatalog.find_or_create('link_color', {'blue' => 100}, {'red' => 0 })
+    expect(Split::Algorithms::WeightedSample.choose_alternative(experiment).class).to eq(Split::Alternative)
+  end
+
+  it "should always return a heavily weighted option" do
+    experiment = Split::ExperimentCatalog.find_or_create('link_color', {'blue' => 100}, {'red' => 0 })
+    expect(Split::Algorithms::WeightedSample.choose_alternative(experiment).name).to eq('blue')
+  end
+
+  it "should return one of the results" do
+    experiment = Split::ExperimentCatalog.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    expect(['red', 'blue']).to include Split::Algorithms::WeightedSample.choose_alternative(experiment).name
+  end
+end

data/spec/algorithms/whiplash_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+describe Split::Algorithms::Whiplash do
+
+  it "should return an algorithm" do
+    experiment = Split::ExperimentCatalog.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    expect(Split::Algorithms::Whiplash.choose_alternative(experiment).class).to eq(Split::Alternative)
+  end
+
+  it "should return one of the results" do
+    experiment = Split::ExperimentCatalog.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    expect(['red', 'blue']).to include Split::Algorithms::Whiplash.choose_alternative(experiment).name
+  end
+
+  it "should guess floats" do
+    expect(Split::Algorithms::Whiplash.send(:arm_guess, 0, 0).class).to eq(Float)
+    expect(Split::Algorithms::Whiplash.send(:arm_guess, 1, 0).class).to eq(Float)
+    expect(Split::Algorithms::Whiplash.send(:arm_guess, 2, 1).class).to eq(Float)
+    expect(Split::Algorithms::Whiplash.send(:arm_guess, 1000, 5).class).to eq(Float)
+    expect(Split::Algorithms::Whiplash.send(:arm_guess, 10, -2).class).to eq(Float)
+  end
+
+end

data/spec/alternative_spec.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+require 'spec_helper'
+require 'split/alternative'
+
+describe Split::Alternative do
+
+  let(:alternative) {
+    Split::Alternative.new('Basket', 'basket_text')
+  }
+
+  let(:alternative2) {
+    Split::Alternative.new('Cart', 'basket_text')
+  }
+
+  let!(:experiment) {
+    Split::ExperimentCatalog.find_or_create({"basket_text" => ["purchase", "refund"]}, "Basket", "Cart")
+  }
+
+  let(:goal1) { "purchase" }
+  let(:goal2) { "refund" }
+
+  it "should have goals" do
+    expect(alternative.goals).to eq(["purchase", "refund"])
+  end
+
+  it "should have and only return the name" do
+    expect(alternative.name).to eq('Basket')
+  end
+
+  describe 'weights' do
+    it "should set the weights" do
+      experiment = Split::Experiment.new('basket_text', :alternatives => [{'Basket' => 0.6}, {"Cart" => 0.4}])
+      first = experiment.alternatives[0]
+      expect(first.name).to eq('Basket')
+      expect(first.weight).to eq(0.6)
+
+      second = experiment.alternatives[1]
+      expect(second.name).to eq('Cart')
+      expect(second.weight).to eq(0.4)
+    end
+
+    it "accepts probability on alternatives" do
+      Split.configuration.experiments = {
+        :my_experiment => {
+          :alternatives => [
+            { :name => "control_opt", :percent => 67 },
+            { :name => "second_opt", :percent => 10 },
+            { :name => "third_opt", :percent => 23 },
+          ]
+        }
+      }
+      experiment = Split::Experiment.new(:my_experiment)
+      first = experiment.alternatives[0]
+      expect(first.name).to eq('control_opt')
+      expect(first.weight).to eq(0.67)
+
+      second = experiment.alternatives[1]
+      expect(second.name).to eq('second_opt')
+      expect(second.weight).to eq(0.1)
+    end
+
+    it "accepts probability on some alternatives" do
+      Split.configuration.experiments = {
+        :my_experiment => {
+          :alternatives => [
+            { :name => "control_opt", :percent => 34 },
+            "second_opt",
+            { :name => "third_opt", :percent => 23 },
+            "fourth_opt",
+          ],
+        }
+      }
+      experiment = Split::Experiment.new(:my_experiment)
+      alts = experiment.alternatives
+      [
+        ["control_opt", 0.34],
+        ["second_opt", 0.215],
+        ["third_opt", 0.23],
+        ["fourth_opt", 0.215]
+      ].each do |h|
+        name, weight = h
+        alt = alts.shift
+        expect(alt.name).to eq(name)
+        expect(alt.weight).to eq(weight)
+      end
+    end
+    #
+    it "allows name param without probability" do
+      Split.configuration.experiments = {
+        :my_experiment => {
+          :alternatives => [
+            { :name => "control_opt" },
+            "second_opt",
+            { :name => "third_opt", :percent => 64 },
+          ],
+        }
+      }
+      experiment = Split::Experiment.new(:my_experiment)
+      alts = experiment.alternatives
+      [
+        ["control_opt", 0.18],
+        ["second_opt", 0.18],
+        ["third_opt", 0.64],
+      ].each do |h|
+        name, weight = h
+        alt = alts.shift
+        expect(alt.name).to eq(name)
+        expect(alt.weight).to eq(weight)
+      end
+    end
+  end
+
+  it "should have a default participation count of 0" do
+    expect(alternative.participant_count).to eq(0)
+  end
+
+  it "should have a default completed count of 0 for each goal" do
+    expect(alternative.completed_count).to eq(0)
+    expect(alternative.completed_count(goal1)).to eq(0)
+    expect(alternative.completed_count(goal2)).to eq(0)
+  end
+
+  it "should belong to an experiment" do
+    expect(alternative.experiment.name).to eq(experiment.name)
+  end
+
+  it "should save to redis" do
+    alternative.save
+    expect(Split.redis.exists('basket_text:Basket')).to be true
+  end
+
+  it "should increment participation count" do
+    old_participant_count = alternative.participant_count
+    alternative.increment_participation
+    expect(alternative.participant_count).to eq(old_participant_count+1)
+  end
+
+  it "should increment completed count for each goal" do
+    old_default_completed_count = alternative.completed_count
+    old_completed_count_for_goal1 = alternative.completed_count(goal1)
+    old_completed_count_for_goal2 = alternative.completed_count(goal2)
+
+    alternative.increment_completion
+    alternative.increment_completion(goal1)
+    alternative.increment_completion(goal2)
+
+    expect(alternative.completed_count).to eq(old_default_completed_count+1)
+    expect(alternative.completed_count(goal1)).to eq(old_completed_count_for_goal1+1)
+    expect(alternative.completed_count(goal2)).to eq(old_completed_count_for_goal2+1)
+  end
+
+  it "can be reset" do
+    alternative.participant_count = 10
+    alternative.set_completed_count(4, goal1)
+    alternative.set_completed_count(5, goal2)
+    alternative.set_completed_count(6)
+    alternative.reset
+    expect(alternative.participant_count).to eq(0)
+    expect(alternative.completed_count(goal1)).to eq(0)
+    expect(alternative.completed_count(goal2)).to eq(0)
+    expect(alternative.completed_count).to eq(0)
+  end
+
+  it "should know if it is the control of an experiment" do
+    expect(alternative.control?).to be_truthy
+    expect(alternative2.control?).to be_falsey
+  end
+
+  describe 'unfinished_count' do
+    it "should be difference between participant and completed counts" do
+      alternative.increment_participation
+      expect(alternative.unfinished_count).to eq(alternative.participant_count)
+    end
+
+    it "should return the correct unfinished_count" do
+      alternative.participant_count = 10
+      alternative.set_completed_count(4, goal1)
+      alternative.set_completed_count(3, goal2)
+      alternative.set_completed_count(2)
+
+      expect(alternative.unfinished_count).to eq(1)
+    end
+  end
+
+  describe 'conversion rate' do
+    it "should be 0 if there are no conversions" do
+      expect(alternative.completed_count).to eq(0)
+      expect(alternative.conversion_rate).to eq(0)
+    end
+
+    it "calculate conversion rate" do
+      expect(alternative).to receive(:participant_count).exactly(6).times.and_return(10)
+      expect(alternative).to receive(:completed_count).and_return(4)
+      expect(alternative.conversion_rate).to eq(0.4)
+
+      expect(alternative).to receive(:completed_count).with(goal1).and_return(5)
+      expect(alternative.conversion_rate(goal1)).to eq(0.5)
+
+      expect(alternative).to receive(:completed_count).with(goal2).and_return(6)
+      expect(alternative.conversion_rate(goal2)).to eq(0.6)
+    end
+  end
+
+  describe "probability winner" do
+    before do
+      experiment.calc_winning_alternatives
+    end
+
+    it "should have a probability of being the winning alternative (p_winner)" do
+      expect(alternative.p_winner).not_to be_nil
+    end
+
+    it "should have a probability of being the winner for each goal" do
+      expect(alternative.p_winner(goal1)).not_to be_nil
+    end
+
+    it "should be possible to set the p_winner" do
+      alternative.set_p_winner(0.5)
+      expect(alternative.p_winner).to eq(0.5)
+    end
+
+    it "should be possible to set the p_winner for each goal" do
+      alternative.set_p_winner(0.5, goal1)
+      expect(alternative.p_winner(goal1)).to eq(0.5)
+    end
+  end
+
+  describe 'z score' do
+
+    it "should return an error string when the control has 0 people" do
+      expect(alternative2.z_score).to eq("Needs 30+ participants.")
+      expect(alternative2.z_score(goal1)).to eq("Needs 30+ participants.")
+      expect(alternative2.z_score(goal2)).to eq("Needs 30+ participants.")
+    end
+
+    it "should return an error string when the data is skewed or incomplete as per the np > 5 test" do
+      control = experiment.control
+      control.participant_count = 100
+      control.set_completed_count(50)
+
+      alternative2.participant_count = 50
+      alternative2.set_completed_count(1)
+
+      expect(alternative2.z_score).to eq("Needs 5+ conversions.")
+    end
+
+    it "should return a float for a z_score given proper data" do
+      control = experiment.control
+      control.participant_count = 120
+      control.set_completed_count(20)
+
+      alternative2.participant_count = 100
+      alternative2.set_completed_count(25)
+
+      expect(alternative2.z_score).to be_kind_of(Float)
+      expect(alternative2.z_score).to_not eq(0)
+    end
+
+    it "should correctly calculate a z_score given proper data" do
+      control = experiment.control
+      control.participant_count = 126
+      control.set_completed_count(89)
+
+      alternative2.participant_count = 142
+      alternative2.set_completed_count(119)
+
+      expect(alternative2.z_score.round(2)).to eq(2.58)
+    end
+
+    it "should be N/A for the control" do
+      control = experiment.control
+      expect(control.z_score).to eq('N/A')
+      expect(control.z_score(goal1)).to eq('N/A')
+      expect(control.z_score(goal2)).to eq('N/A')
+    end
+
+    it "should not blow up for Conversion Rates > 1" do
+      control = experiment.control
+      control.participant_count = 3474
+      control.set_completed_count(4244)
+
+      alternative2.participant_count = 3434
+      alternative2.set_completed_count(4358)
+
+      expect { control.z_score }.not_to raise_error
+      expect { alternative2.z_score }.not_to raise_error
+    end
+  end
+
+  describe "extra_info" do
+    it "reads saved value of recorded_info in redis" do
+      saved_recorded_info = {"key_1" => 1, "key_2" => "2"}
+      Split.redis.hset "#{alternative.experiment_name}:#{alternative.name}", 'recorded_info', saved_recorded_info.to_json
+      extra_info = alternative.extra_info
+
+      expect(extra_info).to eql(saved_recorded_info)
+    end
+  end
+
+  describe "record_extra_info" do
+    it "saves key" do
+      alternative.record_extra_info("signup", 1)
+      expect(alternative.extra_info["signup"]).to eql(1)
+    end
+
+    it "adds value to saved key's value second argument is number" do
+      alternative.record_extra_info("signup", 1)
+      alternative.record_extra_info("signup", 2)
+      expect(alternative.extra_info["signup"]).to eql(3)
+    end
+
+    it "sets saved's key value to the second argument if it's a string" do
+      alternative.record_extra_info("signup", "Value 1")
+      expect(alternative.extra_info["signup"]).to eql("Value 1")
+
+      alternative.record_extra_info("signup", "Value 2")
+      expect(alternative.extra_info["signup"]).to eql("Value 2")
+    end
+  end
+end

data/spec/combined_experiments_helper_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+require 'spec_helper'
+require 'split/combined_experiments_helper'
+
+describe Split::CombinedExperimentsHelper do
+  include Split::CombinedExperimentsHelper
+
+  describe 'ab_combined_test' do
+    let!(:config_enabled) { true }
+    let!(:combined_experiments) { [:exp_1_click, :exp_1_scroll ]}
+    let!(:allow_multiple_experiments) { true }
+
+    before do
+      Split.configuration.experiments = {
+        :combined_exp_1 => {
+          :alternatives => [ {"control"=> 0.5}, {"test-alt"=> 0.5} ],
+          :metric => :my_metric,
+          :combined_experiments => combined_experiments
+        }
+      }
+      Split.configuration.enabled = config_enabled
+      Split.configuration.allow_multiple_experiments = allow_multiple_experiments
+    end
+
+    context 'without config enabled' do
+      let!(:config_enabled) { false }
+
+      it "raises an error" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError )
+      end
+    end
+
+    context 'multiple experiments disabled' do
+      let!(:allow_multiple_experiments) { false }
+
+      it "raises an error if multiple experiments is disabled" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError)
+      end
+    end
+
+    context 'without combined experiments' do
+      let!(:combined_experiments) { nil }
+
+      it "raises an error" do
+        expect(lambda { ab_combined_test :combined_exp_1 }).to raise_error(Split::InvalidExperimentsFormatError )
+      end
+    end
+
+    it "uses same alternative for all sub experiments and returns the alternative" do
+      allow(self).to receive(:get_alternative) { "test-alt" }
+      expect(self).to receive(:ab_test).with(:exp_1_click, {"control"=>0.5}, {"test-alt"=>0.5}) { "test-alt" }
+      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"control" => 0, "test-alt" => 1}])
+
+      expect(ab_combined_test('combined_exp_1')).to eq('test-alt')
+    end
+  end
+end