split 0.5.0 → 0.6.0

data/spec/helper_spec.rb

@@ -5,6 +5,10 @@ require 'spec_helper'
 describe Split::Helper do
   include Split::Helper
 
+  let(:experiment) {
+    Split::Experiment.find_or_create('link_color', 'blue', 'red')
+  }
+
   describe "ab_test" do
 
     it "should not raise an error when passed strings for alternatives" do
@@ -19,13 +23,20 @@ describe Split::Helper do
       lambda { ab_test('xyz', :a, :b, :c) }.should raise_error(ArgumentError)
     end
 
+    it "should not raise error when passed an array for goals" do
+      lambda { ab_test({'link_color' => ["purchase", "refund"]}, 'blue', 'red') }.should_not raise_error
+    end
+
+    it "should not raise error when passed just one goal" do
+      lambda { ab_test({'link_color' => "purchase"}, 'blue', 'red') }.should_not raise_error
+    end
+
     it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
       ab_test('link_color', 'blue', 'red')
       ['red', 'blue'].should include(ab_user['link_color'])
     end
 
     it "should increment the participation counter after assignment to a new user" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
 
       previous_red_count = Split::Alternative.new('red', 'link_color').participant_count
       previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
@@ -38,22 +49,37 @@ describe Split::Helper do
       (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count + 1)
     end
 
+    it 'should not increment the counter for an experiment that the user is not participating in' do
+      ab_test('link_color', 'blue', 'red')
+      e = Split::Experiment.find_or_create('button_size', 'small', 'big')
+      lambda {
+        # User shouldn't participate in this second experiment
+        ab_test('button_size', 'small', 'big')
+      }.should_not change { e.participant_count }
+    end
+
+    it 'should not increment the counter for an ended experiment' do
+      e = Split::Experiment.find_or_create('button_size', 'small', 'big')
+      e.winner = 'small'
+      lambda {
+        a = ab_test('button_size', 'small', 'big')
+        a.should eq('small')
+      }.should_not change { e.participant_count }
+    end
+
     it "should return the given alternative for an existing user" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative = ab_test('link_color', 'blue', 'red')
       repeat_alternative = ab_test('link_color', 'blue', 'red')
       alternative.should eql repeat_alternative
     end
 
     it 'should always return the winner if one is present' do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       experiment.winner = "orange"
 
       ab_test('link_color', 'blue', 'red').should == 'orange'
     end
 
     it "should allow the alternative to be force by passing it in the params" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       @params = {'link_color' => 'blue'}
       alternative = ab_test('link_color', 'blue', 'red')
       alternative.should eql('blue')
@@ -80,7 +106,7 @@ describe Split::Helper do
     it "should allow alternative weighting interface as a single hash" do
       ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2)
       experiment = Split::Experiment.find('link_color')
-      experiment.alternative_names.should eql(['blue', 'red'])
+      experiment.alternatives.map(&:name).should eql(['blue', 'red'])
       # TODO: persist alternative weights
       # experiment.alternatives.collect{|a| a.weight}.should == [0.01, 0.2]
     end
@@ -107,7 +133,6 @@ describe Split::Helper do
     end
 
     it "should not over-write a finished key when an experiment is on a later version" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       experiment.increment_version
       ab_user = { experiment.key => 'blue', experiment.finished_key => true }
       finshed_session = ab_user.dup
@@ -132,16 +157,38 @@ describe Split::Helper do
     end
 
     it "should set experiment's finished key if reset is false" do
-      finished(@experiment_name, :reset => false)
+      finished(@experiment_name, {:reset => false})
       ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
     end
 
     it 'should not increment the counter if reset is false and the experiment has been already finished' do
-      2.times { finished(@experiment_name, :reset => false) }
+      2.times { finished(@experiment_name, {:reset => false}) }
       new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
       new_completion_count.should eql(@previous_completion_count + 1)
     end
 
+    it 'should not increment the counter for an experiment that the user is not participating in' do
+      ab_test('button_size', 'small', 'big')
+
+      # So, user should be participating in the link_color experiment and
+      # receive the control for button_size. As the user is not participating in
+      # the button size experiment, finishing it should not increase the
+      # completion count for that alternative.
+      lambda {
+        finished('button_size')
+      }.should_not change { Split::Alternative.new('small', 'button_size').completed_count }
+    end
+
+    it 'should not increment the counter for an ended experiment' do
+      e = Split::Experiment.find_or_create('button_size', 'small', 'big')
+      e.winner = 'small'
+      a = ab_test('button_size', 'small', 'big')
+      a.should eq('small')
+      lambda {
+        finished('button_size')
+      }.should_not change { Split::Alternative.new(a, 'button_size').completed_count }
+    end
+
     it "should clear out the user's participation from their session" do
       ab_user.should eql(@experiment.key => @alternative_name)
       finished(@experiment_name)
@@ -150,7 +197,7 @@ describe Split::Helper do
 
     it "should not clear out the users session if reset is false" do
       ab_user.should eql(@experiment.key => @alternative_name)
-      finished(@experiment_name, :reset => false)
+      finished(@experiment_name, {:reset => false})
       ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
     end
 
@@ -180,89 +227,107 @@ describe Split::Helper do
       doing_other_tests?(@experiment.key).should be false
     end
 
-    it "passes reset option from config" do
+  end
+
+  context "finished with config" do
+    it "passes reset option" do
       Split.configuration.experiments = {
-        @experiment_name => {
-          :alternatives => @alternatives,
+        :my_experiment => {
+          :alternatives => ["one", "two"],
           :resettable => false,
         }
       }
-      finished @experiment_name
-      ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+      alternative = ab_test(:my_experiment)
+      experiment = Split::Experiment.find :my_experiment
+
+      finished :my_experiment
+      ab_user.should eql(experiment.key => alternative, experiment.finished_key => true)
     end
+  end
 
-    context "with metric name" do
-      before { Split.configuration.experiments = {} }
-      before { Split::Alternative.stub(:new).and_call_original }
-
-      def should_finish_experiment(experiment_name, should_finish=true)
-        alts = Split.configuration.experiments[experiment_name][:alternatives]
-        experiment = Split::Experiment.find_or_create(experiment_name, *alts)
-        alt_name = ab_user[experiment.key] = alts.first
-        alt = mock('alternative')
-        alt.stub(:name).and_return(alt_name)
-        Split::Alternative.stub(:new).with(alt_name, experiment_name).and_return(alt)
-        if should_finish
-          alt.should_receive(:increment_completion)
-        else
-          alt.should_not_receive(:increment_completion)
-        end
+  context "finished with metric name" do
+    before { Split.configuration.experiments = {} }
+    before { Split::Alternative.stub(:new).and_call_original }
+
+    def should_finish_experiment(experiment_name, should_finish=true)
+      alts = Split.configuration.experiments[experiment_name][:alternatives]
+      experiment = Split::Experiment.find_or_create(experiment_name, *alts)
+      alt_name = ab_user[experiment.key] = alts.first
+      alt = mock('alternative')
+      alt.stub(:name).and_return(alt_name)
+      Split::Alternative.stub(:new).with(alt_name, experiment_name.to_s).and_return(alt)
+      if should_finish
+        alt.should_receive(:increment_completion)
+      else
+        alt.should_not_receive(:increment_completion)
       end
+    end
 
-      it "completes the test" do
-        Split.configuration.experiments[:my_experiment] = {
-          :alternatives => [ "control_opt", "other_opt" ],
-          :metric => :my_metric
-        }
-        should_finish_experiment :my_experiment
-        finished :my_metric
-      end
+    it "completes the test" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "other_opt" ],
+        :metric => :my_metric
+      }
+      should_finish_experiment :my_experiment
+      finished :my_metric
+    end
 
-      it "completes all relevant tests" do
-        Split.configuration.experiments = {
-          :exp_1 => {
-            :alternatives => [ "1-1", "1-2" ],
-            :metric => :my_metric
-          },
-          :exp_2 => {
-            :alternatives => [ "2-1", "2-2" ],
-            :metric => :another_metric
-          },
-          :exp_3 => {
-            :alternatives => [ "3-1", "3-2" ],
-            :metric => :my_metric
-          },
-        }
-        should_finish_experiment :exp_1
-        should_finish_experiment :exp_2, false
-        should_finish_experiment :exp_3
-        finished :my_metric
-      end
+    it "completes all relevant tests" do
+      Split.configuration.experiments = {
+        :exp_1 => {
+          :alternatives => [ "1-1", "1-2" ],
+          :metric => :my_metric
+        },
+        :exp_2 => {
+          :alternatives => [ "2-1", "2-2" ],
+          :metric => :another_metric
+        },
+        :exp_3 => {
+          :alternatives => [ "3-1", "3-2" ],
+          :metric => :my_metric
+        },
+      }
+      should_finish_experiment :exp_1
+      should_finish_experiment :exp_2, false
+      should_finish_experiment :exp_3
+      finished :my_metric
+    end
 
-      it "passes reset option" do
-        Split.configuration.experiments[@experiment_name] = {
-          :alternatives => @alternatives,
+    it "passes reset option" do
+      Split.configuration.experiments = {
+        :my_exp => {
+          :alternatives => ["one", "two"],
           :metric => :my_metric,
           :resettable => false,
         }
-        finished :my_metric
-        ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
-      end
+      }
+      alternative_name = ab_test(:my_exp)
+      exp = Split::Experiment.find :my_exp
+
+      finished :my_metric
+      ab_user[exp.key].should == alternative_name
+      ab_user[exp.finished_key].should == true
+    end
 
-      it "passes through options" do
-        Split.configuration.experiments[@experiment_name] = {
-          :alternatives => @alternatives,
+    it "passes through options" do
+      Split.configuration.experiments = {
+        :my_exp => {
+          :alternatives => ["one", "two"],
           :metric => :my_metric,
         }
-        finished :my_metric, :reset => false
-        ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
-      end
+      }
+      alternative_name = ab_test(:my_exp)
+      exp = Split::Experiment.find :my_exp
+
+      finished :my_metric, :reset => false
+      ab_user[exp.key].should == alternative_name
+      ab_user[exp.finished_key].should == true
     end
+
   end
 
   describe 'conversions' do
     it 'should return a conversion rate for an alternative' do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
 
       previous_convertion_rate = Split::Alternative.new(alternative_name, 'link_color').conversion_rate
@@ -282,13 +347,11 @@ describe Split::Helper do
 
     describe 'ab_test' do
       it 'should return the control' do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
         alternative = ab_test('link_color', 'blue', 'red')
         alternative.should eql experiment.control.name
       end
 
       it "should not increment the participation count" do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
 
         previous_red_count = Split::Alternative.new('red', 'link_color').participant_count
         previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
@@ -304,7 +367,6 @@ describe Split::Helper do
 
     describe 'finished' do
       it "should not increment the completed count" do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
         alternative_name = ab_test('link_color', 'blue', 'red')
 
         previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count
@@ -318,23 +380,35 @@ describe Split::Helper do
     end
   end
 
-  describe 'when ip address is ignored' do
-    before(:each) do
-      @request = OpenStruct.new(:ip => '81.19.48.130')
-      Split.configure do |c|
-        c.ignore_ip_addresses << '81.19.48.130'
+
+  describe 'when providing custom ignore logic' do
+    context "using a proc to configure custom logic" do
+
+      before(:each) do
+        Split.configure do |c|
+          c.ignore_filter = proc{|request| !!"i_am_going_to_be_disabled" }
+        end
+      end
+
+      it "ignores the ab_test" do
+        ab_test('link_color', 'blue', 'red')
+
+        red_count = Split::Alternative.new('red', 'link_color').participant_count
+        blue_count = Split::Alternative.new('blue', 'link_color').participant_count
+        (red_count + blue_count).should be(0)
       end
     end
+  end
+
+  shared_examples_for "a disabled test" do
 
     describe 'ab_test' do
       it 'should return the control' do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
         alternative = ab_test('link_color', 'blue', 'red')
         alternative.should eql experiment.control.name
       end
 
       it "should not increment the participation count" do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
 
         previous_red_count = Split::Alternative.new('red', 'link_color').participant_count
         previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
@@ -350,7 +424,6 @@ describe Split::Helper do
 
     describe 'finished' do
       it "should not increment the completed count" do
-        experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
         alternative_name = ab_test('link_color', 'blue', 'red')
 
         previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count
@@ -364,16 +437,58 @@ describe Split::Helper do
     end
   end
 
+  describe 'when ip address is ignored' do
+
+    context "individually" do
+
+      before(:each) do
+        @request = OpenStruct.new(:ip => '81.19.48.130')
+        Split.configure do |c|
+          c.ignore_ip_addresses << '81.19.48.130'
+        end
+      end
+
+      it_behaves_like "a disabled test"
+
+    end
+
+    context "for a range" do
+
+      before(:each) do
+        @request = OpenStruct.new(:ip => '81.19.48.129')
+        Split.configure do |c|
+          c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
+        end
+      end
+
+      it_behaves_like "a disabled test"
+
+    end
+
+    context "using both a range and a specific value" do
+
+      before(:each) do
+        @request = OpenStruct.new(:ip => '81.19.48.128')
+        Split.configure do |c|
+          c.ignore_ip_addresses << '81.19.48.130'
+          c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
+        end
+      end
+
+      it_behaves_like "a disabled test"
+
+    end
+
+  end
+
   describe 'versioned experiments' do
     it "should use version zero if no version is present" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
       experiment.version.should eql(0)
       ab_user.should eql({'link_color' => alternative_name})
     end
 
     it "should save the version of the experiment to the session" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       experiment.reset
       experiment.version.should eql(1)
       alternative_name = ab_test('link_color', 'blue', 'red')
@@ -381,7 +496,6 @@ describe Split::Helper do
     end
 
     it "should load the experiment even if the version is not 0" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       experiment.reset
       experiment.version.should eql(1)
       alternative_name = ab_test('link_color', 'blue', 'red')
@@ -391,7 +505,6 @@ describe Split::Helper do
     end
 
     it "should reset the session of a user on an older version of the experiment" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
       ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
@@ -409,7 +522,6 @@ describe Split::Helper do
     end
 
     it "should cleanup old versions of experiments from the session" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
       ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
@@ -425,7 +537,6 @@ describe Split::Helper do
     end
 
     it "should only count completion of users on the current version" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
       ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
@@ -554,6 +665,18 @@ describe Split::Helper do
             end
           end
         end
+
+        context 'and preloaded config given' do
+          before do
+            Split.configuration.experiments[:link_color] = {
+              :alternatives => [ "blue", "red" ],
+            }
+          end
+
+          it "uses first alternative" do
+            ab_test(:link_color).should eq("blue")
+          end
+        end
       end
 
       describe 'finished' do
@@ -577,34 +700,56 @@ describe Split::Helper do
 
   context "with preloaded config" do
     before { Split.configuration.experiments = {}}
-  
+
     it "pulls options from config file" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [ "control_opt", "other_opt" ],
+        :goals => ["goal1", "goal2"]
       }
       ab_test :my_experiment
-      Split::Experiment.find(:my_experiment).alternative_names.should == [ "control_opt", "other_opt" ]
+      Split::Experiment.new(:my_experiment).alternatives.map(&:name).should == [ "control_opt", "other_opt" ]
+      Split::Experiment.new(:my_experiment).goals.should == [ "goal1", "goal2" ]
     end
-  
+
     it "can be called multiple times" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [ "control_opt", "other_opt" ],
+        :goals => ["goal1", "goal2"]
       }
       5.times { ab_test :my_experiment }
-      experiment = Split::Experiment.find(:my_experiment)
-      experiment.alternative_names.should == [ "control_opt", "other_opt" ]
+      experiment = Split::Experiment.new(:my_experiment)
+      experiment.alternatives.map(&:name).should == [ "control_opt", "other_opt" ]
+      experiment.goals.should == [ "goal1", "goal2" ]
       experiment.participant_count.should == 1
     end
-  
+
+    it "accepts multiple goals" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "other_opt" ],
+        :goals => [ "goal1", "goal2", "goal3" ]
+      }
+      ab_test :my_experiment
+      experiment = Split::Experiment.new(:my_experiment)
+      experiment.goals.should == [ "goal1", "goal2", "goal3" ]
+    end
+
+    it "allow specifying goals to be optional" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "other_opt" ]
+      }
+      experiment = Split::Experiment.new(:my_experiment)
+      experiment.goals.should == []
+    end
+
     it "accepts multiple alternatives" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [ "control_opt", "second_opt", "third_opt" ],
       }
       ab_test :my_experiment
-      experiment = Split::Experiment.find(:my_experiment)
-      experiment.alternative_names.should == [ "control_opt", "second_opt", "third_opt" ]
+      experiment = Split::Experiment.new(:my_experiment)
+      experiment.alternatives.map(&:name).should == [ "control_opt", "second_opt", "third_opt" ]
     end
-  
+
     it "accepts probability on alternatives" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [
@@ -614,11 +759,11 @@ describe Split::Helper do
         ],
       }
       ab_test :my_experiment
-      experiment = Split::Experiment.find(:my_experiment)
+      experiment = Split::Experiment.new(:my_experiment)
       experiment.alternatives.collect{|a| [a.name, a.weight]}.should == [['control_opt', 0.67], ['second_opt', 0.1], ['third_opt', 0.23]]
-      
+
     end
-  
+
     it "accepts probability on some alternatives" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [
@@ -629,12 +774,12 @@ describe Split::Helper do
         ],
       }
       ab_test :my_experiment
-      experiment = Split::Experiment.find(:my_experiment)
+      experiment = Split::Experiment.new(:my_experiment)
       names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
       names_and_weights.should == [['control_opt', 0.34], ['second_opt', 0.215], ['third_opt', 0.23], ['fourth_opt', 0.215]]
       names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0
     end
-  
+
     it "allows name param without probability" do
       Split.configuration.experiments[:my_experiment] = {
         :alternatives => [
@@ -644,7 +789,7 @@ describe Split::Helper do
         ],
       }
       ab_test :my_experiment
-      experiment = Split::Experiment.find(:my_experiment)
+      experiment = Split::Experiment.new(:my_experiment)
       names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
       names_and_weights.should ==  [['control_opt', 0.18], ['second_opt', 0.18], ['third_opt', 0.64]]
       names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0
@@ -667,7 +812,6 @@ describe Split::Helper do
   end
 
   it 'should handle multiple experiments correctly' do
-    experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
     experiment2 = Split::Experiment.find_or_create('link_color2', 'blue', 'red')
     alternative_name = ab_test('link_color', 'blue', 'red')
     alternative_name2 = ab_test('link_color2', 'blue', 'red')
@@ -677,4 +821,43 @@ describe Split::Helper do
       alt.unfinished_count.should eq(0)
     end
   end
+
+  context "with goals" do
+    before do
+      @experiment = {'link_color' => ["purchase", "refund"]}
+      @alternatives = ['blue', 'red']
+      @experiment_name, @goals = normalize_experiment(@experiment)
+      @goal1 = @goals[0]
+      @goal2 = @goals[1]
+    end
+
+    it "should normalize experiment" do
+      @experiment_name.should eql("link_color")
+      @goals.should eql(["purchase", "refund"])
+    end
+
+    describe "ab_test" do
+      it "should allow experiment goals interface as a single hash" do
+        ab_test(@experiment, *@alternatives)
+        experiment = Split::Experiment.find('link_color')
+        experiment.goals.should eql(['purchase', "refund"])
+      end
+    end
+
+    describe "finished" do
+      before do
+        @alternative_name = ab_test(@experiment, *@alternatives)
+      end
+
+      it "should increment the counter for the specified-goal completed alternative" do
+        @previous_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
+        @previous_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
+        finished({"link_color" => "purchase"})
+        new_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
+        new_completion_count_for_goal1.should eql(@previous_completion_count_for_goal1 + 1)
+        new_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
+        new_completion_count_for_goal2.should eql(@previous_completion_count_for_goal2)
+      end
+    end
+  end
 end