split 0.4.6 → 0.5.0

data/spec/helper_spec.rb

@@ -5,13 +5,20 @@ require 'spec_helper'
 describe Split::Helper do
   include Split::Helper
 
-  before(:each) do
-    Split.redis.flushall
-    @session = {}
-    params = nil
-  end
-
   describe "ab_test" do
+
+    it "should not raise an error when passed strings for alternatives" do
+      lambda { ab_test('xyz', '1', '2', '3') }.should_not raise_error
+    end
+
+    it "should raise the appropriate error when passed integers for alternatives" do
+      lambda { ab_test('xyz', 1, 2, 3) }.should raise_error(ArgumentError)
+    end
+
+    it "should raise the appropriate error when passed symbols for alternatives" do
+      lambda { ab_test('xyz', :a, :b, :c) }.should raise_error(ArgumentError)
+    end
+
     it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
       ab_test('link_color', 'blue', 'red')
       ['red', 'blue'].should include(ab_user['link_color'])
@@ -74,12 +81,14 @@ describe Split::Helper do
       ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2)
       experiment = Split::Experiment.find('link_color')
       experiment.alternative_names.should eql(['blue', 'red'])
+      # TODO: persist alternative weights
+      # experiment.alternatives.collect{|a| a.weight}.should == [0.01, 0.2]
     end
 
     it "should only let a user participate in one experiment at a time" do
-      ab_test('link_color', 'blue', 'red')
+      link_color = ab_test('link_color', 'blue', 'red')
       ab_test('button_size', 'small', 'big')
-      ab_user['button_size'].should eql('small')
+      ab_user.should eql({'link_color' => link_color})
       big = Split::Alternative.new('big', 'button_size')
       big.participant_count.should eql(0)
       small = Split::Alternative.new('small', 'button_size')
@@ -92,10 +101,19 @@ describe Split::Helper do
       end
       link_color = ab_test('link_color', 'blue', 'red')
       button_size = ab_test('button_size', 'small', 'big')
-      ab_user['button_size'].should eql(button_size)
+      ab_user.should eql({'link_color' => link_color, 'button_size' => button_size})
       button_size_alt = Split::Alternative.new(button_size, 'button_size')
       button_size_alt.participant_count.should eql(1)
     end
+
+    it "should not over-write a finished key when an experiment is on a later version" do
+      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
+      experiment.increment_version
+      ab_user = { experiment.key => 'blue', experiment.finished_key => true }
+      finshed_session = ab_user.dup
+      ab_test('link_color', 'blue', 'red')
+      ab_user.should eql(finshed_session)
+    end
   end
 
   describe 'finished' do
@@ -115,7 +133,7 @@ describe Split::Helper do
 
     it "should set experiment's finished key if reset is false" do
       finished(@experiment_name, :reset => false)
-      session[:split].should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+      ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
     end
 
     it 'should not increment the counter if reset is false and the experiment has been already finished' do
@@ -125,36 +143,121 @@ describe Split::Helper do
     end
 
     it "should clear out the user's participation from their session" do
-      session[:split].should eql(@experiment.key => @alternative_name)
+      ab_user.should eql(@experiment.key => @alternative_name)
       finished(@experiment_name)
-      session[:split].should == {}
+      ab_user.should == {}
     end
 
     it "should not clear out the users session if reset is false" do
-      session[:split].should eql(@experiment.key => @alternative_name)
+      ab_user.should eql(@experiment.key => @alternative_name)
       finished(@experiment_name, :reset => false)
-      session[:split].should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+      ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
     end
 
     it "should reset the users session when experiment is not versioned" do
-      session[:split].should eql(@experiment.key => @alternative_name)
+      ab_user.should eql(@experiment.key => @alternative_name)
       finished(@experiment_name)
-      session[:split].should eql({})
+      ab_user.should eql({})
     end
 
     it "should reset the users session when experiment is versioned" do
       @experiment.increment_version
       @alternative_name = ab_test(@experiment_name, *@alternatives)
 
-      session[:split].should eql(@experiment.key => @alternative_name)
+      ab_user.should eql(@experiment.key => @alternative_name)
       finished(@experiment_name)
-      session[:split].should eql({})
+      ab_user.should eql({})
     end
 
     it "should do nothing where the experiment was not started by this user" do
-      session[:split] = nil
+      ab_user = nil
       lambda { finished('some_experiment_not_started_by_the_user') }.should_not raise_exception
     end
+
+    it 'should not be doing other tests when it has completed one that has :reset => false' do
+      ab_user[@experiment.key] = @alternative_name
+      ab_user[@experiment.finished_key] = true
+      doing_other_tests?(@experiment.key).should be false
+    end
+
+    it "passes reset option from config" do
+      Split.configuration.experiments = {
+        @experiment_name => {
+          :alternatives => @alternatives,
+          :resettable => false,
+        }
+      }
+      finished @experiment_name
+      ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+    end
+
+    context "with metric name" do
+      before { Split.configuration.experiments = {} }
+      before { Split::Alternative.stub(:new).and_call_original }
+
+      def should_finish_experiment(experiment_name, should_finish=true)
+        alts = Split.configuration.experiments[experiment_name][:alternatives]
+        experiment = Split::Experiment.find_or_create(experiment_name, *alts)
+        alt_name = ab_user[experiment.key] = alts.first
+        alt = mock('alternative')
+        alt.stub(:name).and_return(alt_name)
+        Split::Alternative.stub(:new).with(alt_name, experiment_name).and_return(alt)
+        if should_finish
+          alt.should_receive(:increment_completion)
+        else
+          alt.should_not_receive(:increment_completion)
+        end
+      end
+
+      it "completes the test" do
+        Split.configuration.experiments[:my_experiment] = {
+          :alternatives => [ "control_opt", "other_opt" ],
+          :metric => :my_metric
+        }
+        should_finish_experiment :my_experiment
+        finished :my_metric
+      end
+
+      it "completes all relevant tests" do
+        Split.configuration.experiments = {
+          :exp_1 => {
+            :alternatives => [ "1-1", "1-2" ],
+            :metric => :my_metric
+          },
+          :exp_2 => {
+            :alternatives => [ "2-1", "2-2" ],
+            :metric => :another_metric
+          },
+          :exp_3 => {
+            :alternatives => [ "3-1", "3-2" ],
+            :metric => :my_metric
+          },
+        }
+        should_finish_experiment :exp_1
+        should_finish_experiment :exp_2, false
+        should_finish_experiment :exp_3
+        finished :my_metric
+      end
+
+      it "passes reset option" do
+        Split.configuration.experiments[@experiment_name] = {
+          :alternatives => @alternatives,
+          :metric => :my_metric,
+          :resettable => false,
+        }
+        finished :my_metric
+        ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+      end
+
+      it "passes through options" do
+        Split.configuration.experiments[@experiment_name] = {
+          :alternatives => @alternatives,
+          :metric => :my_metric,
+        }
+        finished :my_metric, :reset => false
+        ab_user.should eql(@experiment.key => @alternative_name, @experiment.finished_key => true)
+      end
+    end
   end
 
   describe 'conversions' do
@@ -198,6 +301,7 @@ describe Split::Helper do
         (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count)
       end
     end
+
     describe 'finished' do
       it "should not increment the completed count" do
         experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
@@ -213,6 +317,7 @@ describe Split::Helper do
       end
     end
   end
+
   describe 'when ip address is ignored' do
     before(:each) do
       @request = OpenStruct.new(:ip => '81.19.48.130')
@@ -242,6 +347,7 @@ describe Split::Helper do
         (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count)
       end
     end
+
     describe 'finished' do
       it "should not increment the completed count" do
         experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
@@ -263,7 +369,7 @@ describe Split::Helper do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
       experiment.version.should eql(0)
-      session[:split].should eql({'link_color' => alternative_name})
+      ab_user.should eql({'link_color' => alternative_name})
     end
 
     it "should save the version of the experiment to the session" do
@@ -271,7 +377,7 @@ describe Split::Helper do
       experiment.reset
       experiment.version.should eql(1)
       alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color:1' => alternative_name})
+      ab_user.should eql({'link_color:1' => alternative_name})
     end
 
     it "should load the experiment even if the version is not 0" do
@@ -279,7 +385,7 @@ describe Split::Helper do
       experiment.reset
       experiment.version.should eql(1)
       alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color:1' => alternative_name})
+      ab_user.should eql({'link_color:1' => alternative_name})
       return_alternative_name = ab_test('link_color', 'blue', 'red')
       return_alternative_name.should eql(alternative_name)
     end
@@ -287,7 +393,7 @@ describe Split::Helper do
     it "should reset the session of a user on an older version of the experiment" do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color' => alternative_name})
+      ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
       alternative.participant_count.should eql(1)
 
@@ -297,7 +403,7 @@ describe Split::Helper do
       alternative.participant_count.should eql(0)
 
       new_alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split]['link_color:1'].should eql(new_alternative_name)
+      ab_user['link_color:1'].should eql(new_alternative_name)
       new_alternative = Split::Alternative.new(new_alternative_name, 'link_color')
       new_alternative.participant_count.should eql(1)
     end
@@ -305,7 +411,7 @@ describe Split::Helper do
     it "should cleanup old versions of experiments from the session" do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color' => alternative_name})
+      ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
       alternative.participant_count.should eql(1)
 
@@ -315,13 +421,13 @@ describe Split::Helper do
       alternative.participant_count.should eql(0)
 
       new_alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color:1' => new_alternative_name})
+      ab_user.should eql({'link_color:1' => new_alternative_name})
     end
 
     it "should only count completion of users on the current version" do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
-      session[:split].should eql({'link_color' => alternative_name})
+      ab_user.should eql({'link_color' => alternative_name})
       alternative = Split::Alternative.new(alternative_name, 'link_color')
 
       experiment.reset
@@ -466,10 +572,109 @@ describe Split::Helper do
           finished('link_color')
         end
       end
-
-
     end
+  end
 
+  context "with preloaded config" do
+    before { Split.configuration.experiments = {}}
+  
+    it "pulls options from config file" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "other_opt" ],
+      }
+      ab_test :my_experiment
+      Split::Experiment.find(:my_experiment).alternative_names.should == [ "control_opt", "other_opt" ]
+    end
+  
+    it "can be called multiple times" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "other_opt" ],
+      }
+      5.times { ab_test :my_experiment }
+      experiment = Split::Experiment.find(:my_experiment)
+      experiment.alternative_names.should == [ "control_opt", "other_opt" ]
+      experiment.participant_count.should == 1
+    end
+  
+    it "accepts multiple alternatives" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [ "control_opt", "second_opt", "third_opt" ],
+      }
+      ab_test :my_experiment
+      experiment = Split::Experiment.find(:my_experiment)
+      experiment.alternative_names.should == [ "control_opt", "second_opt", "third_opt" ]
+    end
+  
+    it "accepts probability on alternatives" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [
+          { :name => "control_opt", :percent => 67 },
+          { :name => "second_opt", :percent => 10 },
+          { :name => "third_opt", :percent => 23 },
+        ],
+      }
+      ab_test :my_experiment
+      experiment = Split::Experiment.find(:my_experiment)
+      experiment.alternatives.collect{|a| [a.name, a.weight]}.should == [['control_opt', 0.67], ['second_opt', 0.1], ['third_opt', 0.23]]
+      
+    end
+  
+    it "accepts probability on some alternatives" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [
+          { :name => "control_opt", :percent => 34 },
+          "second_opt",
+          { :name => "third_opt", :percent => 23 },
+          "fourth_opt",
+        ],
+      }
+      ab_test :my_experiment
+      experiment = Split::Experiment.find(:my_experiment)
+      names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
+      names_and_weights.should == [['control_opt', 0.34], ['second_opt', 0.215], ['third_opt', 0.23], ['fourth_opt', 0.215]]
+      names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0
+    end
+  
+    it "allows name param without probability" do
+      Split.configuration.experiments[:my_experiment] = {
+        :alternatives => [
+          { :name => "control_opt" },
+          "second_opt",
+          { :name => "third_opt", :percent => 64 },
+        ],
+      }
+      ab_test :my_experiment
+      experiment = Split::Experiment.find(:my_experiment)
+      names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
+      names_and_weights.should ==  [['control_opt', 0.18], ['second_opt', 0.18], ['third_opt', 0.64]]
+      names_and_weights.inject(0){|sum, nw| sum + nw[1]}.should == 1.0
+    end
+
+    it "fails gracefully if config is missing experiment" do
+      Split.configuration.experiments = { :other_experiment => { :foo => "Bar" } }
+      lambda { ab_test :my_experiment }.should raise_error(/not found/i)
+    end
+
+    it "fails gracefully if config is missing" do
+      Split.configuration.experiments = nil
+      lambda { ab_test :my_experiment }.should raise_error(/not found/i)
+    end
+
+    it "fails gracefully if config is missing alternatives" do
+      Split.configuration.experiments[:my_experiment] = { :foo => "Bar" }
+      lambda { ab_test :my_experiment }.should raise_error(/alternatives/i)
+    end
   end
 
+  it 'should handle multiple experiments correctly' do
+    experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
+    experiment2 = Split::Experiment.find_or_create('link_color2', 'blue', 'red')
+    alternative_name = ab_test('link_color', 'blue', 'red')
+    alternative_name2 = ab_test('link_color2', 'blue', 'red')
+    finished('link_color2')
+
+    experiment2.alternatives.each do |alt|
+      alt.unfinished_count.should eq(0)
+    end
+  end
 end

data/spec/metric_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require 'split/metric'
+
+describe Split::Metric do
+  describe 'possible experiments' do
+    it "should load the experiment if there is one, but no metric" do
+      experiment = Split::Experiment.find_or_create('color', 'red', 'blue')
+      Split::Metric.possible_experiments('color').should == [experiment]
+    end
+
+    it "should load the experiments in a metric" do
+      experiment1 = Split::Experiment.find_or_create('color', 'red', 'blue')
+      experiment2 = Split::Experiment.find_or_create('size', 'big', 'small')
+
+      metric = Split::Metric.new(:name => 'purchase', :experiments => [experiment1, experiment2])
+      metric.save
+      Split::Metric.possible_experiments('purchase').should include(experiment1, experiment2)
+    end
+
+    it "should load both the metric experiments and an experiment with the same name" do
+      experiment1 = Split::Experiment.find_or_create('purchase', 'red', 'blue')
+      experiment2 = Split::Experiment.find_or_create('size', 'big', 'small')
+
+      metric = Split::Metric.new(:name => 'purchase', :experiments => [experiment2])
+      metric.save
+      Split::Metric.possible_experiments('purchase').should include(experiment1, experiment2)
+    end
+  end
+
+end

data/spec/persistence/cookie_adapter_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe Split::Persistence::CookieAdapter do
+
+  let(:context) { mock(:cookies => CookiesMock.new) }
+  subject { Split::Persistence::CookieAdapter.new(context) }
+
+  describe "#[] and #[]=" do
+    it "should set and return the value for given key" do
+      subject["my_key"] = "my_value"
+      subject["my_key"].should eq("my_value")
+    end
+  end
+
+  describe "#delete" do
+    it "should delete the given key" do
+      subject["my_key"] = "my_value"
+      subject.delete("my_key")
+      subject["my_key"].should be_nil
+    end
+  end
+
+  describe "#keys" do
+    it "should return an array of the session's stored keys" do
+      subject["my_key"] = "my_value"
+      subject["my_second_key"] = "my_second_value"
+      subject.keys.should =~ ["my_key", "my_second_key"]
+    end
+  end
+
+end