split 0.4.6 → 0.5.0

data/lib/split/version.rb

@@ -1,6 +1,6 @@
 module Split
   MAJOR = 0
-  MINOR = 4
-  TINY  = 6
-  VERSION = [MAJOR, MINOR, TINY].join('.')
+  MINOR = 5
+  PATCH  = 0
+  VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/spec/algorithms/weighted_sample_spec.rb

@@ -0,0 +1,18 @@
+require "spec_helper"
+
+describe Split::Algorithms::WeightedSample do
+  it "should return an alternative" do
+    experiment = Split::Experiment.find_or_create('link_color', {'blue' => 100}, {'red' => 0 })
+    Split::Algorithms::WeightedSample.choose_alternative(experiment).class.should == Split::Alternative
+  end
+
+  it "should always return a heavily weighted option" do
+    experiment = Split::Experiment.find_or_create('link_color', {'blue' => 100}, {'red' => 0 })
+    Split::Algorithms::WeightedSample.choose_alternative(experiment).name.should == 'blue'
+  end
+  
+  it "should return one of the results" do
+    experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    ['red', 'blue'].should include Split::Algorithms::WeightedSample.choose_alternative(experiment).name
+  end
+end

data/spec/algorithms/whiplash_spec.rb

@@ -0,0 +1,23 @@
+require "spec_helper"
+
+describe Split::Algorithms::Whiplash do
+
+  it "should return an algorithm" do
+    experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    Split::Algorithms::Whiplash.choose_alternative(experiment).class.should == Split::Alternative
+  end
+
+  it "should return one of the results" do
+    experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
+    ['red', 'blue'].should include Split::Algorithms::Whiplash.choose_alternative(experiment).name
+  end
+  
+  it "should guess floats" do
+    Split::Algorithms::Whiplash.send(:arm_guess, 0, 0).class.should == Float
+    Split::Algorithms::Whiplash.send(:arm_guess, 1, 0).class.should == Float
+    Split::Algorithms::Whiplash.send(:arm_guess, 2, 1).class.should == Float
+    Split::Algorithms::Whiplash.send(:arm_guess, 1000, 5).class.should == Float
+    Split::Algorithms::Whiplash.send(:arm_guess, 10, -2).class.should == Float
+  end
+  
+end

data/spec/alternative_spec.rb

@@ -2,19 +2,81 @@ require 'spec_helper'
 require 'split/alternative'
 
 describe Split::Alternative do
-  before(:each) { Split.redis.flushall }
 
   it "should have a name" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
     alternative = Split::Alternative.new('Basket', 'basket_text')
     alternative.name.should eql('Basket')
   end
 
   it "return only the name" do
-    experiment = Split::Experiment.new('basket_text', {'Basket' => 0.6}, {"Cart" => 0.4})
+    experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}])
     alternative = Split::Alternative.new('Basket', 'basket_text')
     alternative.name.should eql('Basket')
   end
+  
+  describe 'weights' do
+  
+    it "should set the weights" do
+      experiment = Split::Experiment.new('basket_text', :alternative_names => [{'Basket' => 0.6}, {"Cart" => 0.4}])
+      first = experiment.alternatives[0]
+      first.name.should == 'Basket'
+      first.weight.should == 0.6
+    
+      second = experiment.alternatives[1]
+      second.name.should == 'Cart'
+      second.weight.should == 0.4  
+    end
+    
+    it "accepts probability on alternatives" do
+      Split.configuration.experiments = {
+        :my_experiment => {
+          :alternatives => [
+            { :name => "control_opt", :percent => 67 },
+            { :name => "second_opt", :percent => 10 },
+            { :name => "third_opt", :percent => 23 },
+          ],
+        }
+      }
+      experiment = Split::Experiment.find(:my_experiment)
+      first = experiment.alternatives[0]
+      first.name.should == 'control_opt'
+      first.weight.should == 0.67
+    
+      second = experiment.alternatives[1]
+      second.name.should == 'second_opt'
+      second.weight.should == 0.1
+    end
+  
+    # it "accepts probability on some alternatives" do
+    #   Split.configuration.experiments[:my_experiment] = {
+    #     :alternatives => [
+    #       { :name => "control_opt", :percent => 34 },
+    #       "second_opt",
+    #       { :name => "third_opt", :percent => 23 },
+    #       "fourth_opt",
+    #     ],
+    #   }
+    #   should start_experiment(:my_experiment).with({"control_opt" => 0.34}, {"second_opt" => 0.215}, {"third_opt" => 0.23}, {"fourth_opt" => 0.215})
+    #   ab_test :my_experiment
+    # end
+    #   
+    # it "allows name param without probability" do
+    #   Split.configuration.experiments[:my_experiment] = {
+    #     :alternatives => [
+    #       { :name => "control_opt" },
+    #       "second_opt",
+    #       { :name => "third_opt", :percent => 64 },
+    #     ],
+    #   }
+    #   should start_experiment(:my_experiment).with({"control_opt" => 0.18}, {"second_opt" => 0.18}, {"third_opt" => 0.64})
+    #   ab_test :my_experiment
+    # end
+  
+    it "should set the weights from a configuration file" do
+    
+    end
+  end
 
   it "should have a default participation count of 0" do
     alternative = Split::Alternative.new('Basket', 'basket_text')
@@ -27,7 +89,7 @@ describe Split::Alternative do
   end
 
   it "should belong to an experiment" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
     experiment.save
     alternative = Split::Alternative.new('Basket', 'basket_text')
     alternative.experiment.name.should eql(experiment.name)
@@ -40,7 +102,7 @@ describe Split::Alternative do
   end
 
   it "should increment participation count" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
     experiment.save
     alternative = Split::Alternative.new('Basket', 'basket_text')
     old_participant_count = alternative.participant_count
@@ -51,7 +113,7 @@ describe Split::Alternative do
   end
 
   it "should increment completed count" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
     experiment.save
     alternative = Split::Alternative.new('Basket', 'basket_text')
     old_completed_count = alternative.participant_count
@@ -71,7 +133,7 @@ describe Split::Alternative do
   end
 
   it "should know if it is the control of an experiment" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
     experiment.save
     alternative = Split::Alternative.new('Basket', 'basket_text')
     alternative.control?.should be_true
@@ -79,6 +141,16 @@ describe Split::Alternative do
     alternative.control?.should be_false
   end
 
+  describe 'unfinished_count' do
+    it "should be difference between participant and completed counts" do
+      experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"])
+      experiment.save
+      alternative = Split::Alternative.new('Basket', 'basket_text')
+      alternative.increment_participation
+      alternative.unfinished_count.should eql(alternative.participant_count)
+    end
+  end
+
   describe 'conversion rate' do
     it "should be 0 if there are no conversions" do
       alternative = Split::Alternative.new('Basket', 'basket_text')
@@ -86,7 +158,7 @@ describe Split::Alternative do
       alternative.conversion_rate.should eql(0)
     end
 
-    it "does something" do
+    it "calculate conversion rate" do
       alternative = Split::Alternative.new('Basket', 'basket_text')
       alternative.stub(:participant_count).and_return(10)
       alternative.stub(:completed_count).and_return(4)
@@ -109,4 +181,4 @@ describe Split::Alternative do
       control.z_score.should eql('N/A')
     end
   end
-end
+end

data/spec/configuration_spec.rb

@@ -1,14 +1,66 @@
 require 'spec_helper'
 
 describe Split::Configuration do
-  it "should provide default values" do
-    config = Split::Configuration.new
-
-    config.ignore_ip_addresses.should eql([])
-    config.robot_regex.should eql(/\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i)
-    config.db_failover.should be_false
-    config.db_failover_on_db_error.should be_a Proc
-    config.allow_multiple_experiments.should be_false
-    config.enabled.should be_true
+
+  before(:each) { @config = Split::Configuration.new }
+
+  it "should provide a default value for ignore_ip_addresses" do
+    @config.ignore_ip_addresses.should eql([])
+  end
+
+  it "should provide default values for db failover" do
+    @config.db_failover.should be_false
+    @config.db_failover_on_db_error.should be_a Proc
+  end
+
+  it "should not allow multiple experiments by default" do
+    @config.allow_multiple_experiments.should be_false
+  end
+
+  it "should be enabled by default" do
+    @config.enabled.should be_true
+  end
+
+  it "disabled is the opposite of enabled" do
+    @config.enabled = false
+    @config.disabled?.should be_true
+  end
+
+  it "should provide a default pattern for robots" do
+    %w[Baidu Gigabot Googlebot libwww-perl lwp-trivial msnbot SiteUptime Slurp WordPress ZIBB ZyBorg].each do |robot|
+      @config.robot_regex.should =~ robot
+    end
+  end
+
+  it "should use the session adapter for persistence by default" do
+    @config.persistence.should eq(Split::Persistence::SessionAdapter)
+  end
+
+  it "should load a metric" do
+    @config.experiments = {:my_experiment=>
+        {:alternatives=>["control_opt", "other_opt"], :metric=>:my_metric}}
+
+    @config.metrics.should_not be_nil
+    @config.metrics.keys.should ==  [:my_metric]
+  end
+  
+  it "should allow loading of experiment using experment_for" do
+    @config.experiments = {:my_experiment=>
+        {:alternatives=>["control_opt", "other_opt"], :metric=>:my_metric}}
+    @config.experiment_for(:my_experiment).should == {:alternatives=>["control_opt", ["other_opt"]]}
+  end
+  
+  it "should normalize experiments" do
+    @config.experiments = {
+      :my_experiment => {
+        :alternatives => [
+          { :name => "control_opt", :percent => 67 },
+          { :name => "second_opt", :percent => 10 },
+          { :name => "third_opt", :percent => 23 },
+        ],
+      }
+    }
+    
+    @config.normalized_experiments.should == {:my_experiment=>{:alternatives=>[{"control_opt"=>0.67}, [{"second_opt"=>0.1}, {"third_opt"=>0.23}]]}}
   end
 end

data/spec/dashboard_helpers_spec.rb

@@ -9,12 +9,9 @@ describe Split::DashboardHelpers do
       confidence_level(Complex(2e-18, -0.03)).should eql('No Change')
     end
 
-    it "should consider a z-score of 1.96 < z < 2.57 as 95% confident" do
-      confidence_level(2.12).should eql('95% confidence')
+    it "should consider a z-score of 1.645 < z < 1.96 as 95% confident" do
+      confidence_level(1.80).should eql('95% confidence')
     end
 
-    it "should consider a z-score of -1.96 > z > -2.57 as 95% confident" do
-      confidence_level(-2.12).should eql('95% confidence')
-    end
   end
 end

data/spec/dashboard_spec.rb

@@ -9,8 +9,6 @@ describe Split::Dashboard do
     @app ||= Split::Dashboard
   end
 
-  before(:each) { Split.redis.flushall }
-
   it "should respond to /" do
     get '/'
     last_response.should be_ok

data/spec/experiment_spec.rb

@@ -1,56 +1,135 @@
 require 'spec_helper'
 require 'split/experiment'
+require 'split/algorithms'
+require 'time'
 
 describe Split::Experiment do
-  before(:each) { Split.redis.flushall }
+  context "with an experiment" do
+    let(:experiment) { Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"]) }
+    
+    it "should have a name" do
+      experiment.name.should eql('basket_text')
+    end
 
-  it "should have a name" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.name.should eql('basket_text')
-  end
+    it "should have alternatives" do
+      experiment.alternatives.length.should be 2
+    end
+    
+    it "should have alternatives with correct names" do
+      experiment.alternatives.collect{|a| a.name}.should == ['Basket', 'Cart']
+    end
+    
+    it "should be resettable by default" do
+      experiment.resettable.should be_true
+    end
+    
+    it "should save to redis" do
+      experiment.save
+      Split.redis.exists('basket_text').should be true
+    end
 
-  it "should have alternatives" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.alternatives.length.should be 2
-  end
+    it "should save the start time to redis" do
+      experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
+      Time.stub(:now => experiment_start_time)
+      experiment.save
 
-  it "should save to redis" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.save
-    Split.redis.exists('basket_text').should be true
-  end
+      Split::Experiment.find('basket_text').start_time.should == experiment_start_time
+    end
+  
+    it "should save the selected algorithm to redis" do
+      experiment_algorithm = Split::Algorithms::Whiplash
+      experiment.algorithm = experiment_algorithm 
+      experiment.save
 
-  it "should save the start time to redis" do
-    experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
-    Time.stub(:now => experiment_start_time)
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.save
+      Split::Experiment.find('basket_text').algorithm.should == experiment_algorithm
+    end
 
-    Split::Experiment.find('basket_text').start_time.should == experiment_start_time
-  end
+    it "should handle not having a start time" do
+      experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
+      Time.stub(:now => experiment_start_time)
+      experiment.save
 
-  it "should handle not having a start time" do
-    experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
-    Time.stub(:now => experiment_start_time)
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.save
+      Split.redis.hdel(:experiment_start_times, experiment.name)
 
-    Split.redis.hdel(:experiment_start_times, experiment.name)
+      Split::Experiment.find('basket_text').start_time.should == nil
+    end
 
-    Split::Experiment.find('basket_text').start_time.should == nil
+    it "should not create duplicates when saving multiple times" do
+      experiment.save
+      experiment.save
+      Split.redis.exists('basket_text').should be true
+      Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"])
+    end
+    
+    describe 'new record?' do
+      it "should know if it hasn't been saved yet" do
+        experiment.new_record?.should be_true
+      end
+
+      it "should know if it has been saved yet" do
+        experiment.save
+        experiment.new_record?.should be_false
+      end
+    end
+    
+    describe 'find' do
+      it "should return an existing experiment" do
+        experiment.save
+        Split::Experiment.find('basket_text').name.should eql('basket_text')
+      end
+
+      it "should return an existing experiment" do
+        Split::Experiment.find('non_existent_experiment').should be_nil
+      end
+    end
+    
+    describe 'control' do
+      it 'should be the first alternative' do
+        experiment.save
+        experiment.control.name.should eql('Basket')
+      end
+    end
   end
-
-  it "should not create duplicates when saving multiple times" do
-    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-    experiment.save
-    experiment.save
-    Split.redis.exists('basket_text').should be true
-    Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"])
+  describe 'initialization' do
+    it "should set the algorithm when passed as an option to the initializer" do
+       experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm =>  Split::Algorithms::Whiplash)
+       experiment.algorithm.should == Split::Algorithms::Whiplash
+    end
+  
+    it "should be possible to make an experiment not resettable" do
+      experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false)
+      experiment.resettable.should be_false    
+    end
+  end
+  
+  describe 'persistent configuration' do
+  
+    it "should persist resettable in redis" do
+      experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :resettable => false)
+      experiment.save
+        
+      e = Split::Experiment.find('basket_text')
+      e.should == experiment
+      e.resettable.should be_false
+    
+    end
+    
+    it "should persist algorithm in redis" do
+      experiment = Split::Experiment.new('basket_text', :alternative_names => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash)
+      experiment.save
+  
+      e = Split::Experiment.find('basket_text')
+      e.should == experiment
+      e.algorithm.should == Split::Algorithms::Whiplash
+    end
   end
 
+  
+  
+  
   describe 'deleting' do
     it 'should delete itself' do
-      experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+      experiment = Split::Experiment.new('basket_text', :alternative_names => [ 'Basket', "Cart"])
       experiment.save
 
       experiment.delete
@@ -66,39 +145,6 @@ describe Split::Experiment do
     end
   end
 
-  describe 'new record?' do
-    it "should know if it hasn't been saved yet" do
-      experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-      experiment.new_record?.should be_true
-    end
-
-    it "should know if it has been saved yet" do
-      experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-      experiment.save
-      experiment.new_record?.should be_false
-    end
-  end
-
-  describe 'find' do
-    it "should return an existing experiment" do
-      experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-      experiment.save
-      Split::Experiment.find('basket_text').name.should eql('basket_text')
-    end
-
-    it "should return an existing experiment" do
-      Split::Experiment.find('non_existent_experiment').should be_nil
-    end
-  end
-
-  describe 'control' do
-    it 'should be the first alternative' do
-      experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
-      experiment.save
-      experiment.control.name.should eql('Basket')
-    end
-  end
-
   describe 'winner' do
     it "should have no winner initially" do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
@@ -150,10 +196,24 @@ describe Split::Experiment do
       experiment.version.should eql(1)
     end
   end
-
+  
+  describe 'algorithm' do
+    let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') }
+    
+    it 'should use the default algorithm if none is specified' do
+      experiment.algorithm.should == Split.configuration.algorithm
+    end
+    
+    it 'should use the user specified algorithm for this experiment if specified' do
+      experiment.algorithm = Split::Algorithms::Whiplash
+      experiment.algorithm.should == Split::Algorithms::Whiplash
+    end
+  end
+  
   describe 'next_alternative' do
+    let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') }
+    
     it "should always return the winner if one exists" do
-      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green')
       green = Split::Alternative.new('green', 'link_color')
       experiment.winner = 'green'
 
@@ -163,6 +223,19 @@ describe Split::Experiment do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green')
       experiment.next_alternative.name.should eql('green')
     end
+    
+    it "should use the specified algorithm if a winner does not exist" do
+      Split.configuration.algorithm.should_receive(:choose_alternative).and_return(Split::Alternative.new('green', 'link_color'))
+      experiment.next_alternative.name.should eql('green')
+    end
+  end
+  
+  describe 'single alternative' do
+    let(:experiment) { Split::Experiment.find_or_create('link_color', 'blue') }
+    
+    it "should always return the color blue" do
+      experiment.next_alternative.name.should eql('blue')
+    end 
   end
 
   describe 'changing an existing experiment' do
@@ -209,7 +282,4 @@ describe Split::Experiment do
       same_experiment.alternatives.map(&:weight).should == [1, 2]
     end
   end
-
-
-
 end