split 0.3.3 → 0.4.0

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - jruby-18mode # JRuby in 1.8 mode
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-18mode
+  - rbx-19mode

data/CHANGELOG.mdown

@@ -1,3 +1,20 @@
+## 0.4.0 (March 7, 2012)
+
+**IMPORTANT**
+
+If using ruby 1.8.x and weighted alternatives you should always pass the control alternative through as the second argument with any other alternatives as a third argument because the order of the hash is not preserved in ruby 1.8, ruby 1.9 users are not affected by this bug.
+
+Features:
+
+  - Experiments now record when they were started (@vrish88, #35)
+  - Old versions of experiments in sessions are now cleaned up
+  - Avoid users participating in multiple experiments at once (#21)
+
+Bugfixes:
+
+  - Overriding alternatives doesn't work for weighted alternatives (@layflags, #34)
+  - confidence_level helper should handle tiny z-scores (#23)
+
 ## 0.3.3 (February 16, 2012)
 
 Bugfixes:

data/README.mdown

@@ -105,12 +105,14 @@ Perhaps you only want to show an alternative to 10% of your visitors because it
 
 To do this you can pass a weight with each alternative in the following ways:
 
-    ab_test('homepage design', 'Old' => 20, 'New' => 2)
+    ab_test('homepage design', {'Old' => 20}, {'New' => 2})
 
     ab_test('homepage design', 'Old', {'New' => 0.1})
 
     ab_test('homepage design', {'Old' => 10}, 'New')
 
+Note: If using ruby 1.8.x and weighted alternatives you should always pass the control alternative through as the second argument with any other alternatives as a third argument because the order of the hash is not preserved in ruby 1.8, ruby 1.9.1+ users are not affected by this bug.
+
 This will only show the new alternative to visitors 1 in 10 times, the default weight for an alternative is 1.
 
 ### Overriding alternatives
@@ -134,6 +136,16 @@ To stop this behaviour you can pass the following option to the `finished` metho
 
 The user will then always see the alternative they started with.
 
+### Multiple experiments at once
+
+By default Split will avoid users participating in multiple experiments at once. This means you are less likely to skew results by adding in more variation to your tests.
+
+To stop this behaviour and allow users to participate in multiple experiments at once enable the `allow_multiple_experiments` config option like so:
+
+    Split.configure do |config|
+      config.allow_multiple_experiments = true
+    end
+
 ## Web Interface
 
 Split comes with a Sinatra-based front end to get an overview of how your experiments are doing.
@@ -169,6 +181,7 @@ You can override the default configuration options of Split like so:
       config.ignore_ip_addresses << '81.19.48.130'
       config.db_failover = true # handle redis errors gracefully
       config.db_failover_on_db_error = proc{|error| Rails.logger.error(error.message) }
+      config.allow_multiple_experiments = true
     end
 
 ### DB failover solution
@@ -251,6 +264,8 @@ Report Issues/Feature requests on [GitHub Issues](http://github.com/andrew/split
 
 Tests can be ran with `rake spec`
 
+[![Build Status](https://secure.travis-ci.org/andrew/split.png?branch=master)](http://travis-ci.org/andrew/split)
+
 ### Note on Patches/Pull Requests
 
  * Fork the project.

data/lib/split/configuration.rb

@@ -4,12 +4,14 @@ module Split
     attr_accessor :ignore_ip_addresses
     attr_accessor :db_failover
     attr_accessor :db_failover_on_db_error
+    attr_accessor :allow_multiple_experiments
 
     def initialize
       @robot_regex = /\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i
       @ignore_ip_addresses = []
       @db_failover = false
       @db_failover_on_db_error = proc{|error|} # e.g. use Rails logger here
+      @allow_multiple_experiments = false
     end
   end
 end

data/lib/split/dashboard.rb

@@ -1,6 +1,7 @@
 require 'sinatra/base'
 require 'split'
 require 'bigdecimal'
+require 'split/dashboard/helpers'
 
 module Split
   class Dashboard < Sinatra::Base
@@ -11,50 +12,7 @@ module Split
     set :static, true
     set :method_override, true
 
-    helpers do
-      def url(*path_parts)
-        [ path_prefix, path_parts ].join("/").squeeze('/')
-      end
-
-      def path_prefix
-        request.env['SCRIPT_NAME']
-      end
-
-      def number_to_percentage(number, precision = 2)
-        round(number * 100)
-      end
-
-      def round(number, precision = 2)
-        BigDecimal.new(number.to_s).round(precision).to_f
-      end
-
-      def confidence_level(z_score)
-        z = z_score.to_f
-        if z > 0.0
-          if z < 1.96
-            'no confidence'
-          elsif z < 2.57
-            '95% confidence'
-          elsif z < 3.29
-            '99% confidence'
-          else
-            '99.9% confidence'
-          end
-        elsif z < 0.0
-          if z > -1.96
-            'no confidence'
-          elsif z > -2.57
-            '95% confidence'
-          elsif z > -3.29
-            '99% confidence'
-          else
-            '99.9% confidence'
-          end
-        else
-          "No Change"
-        end
-      end
-    end
+    helpers Split::DashboardHelpers
 
     get '/' do
       @experiments = Split::Experiment.all

data/lib/split/dashboard/helpers.rb

@@ -0,0 +1,48 @@
+module Split
+  module DashboardHelpers
+    def url(*path_parts)
+      [ path_prefix, path_parts ].join("/").squeeze('/')
+    end
+
+    def path_prefix
+      request.env['SCRIPT_NAME']
+    end
+
+    def number_to_percentage(number, precision = 2)
+      round(number * 100)
+    end
+
+    def round(number, precision = 2)
+      BigDecimal.new(number.to_s).round(precision).to_f
+    end
+
+    def confidence_level(z_score)
+      return z_score if z_score.is_a? String
+
+      z = round(z_score.to_f, 3)
+      if z > 0.0
+        if z < 1.96
+          'no confidence'
+        elsif z < 2.57
+          '95% confidence'
+        elsif z < 3.29
+          '99% confidence'
+        else
+          '99.9% confidence'
+        end
+      elsif z < 0.0
+        if z > -1.96
+          'no confidence'
+        elsif z > -2.57
+          '95% confidence'
+        elsif z > -3.29
+          '99% confidence'
+        else
+          '99.9% confidence'
+        end
+      else
+        "No Change"
+      end
+    end
+  end
+end

data/lib/split/dashboard/public/style.css

@@ -218,6 +218,11 @@ body {
   float:right;
 }
 
+.inline-controls small {
+  color: #888;
+  font-size: 11px;
+}
+
 .inline-controls form {
   display: inline-block;
   font-size: 10px;

data/lib/split/dashboard/views/_experiment.erb

@@ -1,7 +1,12 @@
 <div class="experiment">
   <div class="experiment-header">
-    <h2>Experiment: <%= experiment.name %> <% if experiment.version > 1 %><span class='version'>v<%= experiment.version %></span><% end %></h2>
+    <h2>
+      Experiment: <%= experiment.name %>
+      <% if experiment.version > 1 %><span class='version'>v<%= experiment.version %></span><% end %>
+    </h2>
+
     <div class='inline-controls'>
+      <small><%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %></small>
       <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
         <input type="submit" value="Reset Data">
       </form>

data/lib/split/experiment.rb

@@ -26,11 +26,15 @@ module Split
       Split.redis.hdel(:experiment_winner, name)
     end
 
-
     def winner=(winner_name)
       Split.redis.hset(:experiment_winner, name, winner_name.to_s)
     end
 
+    def start_time
+      t = Split.redis.hget(:experiment_start_times, @name)
+      Time.parse(t) if t
+    end
+
     def alternatives
       @alternatives.dup
     end
@@ -92,6 +96,7 @@ module Split
     def save
       if new_record?
         Split.redis.sadd(:experiments, name)
+        Split.redis.hset(:experiment_start_times, @name, Time.now)
         @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) }
       end
     end

data/lib/split/helper.rb

@@ -1,17 +1,19 @@
 module Split
   module Helper
-    def ab_test(experiment_name, *alternatives)
+    def ab_test(experiment_name, control, *alternatives)
+      puts 'WARNING: You should always pass the control alternative through as the second argument with any other alternatives as the third because the order of the hash is not preserved in ruby 1.8' if RUBY_VERSION.match(/1\.8/) && alternatives.length.zero?
       begin
-        experiment = Split::Experiment.find_or_create(experiment_name, *alternatives)
+        experiment = Split::Experiment.find_or_create(experiment_name, *([control] + alternatives))
         if experiment.winner
           ret = experiment.winner.name
         else
-          if forced_alternative = override(experiment.name, alternatives)
+          if forced_alternative = override(experiment.name, experiment.alternative_names)
             ret = forced_alternative
           else
-            begin_experiment(experiment, experiment.control.name) if exclude_visitor?
+            clean_old_versions(experiment)
+            begin_experiment(experiment) if exclude_visitor? or not_allowed_to_test?(experiment.key)
 
-            if ab_user[experiment.key]
+            if ab_user[experiment.key] 
               ret = ab_user[experiment.key]
             else
               alternative = experiment.next_alternative
@@ -24,7 +26,7 @@ module Split
       rescue Errno::ECONNREFUSED => e
         raise unless Split.configuration.db_failover
         Split.configuration.db_failover_on_db_error.call(e)
-        ret = Hash === (a0 = alternatives.first) ? a0.keys.first : a0
+        ret = Hash === control ? control.keys.first : control
       end
       if block_given?
         if defined?(capture) # a block in a rails view
@@ -56,7 +58,8 @@ module Split
       params[experiment_name] if defined?(params) && alternatives.include?(params[experiment_name])
     end
 
-    def begin_experiment(experiment, alternative_name)
+    def begin_experiment(experiment, alternative_name = nil)
+      alternative_name ||= experiment.control.name
       ab_user[experiment.key] = alternative_name
     end
 
@@ -68,6 +71,28 @@ module Split
       is_robot? or is_ignored_ip_address?
     end
 
+    def not_allowed_to_test?(experiment_key)
+      !Split.configuration.allow_multiple_experiments && doing_other_tests?(experiment_key)
+    end
+
+    def doing_other_tests?(experiment_key)
+      ab_user.keys.reject{|k| k == experiment_key}.length > 0
+    end
+
+    def clean_old_versions(experiment)
+      old_versions(experiment).each do |old_key|
+        ab_user.delete old_key
+      end
+    end
+
+    def old_versions(experiment)
+      if experiment.version > 0
+        ab_user.keys.select{|k| k.match(Regexp.new(experiment.name))}.reject{|k| k == experiment.key}
+      else
+        []
+      end
+    end
+
     def is_robot?
       request.user_agent =~ Split.configuration.robot_regex
     end

data/lib/split/version.rb

@@ -1,3 +1,3 @@
 module Split
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/spec/configuration_spec.rb

@@ -8,5 +8,6 @@ describe Split::Configuration do
     config.robot_regex.should eql(/\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i)
     config.db_failover.should be_false
     config.db_failover_on_db_error.should be_a Proc
+    config.allow_multiple_experiments.should be_false
   end
 end

data/spec/dashboard_helpers_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'split/dashboard/helpers'
+
+include Split::DashboardHelpers
+
+describe Split::DashboardHelpers do
+  describe 'confidence_level' do
+    it 'should handle very small numbers' do
+      confidence_level(0.000000000000006).should eql('No Change')
+    end
+  end
+end

data/spec/dashboard_spec.rb

@@ -51,4 +51,26 @@ describe Split::Dashboard do
     last_response.should be_redirect
     experiment.winner.name.should eql('red')
   end
+
+  it "should display the start date" do
+    experiment_start_time = Time.parse('2011-07-07')
+    Time.stub(:now => experiment_start_time)
+    experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
+
+    get '/'
+
+    last_response.body.should include('<small>2011-07-07</small>')
+  end
+
+  it "should handle experiments without a start date" do
+    experiment_start_time = Time.parse('2011-07-07')
+    Time.stub(:now => experiment_start_time)
+    experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
+
+    Split.redis.hdel(:experiment_start_times, experiment.name)
+
+    get '/'
+
+    last_response.body.should include('<small>Unknown</small>')
+  end
 end

data/spec/experiment_spec.rb

@@ -20,6 +20,26 @@ describe Split::Experiment do
     Split.redis.exists('basket_text').should be true
   end
 
+  it "should save the start time to redis" do
+    experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
+    Time.stub(:now => experiment_start_time)
+    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment.save
+
+    Split::Experiment.find('basket_text').start_time.should == experiment_start_time
+  end
+  
+  it "should handle not having a start time" do
+    experiment_start_time = Time.parse("Sat Mar 03 14:01:03")
+    Time.stub(:now => experiment_start_time)
+    experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
+    experiment.save
+
+    Split.redis.hdel(:experiment_start_times, experiment.name)
+
+    Split::Experiment.find('basket_text').start_time.should == nil
+  end
+
   it "should not create duplicates when saving multiple times" do
     experiment = Split::Experiment.new('basket_text', 'Basket', "Cart")
     experiment.save

data/spec/helper_spec.rb

@@ -50,6 +50,13 @@ describe Split::Helper do
       @params = {'link_color' => 'blue'}
       alternative = ab_test('link_color', 'blue', 'red')
       alternative.should eql('blue')
+      alternative = ab_test('link_color', {'blue' => 1}, 'red' => 5)
+      alternative.should eql('blue')
+      @params = {'link_color' => 'red'}
+      alternative = ab_test('link_color', 'blue', 'red')
+      alternative.should eql('red')
+      alternative = ab_test('link_color', {'blue' => 5}, 'red' => 1)
+      alternative.should eql('red')
     end
 
     it "should allow passing a block" do
@@ -58,17 +65,37 @@ describe Split::Helper do
       ret.should eql("shared/#{alt}")
     end
 
-    it  "should allow the share of visitors see an alternative to be specificed" do
+    it "should allow the share of visitors see an alternative to be specificed" do
       ab_test('link_color', {'blue' => 0.8}, {'red' => 20})
       ['red', 'blue'].should include(ab_user['link_color'])
     end
 
-    it  "should allow alternative weighting interface as a single hash" do
-      ab_test('link_color', 'blue' => 0.01, 'red' => 0.2)
+    it "should allow alternative weighting interface as a single hash" do
+      ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2)
       experiment = Split::Experiment.find('link_color')
       experiment.alternative_names.should eql(['blue', 'red'])
     end
 
+    it "should only let a user participate in one experiment at a time" do
+      ab_test('link_color', 'blue', 'red')
+      ab_test('button_size', 'small', 'big')
+      ab_user['button_size'].should eql('small')
+      big = Split::Alternative.new('big', 'button_size')
+      big.participant_count.should eql(0)
+      small = Split::Alternative.new('small', 'button_size')
+      small.participant_count.should eql(0)
+    end
+    
+    it "should let a user participate in many experiment with allow_multiple_experiments option" do
+      Split.configure do |config|
+        config.allow_multiple_experiments = true
+      end
+      link_color = ab_test('link_color', 'blue', 'red')
+      button_size = ab_test('button_size', 'small', 'big')
+      ab_user['button_size'].should eql(button_size)
+      button_size_alt = Split::Alternative.new(button_size, 'button_size')
+      button_size_alt.participant_count.should eql(1)
+    end
   end
 
   describe 'finished' do
@@ -258,6 +285,22 @@ describe Split::Helper do
       new_alternative.participant_count.should eql(1)
     end
 
+    it "should cleanup old versions of experiments from the session" do
+      experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
+      alternative_name = ab_test('link_color', 'blue', 'red')
+      session[:split].should eql({'link_color' => alternative_name})
+      alternative = Split::Alternative.new(alternative_name, 'link_color')
+      alternative.participant_count.should eql(1)
+
+      experiment.reset
+      experiment.version.should eql(1)
+      alternative = Split::Alternative.new(alternative_name, 'link_color')
+      alternative.participant_count.should eql(0)
+
+      new_alternative_name = ab_test('link_color', 'blue', 'red')
+      session[:split].should eql({'link_color:1' => new_alternative_name})
+    end
+
     it "should only count completion of users on the current version" do
       experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red')
       alternative_name = ab_test('link_color', 'blue', 'red')
@@ -329,14 +372,12 @@ describe Split::Helper do
           ab_test('link_color', 'blue', 'red')
         end
         it 'should always use first alternative' do
-          10.times do
-            ab_test('link_color', 'blue', 'red').should eq('blue')
-            ab_test('link_color', 'blue' => 0.01, 'red' => 0.2).should eq('blue')
-            ab_test('link_color', {'blue' => 0.8}, {'red' => 20}).should eq('blue')
-            ab_test('link_color', 'blue', 'red') do |alternative|
-              "shared/#{alternative}"
-            end.should eq('shared/blue')
-          end
+          ab_test('link_color', 'blue', 'red').should eq('blue')
+          ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2).should eq('blue')
+          ab_test('link_color', {'blue' => 0.8}, {'red' => 20}).should eq('blue')
+          ab_test('link_color', 'blue', 'red') do |alternative|
+            "shared/#{alternative}"
+          end.should eq('shared/blue')
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: split
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-16 00:00:00.000000000Z
+date: 2012-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70173932070820 !ruby/object:Gem::Requirement
+  requirement: &70291376942100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '2.1'
   type: :runtime
   prerelease: false
-  version_requirements: *70173932070820
+  version_requirements: *70291376942100
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70173932070060 !ruby/object:Gem::Requirement
+  requirement: &70291376938960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *70173932070060
+  version_requirements: *70291376938960
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70173932069160 !ruby/object:Gem::Requirement
+  requirement: &70291376938260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 1.2.6
   type: :runtime
   prerelease: false
-  version_requirements: *70173932069160
+  version_requirements: *70291376938260
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70173932068040 !ruby/object:Gem::Requirement
+  requirement: &70291376937280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70173932068040
+  version_requirements: *70291376937280
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70173932066460 !ruby/object:Gem::Requirement
+  requirement: &70291376930680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: '1.0'
   type: :development
   prerelease: false
-  version_requirements: *70173932066460
+  version_requirements: *70291376930680
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70173932065340 !ruby/object:Gem::Requirement
+  requirement: &70291376929580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: '2.6'
   type: :development
   prerelease: false
-  version_requirements: *70173932065340
+  version_requirements: *70291376929580
 - !ruby/object:Gem::Dependency
   name: rack-test
-  requirement: &70173932064500 !ruby/object:Gem::Requirement
+  requirement: &70291376928420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: '0.6'
   type: :development
   prerelease: false
-  version_requirements: *70173932064500
+  version_requirements: *70291376928420
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &70173932062500 !ruby/object:Gem::Requirement
+  requirement: &70291376927240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,7 +98,7 @@ dependencies:
         version: '0.4'
   type: :development
   prerelease: false
-  version_requirements: *70173932062500
+  version_requirements: *70291376927240
 description: 
 email:
 - andrewnez@gmail.com
@@ -107,6 +107,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG.mdown
 - Gemfile
 - Guardfile
@@ -117,6 +118,7 @@ files:
 - lib/split/alternative.rb
 - lib/split/configuration.rb
 - lib/split/dashboard.rb
+- lib/split/dashboard/helpers.rb
 - lib/split/dashboard/public/dashboard.js
 - lib/split/dashboard/public/reset.css
 - lib/split/dashboard/public/style.css
@@ -128,6 +130,7 @@ files:
 - lib/split/version.rb
 - spec/alternative_spec.rb
 - spec/configuration_spec.rb
+- spec/dashboard_helpers_spec.rb
 - spec/dashboard_spec.rb
 - spec/experiment_spec.rb
 - spec/helper_spec.rb
@@ -153,13 +156,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: split
-rubygems_version: 1.8.6
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Rack based split testing framework
 test_files:
 - spec/alternative_spec.rb
 - spec/configuration_spec.rb
+- spec/dashboard_helpers_spec.rb
 - spec/dashboard_spec.rb
 - spec/experiment_spec.rb
 - spec/helper_spec.rb