split 0.6.5 → 0.6.6

data/.travis.yml

@@ -16,6 +16,7 @@ matrix:
     - rvm: ruby-head
     - rvm: rbx-18mode
     - rvm: rbx-19mode
+    - rvm: jruby-head
   exclude:
     - rvm: 1.8.7
       gemfile: gemfiles/4.0.gemfile

data/CHANGELOG.mdown

@@ -1,3 +1,19 @@
+## 0.6.6 (October 15th, 2013)
+
+Features:
+
+  - Sort experiments on Dashboard so "active" ones without a winner appear first (@swrobel, #204)
+  - Starting tests manually (@duksis, #209)
+
+Bugfixes:
+
+  - Only trigger completion callback with valid Trial (@segfaultAX, #208)
+  - Fixed bug with `resettable` when using `normalize_experiments` (@jonashuckestein, #213)
+
+Misc:
+
+  - Added more bots to filter list (@lbeder, #214, #215, #216)
+
 ## 0.6.5 (August 23, 2013)
 
 Features:

data/README.mdown

@@ -165,6 +165,13 @@ If you have an experiment called `button_color` with alternatives called `red` a
 
 will always have red buttons. This won't be stored in your session or count towards to results, unless you set the `store_override` configuration option.
 
+### Starting experiments manually
+
+By default new AB tests will be active right after deployment. In case you would like to start new test a while after
+the deploy, you can do it by setting the `start_manually` configuration option to `true`.
+
+After choosing this option tests won't be started right after deploy, but after pressing the `Start` button in Split admin dashboard.
+
 ### Reset after completion
 
 When a user completes a test their session is reset so that they may start the test again in the future.
@@ -336,6 +343,8 @@ Split.configure do |config|
   config.allow_multiple_experiments = true
   config.enabled = true
   config.persistence = Split::Persistence::SessionAdapter
+  #config.start_manually = false ## new test will have to be started manually from the admin panel. default false
+  config.include_rails_helper = true
 end
 ```
 

data/lib/split/algorithms/whiplash.rb

@@ -1,4 +1,4 @@
-# A multi-armed bandit implementation inspired by 
+# A multi-armed bandit implementation inspired by
 # @aaronsw and victorykit/whiplash
 require 'simple-random'
 

data/lib/split/configuration.rb

@@ -12,59 +12,78 @@ module Split
     attr_accessor :persistence
     attr_accessor :algorithm
     attr_accessor :store_override
+    attr_accessor :start_manually
     attr_accessor :on_trial_choose
     attr_accessor :on_trial_complete
     attr_accessor :on_experiment_reset
     attr_accessor :on_experiment_delete
+    attr_accessor :include_rails_helper
 
     attr_reader :experiments
 
     def bots
       @bots ||= {
         # Indexers
-        "AdsBot-Google" => 'Google Adwords',
+        'AdsBot-Google' => 'Google Adwords',
         'Baidu' => 'Chinese search engine',
+        'Baiduspider' => 'Chinese search engine',
+        'bingbot' => 'Microsoft bing bot',
+        'Butterfly' => 'Topsy Labs',
         'Gigabot' => 'Gigabot spider',
         'Googlebot' => 'Google spider',
+        'MJ12bot' => 'Majestic-12 spider',
         'msnbot' => 'Microsoft bot',
-        'bingbot' => 'Microsoft bing bot',
         'rogerbot' => 'SeoMoz spider',
+        'PaperLiBot' => 'PaperLi is another content curation service',
         'Slurp' => 'Yahoo spider',
         'Sogou' => 'Chinese search engine',
-        "spider" => 'generic web spider',
+        'spider' => 'generic web spider',
+        'UnwindFetchor' => 'Gnip crawler',
         'WordPress' => 'WordPress spider',
-        'ZIBB' => 'ZIBB spider',
         'YandexBot' => 'Yandex spider',
+        'ZIBB' => 'ZIBB spider',
+
         # HTTP libraries
         'Apache-HttpClient' => 'Java http library',
         'AppEngine-Google' => 'Google App Engine',
-        "curl" => 'curl unix CLI http client',
+        'curl' => 'curl unix CLI http client',
         'ColdFusion' => 'ColdFusion http library',
-        "EventMachine HttpClient" => 'Ruby http library',
-        "Go http package" => 'Go http library',
+        'EventMachine HttpClient' => 'Ruby http library',
+        'Go http package' => 'Go http library',
         'Java' => 'Generic Java http library',
         'libwww-perl' => 'Perl client-server library loved by script kids',
         'lwp-trivial' => 'Another Perl library loved by script kids',
-        "Python-urllib" => 'Python http library',
-        "PycURL" => 'Python http library',
-        "Test Certificate Info" => 'C http library?',
-        "Wget" => 'wget unix CLI http client',
+        'Python-urllib' => 'Python http library',
+        'PycURL' => 'Python http library',
+        'Test Certificate Info' => 'C http library?',
+        'Wget' => 'wget unix CLI http client',
+
         # URL expanders / previewers
         'awe.sm' => 'Awe.sm URL expander',
-        "bitlybot" => 'bit.ly bot',
-        "facebookexternalhit" => 'facebook bot',
+        'bitlybot' => 'bit.ly bot',
+        'bot@linkfluence.net' => 'Linkfluence bot',
+        'facebookexternalhit' => 'facebook bot',
+        'Feedfetcher-Google' => 'Google Feedfetcher',
+        'https://developers.google.com/+/web/snippet' => 'Google+ Snippet Fetcher',
         'LongURL' => 'URL expander service',
+        'NING' => 'NING - Yet Another Twitter Swarmer',
+        'redditbot' => 'Reddit Bot',
+        'ShortLinkTranslate' => 'Link shortener',
+        'TweetmemeBot' => 'TweetMeMe Crawler',
         'Twitterbot' => 'Twitter URL expander',
         'UnwindFetch' => 'Gnip URL expander',
+        'vkShare' => 'VKontake Sharer',
+
         # Uptime monitoring
         'check_http' => 'Nagios monitor',
         'NewRelicPinger' => 'NewRelic monitor',
         'Panopta' => 'Monitoring service',
-        "Pingdom" => 'Pingdom monitoring',
+        'Pingdom' => 'Pingdom monitoring',
         'SiteUptime' => 'Site monitoring services',
+
         # ???
-        "DigitalPersona Fingerprint Software" => 'HP Fingerprint scanner',
-        "ShowyouBot" => 'Showyou iOS app spider',
+        'DigitalPersona Fingerprint Software' => 'HP Fingerprint scanner',
+        'ShowyouBot' => 'Showyou iOS app spider',
         'ZyBorg' => 'Zyborg? Hmmm....',
       }
     end
@@ -117,6 +136,10 @@ module Split
           if goals = value_for(settings, :goals)
             experiment_config[experiment_name.to_sym][:goals] = goals
           end
+
+          if (resettable = value_for(settings, :resettable)) != nil
+            experiment_config[experiment_name.to_sym][:resettable] = resettable
+          end
         end
 
         experiment_config
@@ -171,13 +194,14 @@ module Split
       @experiments = {}
       @persistence = Split::Persistence::SessionAdapter
       @algorithm = Split::Algorithms::WeightedSample
+      @include_rails_helper = true
     end
 
     private
 
     def value_for(hash, key)
       if hash.kind_of?(Hash)
-        hash[key.to_s] || hash[key.to_sym]
+        hash.has_key?(key.to_s) ? hash[key.to_s] : hash[key.to_sym]
       end
     end
 

data/lib/split/dashboard/views/_experiment.erb

@@ -14,9 +14,15 @@
     <% if goal.nil? %>
       <div class='inline-controls'>
         <small><%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %></small>
-        <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
-          <input type="submit" value="Reset Data">
-        </form>
+        <% if experiment.start_time %>
+          <form action="<%= url "/reset/#{experiment.name}" %>" method='post' onclick="return confirmReset()">
+            <input type="submit" value="Reset Data">
+          </form>
+        <% else%>
+          <form action="<%= url "/start/#{experiment.name}" %>" method='post'>
+            <input type="submit" value="Start">
+          </form>
+        <% end %>
         <form action="<%= url "/#{experiment.name}" %>" method='post' onclick="return confirmDelete()">
           <input type="hidden" name="_method" value="delete"/>
           <input type="submit" value="Delete" class="red">
@@ -95,4 +101,4 @@
       <td>N/A</td>
     </tr>
   </table>
-</div>
+</div>

data/lib/split/dashboard.rb

@@ -15,7 +15,8 @@ module Split
     helpers Split::DashboardHelpers
 
     get '/' do
-      @experiments = Split::Experiment.all
+      # Display experiments without a winner at the top of the dashboard
+      @experiments = Split::Experiment.all_active_first
       # Display Rails Environment mode (or Rack version if not using Rails)
       if Object.const_defined?('Rails')
         @current_env = Rails.env.titlecase
@@ -32,6 +33,12 @@ module Split
       redirect url('/')
     end
 
+    post '/start/:experiment' do
+      @experiment = Split::Experiment.find(params[:experiment])
+      @experiment.start
+      redirect url('/')
+    end
+
     post '/reset/:experiment' do
       @experiment = Split::Experiment.find(params[:experiment])
       @experiment.reset
@@ -44,4 +51,4 @@ module Split
       redirect url('/')
     end
   end
-end
+end

data/lib/split/encapsulated_helper.rb

@@ -0,0 +1,58 @@
+# Split's helper exposes all kinds of methods we don't want to
+# mix into our model classes.
+#
+# This module exposes only two methods
+#  - ab_test and 
+#  - ab_test_finished
+# that can safely be mixed into any class.
+#
+# Passes the instance of the class that it's mixed into to the
+# Split persistence adapter as context.
+#
+module Split
+  module EncapsulatedHelper
+
+    class ContextShim
+      include Split::Helper
+      def initialize(context, original_params)
+        @context = context
+        @_params = original_params
+      end
+      def ab_user
+        @ab_user ||= Split::Persistence.adapter.new(@context)
+      end
+      def params
+        @_params
+      end
+    end
+
+    def ab_test(*arguments)
+      ret = split_context_shim.ab_test(*arguments)
+      # TODO there must be a better way to pass a block straight
+      # through to the original ab_test
+      if block_given?
+        if defined?(capture) # a block in a rails view
+          block = Proc.new { yield(ret) }
+          concat(capture(ret, &block))
+          false
+        else
+          yield(ret)
+        end
+      else
+        ret
+      end
+    end
+
+    def ab_test_finished(*arguments)
+      split_context_shim.finished *arguments
+    end
+
+    private
+
+    # instantiate and memoize a context shim in case of multiple ab_test* calls
+    def split_context_shim
+      _params = defined?(params) ? params : {}
+      @split_context_shim ||= ContextShim.new(self, _params)
+    end
+  end
+end

data/lib/split/engine.rb

@@ -1,8 +1,10 @@
 module Split
   class Engine < ::Rails::Engine
     initializer "split" do |app|
-      ActionController::Base.send :include, Split::Helper
-      ActionController::Base.helper Split::Helper
+      if Split.configuration.include_rails_helper
+        ActionController::Base.send :include, Split::Helper
+        ActionController::Base.helper Split::Helper
+      end
     end
   end
-end
+end

data/lib/split/experiment.rb

@@ -41,6 +41,11 @@ module Split
       Split.redis.smembers(:experiments).map {|e| find(e)}
     end
 
+    # Return experiments without a winner (considered "active") first
+    def self.all_active_first
+      all.sort_by{|e| e.winner ? 1 : 0} # sort_by hack since true/false isn't sortable
+    end
+
     def self.find(name)
       if Split.redis.exists(name)
         obj = self.new name
@@ -65,7 +70,7 @@ module Split
 
       if new_record?
         Split.redis.sadd(:experiments, name)
-        Split.redis.hset(:experiment_start_times, @name, Time.now.to_i)
+        start unless Split.configuration.start_manually
         @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name)}
         @goals.reverse.each {|a| Split.redis.lpush(goals_key, a)} unless @goals.nil?
       else
@@ -155,6 +160,10 @@ module Split
       Split.redis.hdel(:experiment_winner, name)
     end
 
+    def start
+      Split.redis.hset(:experiment_start_times, @name, Time.now.to_i)
+    end
+
     def start_time
       t = Split.redis.hget(:experiment_start_times, @name)
       if t

data/lib/split/extensions/string.rb

@@ -1,5 +1,5 @@
 class String
-  # Constatntize is often provided by ActiveSupport, but ActiveSupport is not a dependency of Split. 
+  # Constatntize is often provided by ActiveSupport, but ActiveSupport is not a dependency of Split.
   unless method_defined?(:constantize)
     def constantize
       names = self.split('::')

data/lib/split/helper.rb

@@ -62,8 +62,7 @@ module Split
       else
         alternative_name = ab_user[experiment.key]
         trial = Trial.new(:experiment => experiment, :alternative => alternative_name, :goals => options[:goals])
-        trial.complete!
-        call_trial_complete_hook(trial)
+        call_trial_complete_hook(trial) if trial.complete!
 
         if should_reset
           reset!(experiment)
@@ -173,7 +172,7 @@ module Split
         ret = experiment.winner.name
       else
         clean_old_versions(experiment)
-        if exclude_visitor? || not_allowed_to_test?(experiment.key)
+        if exclude_visitor? || not_allowed_to_test?(experiment.key) || not_started?(experiment)
           ret = experiment.control.name
         else
           if ab_user[experiment.key]
@@ -189,6 +188,10 @@ module Split
       ret
     end
 
+    def not_started?(experiment)
+      experiment.start_time.nil?
+    end
+
     def call_trial_choose_hook(trial)
       send(Split.configuration.on_trial_choose, trial) if Split.configuration.on_trial_choose
     end

data/lib/split/version.rb

@@ -1,6 +1,6 @@
 module Split
   MAJOR   = 0
   MINOR   = 6
-  PATCH   = 5
+  PATCH   = 6
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/lib/split.rb

@@ -7,6 +7,7 @@
    helper
    metric
    persistence
+   encapsulated_helper
    trial
    version].each do |f|
   require "split/#{f}"

data/spec/algorithms/weighted_sample_spec.rb

@@ -10,7 +10,7 @@ describe Split::Algorithms::WeightedSample do
     experiment = Split::Experiment.find_or_create('link_color', {'blue' => 100}, {'red' => 0 })
     Split::Algorithms::WeightedSample.choose_alternative(experiment).name.should == 'blue'
   end
-  
+
   it "should return one of the results" do
     experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
     ['red', 'blue'].should include Split::Algorithms::WeightedSample.choose_alternative(experiment).name

data/spec/algorithms/whiplash_spec.rb

@@ -11,7 +11,7 @@ describe Split::Algorithms::Whiplash do
     experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 1 })
     ['red', 'blue'].should include Split::Algorithms::Whiplash.choose_alternative(experiment).name
   end
-  
+
   it "should guess floats" do
     Split::Algorithms::Whiplash.send(:arm_guess, 0, 0).class.should == Float
     Split::Algorithms::Whiplash.send(:arm_guess, 1, 0).class.should == Float
@@ -19,5 +19,5 @@ describe Split::Algorithms::Whiplash do
     Split::Algorithms::Whiplash.send(:arm_guess, 1000, 5).class.should == Float
     Split::Algorithms::Whiplash.send(:arm_guess, 10, -2).class.should == Float
   end
-  
+
 end

data/spec/configuration_spec.rb

@@ -86,7 +86,7 @@ describe Split::Configuration do
         end
 
         it 'should normalize experiments' do
-          @config.normalized_experiments.should == {:my_experiment=>{:alternatives=>["Control Opt", ["Alt One", "Alt Two"]]}}
+          @config.normalized_experiments.should == {:my_experiment=>{:resettable=>false,:alternatives=>["Control Opt", ["Alt One", "Alt Two"]]}}
         end
       end
 
@@ -112,7 +112,7 @@ describe Split::Configuration do
         end
 
         it "should normalize experiments" do
-          @config.normalized_experiments.should == {:my_experiment=>{:alternatives=>[{"Control Opt"=>0.67},
+          @config.normalized_experiments.should == {:my_experiment=>{:resettable=>false,:alternatives=>[{"Control Opt"=>0.67},
             [{"Alt One"=>0.1}, {"Alt Two"=>0.23}]]}, :another_experiment=>{:alternatives=>["a", ["b"]]}}
         end
 
@@ -139,7 +139,7 @@ describe Split::Configuration do
         end
 
         it "should normalize experiments" do
-          @config.normalized_experiments.should == {:my_experiment=>{:alternatives=>["Control Opt", ["Alt One", "Alt Two"]]}}
+          @config.normalized_experiments.should == {:my_experiment=>{:resettable=>false,:alternatives=>["Control Opt", ["Alt One", "Alt Two"]]}}
         end
       end
 

data/spec/dashboard_spec.rb

@@ -5,39 +5,43 @@ require 'split/dashboard'
 describe Split::Dashboard do
   include Rack::Test::Methods
 
-  let(:link_color) {
-    Split::Experiment.find_or_create('link_color', 'blue', 'red')
-  }
+  def app
+    @app ||= Split::Dashboard
+  end
 
   def link(color)
-    Split::Alternative.new(color, 'link_color')
+    Split::Alternative.new(color, experiment.name)
   end
 
-  let(:red_link) {
-    link("red")
+  let(:experiment) {
+    Split::Experiment.find_or_create('link_color', 'blue', 'red')
   }
 
-  let(:blue_link) {
-    link("blue")
-  }
-
-  def app
-    @app ||= Split::Dashboard
-  end
+  let(:red_link) { link("red") }
+  let(:blue_link) { link("blue") }
 
   it "should respond to /" do
     get '/'
     last_response.should be_ok
   end
 
-  it "should reset an experiment" do
-    experiment = link_color
+  it "should start experiment" do
+    Split.configuration.start_manually = true
+    experiment
+    get '/'
+    last_response.body.should include('Start')
 
+    post "/start/#{experiment.name}"
+    get '/'
+    last_response.body.should include('Reset Data')
+  end
+
+  it "should reset an experiment" do
     red_link.participant_count = 5
     blue_link.participant_count = 7
     experiment.winner = 'blue'
 
-    post '/reset/link_color'
+    post "/reset/#{experiment.name}"
 
     last_response.should be_redirect
 
@@ -50,17 +54,14 @@ describe Split::Dashboard do
   end
 
   it "should delete an experiment" do
-    experiment = link_color
-    delete '/link_color'
+    delete "/#{experiment.name}"
     last_response.should be_redirect
-    Split::Experiment.find('link_color').should be_nil
+    Split::Experiment.find(experiment.name).should be_nil
   end
 
   it "should mark an alternative as the winner" do
-    experiment = link_color
     experiment.winner.should be_nil
-
-    post '/link_color', :alternative => 'red'
+    post "/#{experiment.name}", :alternative => 'red'
 
     last_response.should be_redirect
     experiment.winner.name.should eql('red')
@@ -69,7 +70,7 @@ describe Split::Dashboard do
   it "should display the start date" do
     experiment_start_time = Time.parse('2011-07-07')
     Time.stub(:now => experiment_start_time)
-    experiment = link_color
+    experiment
 
     get '/'
 
@@ -79,12 +80,10 @@ describe Split::Dashboard do
   it "should handle experiments without a start date" do
     experiment_start_time = Time.parse('2011-07-07')
     Time.stub(:now => experiment_start_time)
-    experiment = link_color
-
     Split.redis.hdel(:experiment_start_times, experiment.name)
 
     get '/'
 
     last_response.body.should include('<small>Unknown</small>')
   end
-end
+end

data/spec/experiment_spec.rb

@@ -51,6 +51,13 @@ describe Split::Experiment do
       Split::Experiment.find('basket_text').start_time.should == experiment_start_time
     end
 
+    it "should not save the start time to redis when start_manually is enabled" do
+      Split.configuration.stub(:start_manually => true)
+      experiment.save
+
+      Split::Experiment.find('basket_text').start_time.should be_nil
+    end
+
     it "should save the selected algorithm to redis" do
       experiment_algorithm = Split::Algorithms::Whiplash
       experiment.algorithm = experiment_algorithm
@@ -148,7 +155,7 @@ describe Split::Experiment do
       e.should == experiment
       e.algorithm.should == Split::Algorithms::Whiplash
     end
-    
+
     it "should persist a new experiment in redis, that does not exist in the configuration file" do
       experiment = Split::Experiment.new('foobar', :alternatives => ['tra', 'la'], :algorithm => Split::Algorithms::Whiplash)
       experiment.save

data/spec/helper_spec.rb

@@ -71,6 +71,15 @@ describe Split::Helper do
       }.should_not change { e.participant_count }
     end
 
+    it 'should not increment the counter for an not started experiment' do
+      Split.configuration.stub(:start_manually => true)
+      e = Split::Experiment.find_or_create('button_size', 'small', 'big')
+      lambda {
+        a = ab_test('button_size', 'small', 'big')
+        a.should eq('small')
+      }.should_not change { e.participant_count }
+    end
+
     it "should return the given alternative for an existing user" do
       alternative = ab_test('link_color', 'blue', 'red')
       repeat_alternative = ab_test('link_color', 'blue', 'red')
@@ -261,6 +270,12 @@ describe Split::Helper do
         self.should_receive(:some_method)
         finished(@experiment_name)
       end
+
+      it "should not call the method without alternative" do
+        ab_user[@experiment.key] = nil
+        self.should_not_receive(:some_method)
+        finished(@experiment_name)
+      end
     end
 
   end
@@ -885,13 +900,15 @@ describe Split::Helper do
       end
 
       it "should increment the counter for the specified-goal completed alternative" do
-        @previous_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
-        @previous_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
-        finished({"link_color" => "purchase"})
-        new_completion_count_for_goal1 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
-        new_completion_count_for_goal1.should eql(@previous_completion_count_for_goal1 + 1)
-        new_completion_count_for_goal2 = Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
-        new_completion_count_for_goal2.should eql(@previous_completion_count_for_goal2)
+        lambda {
+          lambda {
+            finished({"link_color" => ["purchase"]})
+          }.should_not change {
+            Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
+          }
+        }.should change {
+          Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
+        }.by(1)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: split
 version: !ruby/object:Gem::Version
-  version: 0.6.5
+  version: 0.6.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-23 00:00:00.000000000 Z
+date: 2013-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -194,6 +194,7 @@ files:
 - lib/split/dashboard/views/_experiment_with_goal_header.erb
 - lib/split/dashboard/views/index.erb
 - lib/split/dashboard/views/layout.erb
+- lib/split/encapsulated_helper.rb
 - lib/split/engine.rb
 - lib/split/exceptions.rb
 - lib/split/experiment.rb