split 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76d9f44d0d50f07ea46e299f93bbb5bdb1333b8b
-  data.tar.gz: 40e88a5a32652b2acbd049562810fb8c8da03da4
+  metadata.gz: 861fb73c59529de9a24f3a78ddd0b5cbe257eb8b
+  data.tar.gz: dd67eac616dd9435c3a0ca448de504b3824bccb8
 SHA512:
-  metadata.gz: c77ff217b8bdf985904c5949b7d401b479e516d6035f1e94ed50aebab5637bf3df9b9d4d22f9aa815a2c8f3a6d54c4cca66a71d4b43aa47c981ffda530a07d8f
-  data.tar.gz: 3bcacf988dbe704ca2460e681a1f0531fa1ca47486c4239d6e6db38234226b23186d55787e7bafefc8cfa9aeff12ce0400bbc6b1a83ed6166e89c9fca491711c
+  metadata.gz: d078b157629b473b654b751c18fe11ddae08486f38bce2fef2bdd71b7a8536ce048204adaf9b1e242e6cf76f1c33650db91994e34a23734ecaf4357663b750cf
+  data.tar.gz: aa994978184569f624b9d5bb3503b93b498ef4601bb1989fe2e16eabee07a0450adc9555c02c251d70d9a73dce65e40bac0e05bf071dc8d229e4ab87c0b817ec

data/.rubocop.yml

@@ -716,7 +716,7 @@ Style/LineEndConcatenation:
                  line end.
   Enabled: false
 
-Style/MethodCallParentheses:
+Style/MethodCallWithoutArgsParentheses:
   Description: 'Do not use parentheses for method calls with no arguments.'
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-args-no-parens'
   Enabled: false
@@ -816,7 +816,7 @@ Style/OneLineConditional:
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#ternary-operator'
   Enabled: false
 
-Style/OpMethod:
+Naming/BinaryOperatorParameter:
   Description: 'When defining binary operators, name the argument other.'
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#other-arg'
   Enabled: false

data/.travis.yml

@@ -1,13 +1,38 @@
 language: ruby
 rvm:
-  - 2.4.1
+  - 1.9.3
+  - 2.0
+  - 2.1
+  - 2.2.0
+  - 2.2.2
+  - 2.4.2
 
 gemfile:
   - gemfiles/4.2.gemfile
   - gemfiles/5.0.gemfile
+  - gemfiles/5.1.gemfile
+
+matrix:
+  exclude:
+  - rvm: 1.9.3
+    gemfile: gemfiles/5.0.gemfile
+  - rvm: 1.9.3
+    gemfile: gemfiles/5.1.gemfile
+  - rvm: 2.0
+    gemfile: gemfiles/5.0.gemfile
+  - rvm: 2.0
+    gemfile: gemfiles/5.1.gemfile
+  - rvm: 2.1
+    gemfile: gemfiles/5.0.gemfile
+  - rvm: 2.1
+    gemfile: gemfiles/5.1.gemfile
+  - rvm: 2.2.0
+    gemfile: gemfiles/5.0.gemfile
+  - rvm: 2.2.0
+    gemfile: gemfiles/5.1.gemfile
 
 before_install:
-  - gem install bundler
+  - gem update --system && gem install bundler
 
 script:
   - RAILS_ENV=test bundle exec rake spec && bundle exec codeclimate-test-reporter

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 3.2.0 (September 21st, 2017)
+
+Features:
+
+- Allow configuration of how often winning alternatives are recalculated (@patbl, #501)
+
+Bugfixes:
+
+- Avoid z_score numeric exception for conversion rates >1 (@cmantas, #503)
+- Fix combined experiments (@semanticart, #502)
+
+## 3.1.1 (August 30th, 2017)
+
+Bugfixes:
+
+- Bring back support for ruby 1.9.3 and greater (rubygems 2.0.0 or greater now required) (@patbl, #498)
+
+Misc:
+
+- Document testing with RSpec (@eliotsykes, #495)
+
 ## 3.1.0 (August 14th, 2017)
 
 Features:

data/README.md

@@ -1,4 +1,4 @@
-# [Split](http://libraries.io/rubygems/split) 
+# [Split](http://libraries.io/rubygems/split)
 
 [![Gem Version](https://badge.fury.io/rb/split.svg)](http://badge.fury.io/rb/split)
 [![Build Status](https://secure.travis-ci.org/splitrb/split.svg?branch=master)](http://travis-ci.org/splitrb/split)
@@ -18,7 +18,7 @@ Split is designed to be hacker friendly, allowing for maximum customisation and
 
 ### Requirements
 
-Split currently requires Ruby 1.9.2 or higher. If your project requires compatibility with Ruby 1.8.x and Rails 2.3, please use v0.8.0.
+Split currently requires Ruby 1.9.3 or higher. If your project requires compatibility with Ruby 1.8.x and Rails 2.3, please use v0.8.0.
 
 Split uses Redis as a datastore.
 
@@ -115,6 +115,14 @@ As per this [blog post](http://www.evanmiller.org/how-not-to-run-an-ab-test.html
 
 The second option uses simulations from a beta distribution to determine the probability that the given alternative is the winner compared to all other alternatives. You can view these probabilities by clicking on the drop-down menu labeled "Confidence." This option should be used when the experiment has more than just 1 control and 1 alternative. It can also be used for a simple, 2-alternative A/B test.
 
+Calculating the beta-distribution simulations for a large number of experiments can be slow, so the results are cached. You can specify how often they should be recalculated (the default is once per day).
+
+```ruby
+Split.configure do |config|
+  config.winning_alternative_recalculation_interval = 3600 # 1 hour
+end
+```
+
 ## Extras
 
 ### Weighted alternatives
@@ -150,6 +158,36 @@ In the event you want to disable all tests without having to know the individual
 
 It is not required to send `SPLIT_DISABLE=false` to activate Split.
 
+To aid testing with RSpec, write `split_helper.rb` and call `use_ab_test(alternatives_by_experiment)` in your specs as instructed below:
+
+```ruby
+# Recommended path for this file is 'spec/support/split_helper.rb', and you will need to ensure it
+# is `require`-d by rails_helper.rb or spec_helper.rb
+module SplitHelper
+
+  # Usage:
+  #
+  # Force a specific experiment alternative to always be returned:
+  #   use_ab_test(signup_form: "single_page")
+  #
+  # Force alternatives for multiple experiments:
+  #   use_ab_test(signup_form: "single_page", pricing: "show_enterprise_prices")
+  #
+  def use_ab_test(alternatives_by_experiment)
+    allow_any_instance_of(Split::Helper).to receive(:ab_test) do |_receiver, experiment|
+      alternative =
+        alternatives_by_experiment.fetch(experiment) { |key| raise "Unknown experiment '#{key}'" }
+    end
+  end
+end
+
+RSpec.configure do |config|
+  # Make the `use_ab_test` method available to all specs:
+  config.include SplitHelper
+end
+```
+
+
 ### Starting experiments manually
 
 By default new A/B tests will be active right after deployment. In case you would like to start new test a while after
@@ -625,7 +663,7 @@ Once you finish one of the goals, the test is considered to be completed, and fi
 
 #### Combined Experiments
 If you want to test how how button color affects signup *and* how it affects login, at the same time. Use combined tests
-Configure like so 
+Configure like so
 ```ruby
   Split.configuration.experiments = {
         :button_color_experiment => {
@@ -646,8 +684,8 @@ Finish each combined test as normal
    ab_finished(:button_color_on_signup)
 ```
 
-**Additional Configuration**: 
-* Be sure to enable `allow_multiple_experiments` 
+**Additional Configuration**:
+* Be sure to enable `allow_multiple_experiments`
 * In Sinatra include the CombinedExperimentsHelper
   ```
     helpers Split::CombinedExperimentsHelper

data/lib/split/alternative.rb

@@ -119,6 +119,9 @@ module Split
       n_a = alternative.participant_count
       n_c = control.participant_count
 
+      # can't calculate zscore for P(x) > 1
+      return 'N/A' if p_a > 1 || p_c > 1
+
       z_score = Split::Zscore.calculate(p_a, n_a, p_c, n_c)
     end
 

data/lib/split/combined_experiments_helper.rb

@@ -6,6 +6,7 @@ module Split
       raise(Split::InvalidExperimentsFormatError, 'Unable to find experiment #{metric_descriptor} in configuration') if experiment[:combined_experiments].nil?
 
       alternative = nil
+      weighted_alternatives = nil
       experiment[:combined_experiments].each do |combined_experiment|
         if alternative.nil?
           if control
@@ -15,9 +16,15 @@ module Split
             alternative = ab_test(combined_experiment, normalized_alternatives[0], *normalized_alternatives[1])
           end
         else
-          ab_test(combined_experiment, [{alternative => 1}])
+          weighted_alternatives ||= experiment[:alternatives].each_with_object({}) do |alt, memo|
+            alt = Alternative.new(alt, experiment[:name]).name
+            memo[alt] = (alt == alternative ? 1 : 0)
+          end
+
+          ab_test(combined_experiment, [weighted_alternatives])
         end
       end
+      alternative
     end
 
     def find_combined_experiment(metric_descriptor)

data/lib/split/configuration.rb

@@ -25,6 +25,7 @@ module Split
     attr_accessor :on_before_experiment_delete
     attr_accessor :include_rails_helper
     attr_accessor :beta_probability_simulations
+    attr_accessor :winning_alternative_recalculation_interval
     attr_accessor :redis
 
     attr_reader :experiments
@@ -217,6 +218,7 @@ module Split
       @algorithm = Split::Algorithms::WeightedSample
       @include_rails_helper = true
       @beta_probability_simulations = 10000
+      @winning_alternative_recalculation_interval = 60 * 60 * 24 # 1 day
       @redis = ENV.fetch(ENV.fetch('REDIS_PROVIDER', 'REDIS_URL'), 'redis://localhost:6379')
     end
 

data/lib/split/experiment.rb

@@ -262,10 +262,11 @@ module Split
     end
 
     def calc_winning_alternatives
-      # Super simple cache so that we only recalculate winning alternatives once per day
-      days_since_epoch = Time.now.utc.to_i / 86400
+      # Cache the winning alternatives so we recalculate them once per the specified interval.
+      intervals_since_epoch =
+        Time.now.utc.to_i / Split.configuration.winning_alternative_recalculation_interval
 
-      if self.calc_time != days_since_epoch
+      if self.calc_time != intervals_since_epoch
         if goals.empty?
           self.estimate_winning_alternative
         else
@@ -274,7 +275,7 @@ module Split
           end
         end
 
-        self.calc_time = days_since_epoch
+        self.calc_time = intervals_since_epoch
 
         self.save
       end

data/lib/split/helper.rb

@@ -10,7 +10,7 @@ module Split
         experiment = ExperimentCatalog.find_or_initialize(metric_descriptor, control, *alternatives)
         alternative = if Split.configuration.enabled
           experiment.save
-          raise(Split::InvalidExperimentsFormatError) unless Split::configuration.experiments&.dig(experiment.name.to_sym,:combined_experiments).nil?
+          raise(Split::InvalidExperimentsFormatError) unless (Split.configuration.experiments || {}).fetch(experiment.name.to_sym, {})[:combined_experiments].nil?
           trial = Trial.new(:user => ab_user, :experiment => experiment,
               :override => override_alternative(experiment.name), :exclude => exclude_visitor?,
               :disabled => split_generically_disabled?)

data/lib/split/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 module Split
   MAJOR   = 3
-  MINOR   = 1
+  MINOR   = 2
   PATCH   = 0
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/spec/alternative_spec.rb

@@ -273,6 +273,18 @@ describe Split::Alternative do
       expect(control.z_score(goal1)).to eq('N/A')
       expect(control.z_score(goal2)).to eq('N/A')
     end
+
+    it "should not blow up for Conversion Rates > 1" do
+      control = experiment.control
+      control.participant_count = 3474
+      control.set_completed_count(4244)
+
+      alternative2.participant_count = 3434
+      alternative2.set_completed_count(4358)
+
+      expect { control.z_score }.not_to raise_error
+      expect { alternative2.z_score }.not_to raise_error
+    end
   end
 
   describe "extra_info" do

data/spec/combined_experiments_helper_spec.rb

@@ -46,12 +46,12 @@ describe Split::CombinedExperimentsHelper do
       end
     end
 
-    it "uses same alternatives for all sub experiments " do
+    it "uses same alternative for all sub experiments and returns the alternative" do
       allow(self).to receive(:get_alternative) { "test-alt" }
       expect(self).to receive(:ab_test).with(:exp_1_click, {"control"=>0.5}, {"test-alt"=>0.5}) { "test-alt" }
-      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"test-alt" => 1}] )
+      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"control" => 0, "test-alt" => 1}])
 
-      ab_combined_test('combined_exp_1')
+      expect(ab_combined_test('combined_exp_1')).to eq('test-alt')
     end
   end
 end

data/spec/experiment_spec.rb

@@ -458,7 +458,7 @@ describe Split::Experiment do
       expect(experiment.alternatives[0].p_winner).to be_within(0.04).of(0.50)
     end
 
-    it "should calculate the probability of being the winning alternative separately for each goal" do
+    it "should calculate the probability of being the winning alternative separately for each goal", :skip => true do
       experiment = Split::ExperimentCatalog.find_or_create({'link_color3' => ["purchase", "refund"]}, 'blue', 'red', 'green')
       goal1 = experiment.goals[0]
       goal2 = experiment.goals[1]

data/spec/persistence/dual_adapter_spec.rb

@@ -47,7 +47,7 @@ describe Split::Persistence::DualAdapter do
   context "when logged in" do
     subject {
       described_class.with_config(
-        logged_in: -> (context) { true },
+        logged_in: lambda { |context| true },
         logged_in_adapter: selected_adapter,
         logged_out_adapter: not_selected_adapter
         ).new(context)
@@ -59,7 +59,7 @@ describe Split::Persistence::DualAdapter do
   context "when not logged in" do
     subject {
       described_class.with_config(
-        logged_in: -> (context) { false },
+        logged_in: lambda { |context| false },
         logged_in_adapter: not_selected_adapter,
         logged_out_adapter: selected_adapter
         ).new(context)

data/split.gemspec

@@ -21,7 +21,8 @@ Gem::Specification.new do |s|
    "mailing_list_uri" => "https://groups.google.com/d/forum/split-ruby"
  }
 
-  s.required_ruby_version = '>= 1.9.2'
+  s.required_ruby_version = '>= 1.9.3'
+  s.required_rubygems_version = '>= 2.0.0'
 
   s.rubyforge_project = "split"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: split
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-14 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -257,12 +257,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.0.0
 requirements: []
 rubyforge_project: split
 rubygems_version: 2.6.4