split 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5fbcc8e3abf30d53af50aa09b77ceab245d58a1
-  data.tar.gz: 0b576e850d6d066a06aaf599b0970622f22c1041
+  metadata.gz: 861fb73c59529de9a24f3a78ddd0b5cbe257eb8b
+  data.tar.gz: dd67eac616dd9435c3a0ca448de504b3824bccb8
 SHA512:
-  metadata.gz: 9a05fd38b874ffa595a76445673c3916c9d00835b641b17416fc41e48eadb731c13377e886889dbecc90aa88c1f2a6264ae11cdb33d9a41043ffde999355b8b1
-  data.tar.gz: 6491c8f886f1b90e90df207103c2fd52c545ae3f9219f1e43f10c67f2d4b15d57542eb7cb27ace064e58f9163f20938dbeb89d7a115b91ed205e28716f7e04dc
+  metadata.gz: d078b157629b473b654b751c18fe11ddae08486f38bce2fef2bdd71b7a8536ce048204adaf9b1e242e6cf76f1c33650db91994e34a23734ecaf4357663b750cf
+  data.tar.gz: aa994978184569f624b9d5bb3503b93b498ef4601bb1989fe2e16eabee07a0450adc9555c02c251d70d9a73dce65e40bac0e05bf071dc8d229e4ab87c0b817ec

data/.rubocop.yml

@@ -716,7 +716,7 @@ Style/LineEndConcatenation:
                  line end.
   Enabled: false
 
-Style/MethodCallParentheses:
+Style/MethodCallWithoutArgsParentheses:
   Description: 'Do not use parentheses for method calls with no arguments.'
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#no-args-no-parens'
   Enabled: false
@@ -816,7 +816,7 @@ Style/OneLineConditional:
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#ternary-operator'
   Enabled: false
 
-Style/OpMethod:
+Naming/BinaryOperatorParameter:
   Description: 'When defining binary operators, name the argument other.'
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#other-arg'
   Enabled: false

data/.travis.yml

@@ -5,7 +5,7 @@ rvm:
   - 2.1
   - 2.2.0
   - 2.2.2
-  - 2.4.1
+  - 2.4.2
 
 gemfile:
   - gemfiles/4.2.gemfile

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 3.2.0 (September 21st, 2017)
+
+Features:
+
+- Allow configuration of how often winning alternatives are recalculated (@patbl, #501)
+
+Bugfixes:
+
+- Avoid z_score numeric exception for conversion rates >1 (@cmantas, #503)
+- Fix combined experiments (@semanticart, #502)
+
 ## 3.1.1 (August 30th, 2017)
 
 Bugfixes:

data/README.md

@@ -115,6 +115,14 @@ As per this [blog post](http://www.evanmiller.org/how-not-to-run-an-ab-test.html
 
 The second option uses simulations from a beta distribution to determine the probability that the given alternative is the winner compared to all other alternatives. You can view these probabilities by clicking on the drop-down menu labeled "Confidence." This option should be used when the experiment has more than just 1 control and 1 alternative. It can also be used for a simple, 2-alternative A/B test.
 
+Calculating the beta-distribution simulations for a large number of experiments can be slow, so the results are cached. You can specify how often they should be recalculated (the default is once per day).
+
+```ruby
+Split.configure do |config|
+  config.winning_alternative_recalculation_interval = 3600 # 1 hour
+end
+```
+
 ## Extras
 
 ### Weighted alternatives

data/lib/split/alternative.rb

@@ -119,6 +119,9 @@ module Split
       n_a = alternative.participant_count
       n_c = control.participant_count
 
+      # can't calculate zscore for P(x) > 1
+      return 'N/A' if p_a > 1 || p_c > 1
+
       z_score = Split::Zscore.calculate(p_a, n_a, p_c, n_c)
     end
 

data/lib/split/combined_experiments_helper.rb

@@ -6,6 +6,7 @@ module Split
       raise(Split::InvalidExperimentsFormatError, 'Unable to find experiment #{metric_descriptor} in configuration') if experiment[:combined_experiments].nil?
 
       alternative = nil
+      weighted_alternatives = nil
       experiment[:combined_experiments].each do |combined_experiment|
         if alternative.nil?
           if control
@@ -15,9 +16,15 @@ module Split
             alternative = ab_test(combined_experiment, normalized_alternatives[0], *normalized_alternatives[1])
           end
         else
-          ab_test(combined_experiment, [{alternative => 1}])
+          weighted_alternatives ||= experiment[:alternatives].each_with_object({}) do |alt, memo|
+            alt = Alternative.new(alt, experiment[:name]).name
+            memo[alt] = (alt == alternative ? 1 : 0)
+          end
+
+          ab_test(combined_experiment, [weighted_alternatives])
         end
       end
+      alternative
     end
 
     def find_combined_experiment(metric_descriptor)

data/lib/split/configuration.rb

@@ -25,6 +25,7 @@ module Split
     attr_accessor :on_before_experiment_delete
     attr_accessor :include_rails_helper
     attr_accessor :beta_probability_simulations
+    attr_accessor :winning_alternative_recalculation_interval
     attr_accessor :redis
 
     attr_reader :experiments
@@ -217,6 +218,7 @@ module Split
       @algorithm = Split::Algorithms::WeightedSample
       @include_rails_helper = true
       @beta_probability_simulations = 10000
+      @winning_alternative_recalculation_interval = 60 * 60 * 24 # 1 day
       @redis = ENV.fetch(ENV.fetch('REDIS_PROVIDER', 'REDIS_URL'), 'redis://localhost:6379')
     end
 

data/lib/split/experiment.rb

@@ -262,10 +262,11 @@ module Split
     end
 
     def calc_winning_alternatives
-      # Super simple cache so that we only recalculate winning alternatives once per day
-      days_since_epoch = Time.now.utc.to_i / 86400
+      # Cache the winning alternatives so we recalculate them once per the specified interval.
+      intervals_since_epoch =
+        Time.now.utc.to_i / Split.configuration.winning_alternative_recalculation_interval
 
-      if self.calc_time != days_since_epoch
+      if self.calc_time != intervals_since_epoch
         if goals.empty?
           self.estimate_winning_alternative
         else
@@ -274,7 +275,7 @@ module Split
           end
         end
 
-        self.calc_time = days_since_epoch
+        self.calc_time = intervals_since_epoch
 
         self.save
       end

data/lib/split/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 module Split
   MAJOR   = 3
-  MINOR   = 1
-  PATCH   = 1
+  MINOR   = 2
+  PATCH   = 0
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/spec/alternative_spec.rb

@@ -273,6 +273,18 @@ describe Split::Alternative do
       expect(control.z_score(goal1)).to eq('N/A')
       expect(control.z_score(goal2)).to eq('N/A')
     end
+
+    it "should not blow up for Conversion Rates > 1" do
+      control = experiment.control
+      control.participant_count = 3474
+      control.set_completed_count(4244)
+
+      alternative2.participant_count = 3434
+      alternative2.set_completed_count(4358)
+
+      expect { control.z_score }.not_to raise_error
+      expect { alternative2.z_score }.not_to raise_error
+    end
   end
 
   describe "extra_info" do

data/spec/combined_experiments_helper_spec.rb

@@ -46,12 +46,12 @@ describe Split::CombinedExperimentsHelper do
       end
     end
 
-    it "uses same alternatives for all sub experiments " do
+    it "uses same alternative for all sub experiments and returns the alternative" do
       allow(self).to receive(:get_alternative) { "test-alt" }
       expect(self).to receive(:ab_test).with(:exp_1_click, {"control"=>0.5}, {"test-alt"=>0.5}) { "test-alt" }
-      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"test-alt" => 1}] )
+      expect(self).to receive(:ab_test).with(:exp_1_scroll, [{"control" => 0, "test-alt" => 1}])
 
-      ab_combined_test('combined_exp_1')
+      expect(ab_combined_test('combined_exp_1')).to eq('test-alt')
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: split
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-30 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -265,7 +265,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 2.0.0
 requirements: []
 rubyforge_project: split
-rubygems_version: 2.6.11
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Rack based split testing framework