split 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f98c552d0a03374e9cc58348c9c5df48b576f09
-  data.tar.gz: 0c7073b3f025cb735a19e3e33da3784e44dc7c9e
+  metadata.gz: b0a3a695f8950186c2af7c9fd33b4201f785656a
+  data.tar.gz: f24adb5a893e0cac0cbf29c06ec86d9ae692c1a2
 SHA512:
-  metadata.gz: a1a5e85d6e9461106bd991def743d9d3482aa196e6b6df029a9179fdc65a4e4c1d5fafd03efdf7b748911d3a86bb7f510e9d934c508650519bfa8b0a8d570ed8
-  data.tar.gz: 9f7ebba804fca8ba59250cfa81d0cac2c83ec97e652e2f5f456d0a4662043cbc6a3742fa9634ac06ec3316a3eb0e39e0e7b76eeef66331b65aa3c2d8716b8562
+  metadata.gz: 41138ef7911a9843cbc3fa81f78b28184087dacfff4d291d820ae32faf343fce0a9552f4d0123d647751ceadf8c1f242d44bd7e476e50a69165a49b13e422e27
+  data.tar.gz: c8cfabeb28e143872e7c81f5d3e4dd28f2cb9c8d274f21a31481eaa52725eed06876ef6959ad44aacb84c47f064c84953d2edffd4767951774011037aff28e96

data/.travis.yml

@@ -1,9 +1,8 @@
 language: ruby
 rvm:
-  - 2.3.1
+  - 2.4.0
 
 gemfile:
-  - gemfiles/4.1.gemfile
   - gemfiles/4.2.gemfile
   - gemfiles/5.0.gemfile
 

data/Appraisals

@@ -1,12 +1,8 @@
-appraise "4.1" do
-  gem "rails", "~> 4.1"
-end
-
 appraise "4.2" do
   gem "rails", "~> 4.2"
 end
 
 appraise "5.0" do
   gem "rails", "~> 5.0"
-  gem "sinatra", github: "sinatra/sinatra"
+  gem "sinatra", git: "https://github.com/sinatra/sinatra"
 end

data/CHANGELOG.md

@@ -1,5 +1,23 @@
+## 3.0.0 (March 30th, 2017)
+
+Features:
+
+- added block randomization algorithm and specs (@hulleywood, #475)
+- Add ab_record_extra_info to allow record extra info to alternative and display on dashboard. (@tranngocsam, #460)
+
+Bugfixes:
+
+- Avoid crashing on Ruby 2.4 for numeric strings (@flori, #470)
+- Fix issue where redis isn't required (@tomciopp , #466)
+
+Misc:
+
+- Avoid variable_size_secure_compare private method (@eliotsykes, #465)
+
 ## 2.2.0 (November 11th, 2016)
 
+**Backwards incompatible!** Redis keys are renamed. Please make sure all running tests are completed before you upgrade, as they will reset.
+
 Features:
 
 - Remove dependency on Redis::Namespace (@bschaeffer, #425)

data/README.md

@@ -439,9 +439,9 @@ You may want to password protect that page, you can do so with `Rack::Auth::Basi
 Split::Dashboard.use Rack::Auth::Basic do |username, password|
   # Protect against timing attacks:
   # - Use & (do not use &&) so that it doesn't short circuit.
-  # - Use `variable_size_secure_compare` to stop length information leaking
-  ActiveSupport::SecurityUtils.variable_size_secure_compare(username, ENV["SPLIT_USERNAME"]) &
-    ActiveSupport::SecurityUtils.variable_size_secure_compare(password, ENV["SPLIT_PASSWORD"])
+  # - Use digests to stop length information leaking
+  ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(username), ::Digest::SHA256.hexdigest(ENV["SPLIT_USERNAME"])) &
+    ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(password), ::Digest::SHA256.hexdigest(ENV["SPLIT_PASSWORD"]))
 end
 
 # Apps without activesupport
@@ -793,6 +793,12 @@ It is possible to specify static weights to favor certain alternatives.
 This algorithm will automatically weight the alternatives based on their relative performance,
 choosing the better-performing ones more often as trials are completed.
 
+`Split::Algorithms::BlockRandomization` is an algorithm that ensures equal
+participation across all alternatives. This algorithm will choose the alternative
+with the fewest participants. In the event of multiple minimum participant alternatives
+(i.e. starting a new "Block") the algorithm will choose a random alternative from
+those minimum participant alternatives.
+
 Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file.
 
 To change the algorithm globally for all experiments, use the following in your initializer:

data/gemfiles/5.0.gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 gem "appraisal"
 gem "codeclimate-test-reporter"
 gem "rails", "~> 5.0"
-gem "sinatra", :github => "sinatra/sinatra"
+gem "sinatra", :git => "https://github.com/sinatra/sinatra"
 
 gemspec :path => "../"

data/lib/split.rb

@@ -1,24 +1,25 @@
 # frozen_string_literal: true
-%w[algorithms
-   alternative
-   configuration
-   exceptions
-   experiment
-   experiment_catalog
-   extensions
-   goals_collection
-   helper
-   metric
-   persistence
-   encapsulated_helper
-   redis_interface
-   trial
-   user
-   version
-   zscore].each do |f|
-  require "split/#{f}"
-end
+require 'redis'
 
+require 'split/algorithms/block_randomization'
+require 'split/algorithms/weighted_sample'
+require 'split/algorithms/whiplash'
+require 'split/alternative'
+require 'split/configuration'
+require 'split/encapsulated_helper'
+require 'split/exceptions'
+require 'split/experiment'
+require 'split/experiment_catalog'
+require 'split/extensions/string'
+require 'split/goals_collection'
+require 'split/helper'
+require 'split/metric'
+require 'split/persistence'
+require 'split/redis_interface'
+require 'split/trial'
+require 'split/user'
+require 'split/version'
+require 'split/zscore'
 require 'split/engine' if defined?(Rails)
 
 module Split

data/lib/split/algorithms/block_randomization.rb

@@ -0,0 +1,22 @@
+# Selects alternative with minimum count of participants
+# If all counts are even (i.e. all are minimum), samples from all possible alternatives
+
+module Split
+  module Algorithms
+    module BlockRandomization
+      class << self
+        def choose_alternative(experiment)
+          minimum_participant_alternatives(experiment.alternatives).sample
+        end
+
+        private
+
+        def minimum_participant_alternatives(alternatives)
+          alternatives_by_count = alternatives.group_by(&:participant_count)
+          min_group = alternatives_by_count.min_by { |k, v| k }
+          min_group.last
+        end
+      end
+    end
+  end
+end

data/lib/split/alternative.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
-require 'split/zscore'
-
-# TODO - take out require and implement using file paths?
-
 module Split
   class Alternative
     attr_accessor :name
     attr_accessor :experiment_name
     attr_accessor :weight
-
-    include Zscore
+    attr_accessor :recorded_info
 
     def initialize(name, experiment_name)
       @experiment_name = experiment_name
@@ -127,10 +122,37 @@ module Split
       z_score = Split::Zscore.calculate(p_a, n_a, p_c, n_c)
     end
 
+    def extra_info
+      data = Split.redis.hget(key, 'recorded_info')
+      if data && data.length > 1
+        begin
+          JSON.parse(data)
+        rescue
+          {}
+        end
+      else
+        {}
+      end
+    end
+
+    def record_extra_info(k, value = 1)
+      @recorded_info = self.extra_info || {}
+
+      if value.kind_of?(Numeric)
+        @recorded_info[k] ||= 0
+        @recorded_info[k] += value
+      else
+        @recorded_info[k] = value
+      end
+
+      Split.redis.hset key, 'recorded_info', (@recorded_info || {}).to_json
+    end
+
     def save
       Split.redis.hsetnx key, 'participant_count', 0
       Split.redis.hsetnx key, 'completed_count', 0
       Split.redis.hsetnx key, 'p_winner', p_winner
+      Split.redis.hsetnx key, 'recorded_info', (@recorded_info || {}).to_json
     end
 
     def validate!
@@ -140,7 +162,7 @@ module Split
     end
 
     def reset
-      Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0
+      Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0, 'recorded_info', nil
       unless goals.empty?
         goals.each do |g|
           field = "completed_count:#{g}"

data/lib/split/dashboard/helpers.rb

@@ -18,7 +18,11 @@ module Split
     end
 
     def round(number, precision = 2)
-      BigDecimal.new(number.to_s).round(precision).to_f
+      begin
+        BigDecimal.new(number.to_s)
+      rescue ArgumentError
+        BigDecimal.new(0)
+      end.round(precision).to_f
     end
 
     def confidence_level(z_score)

data/lib/split/dashboard/views/_experiment.erb

@@ -5,6 +5,25 @@
 <% end %>
 
 <% experiment.calc_winning_alternatives %>
+<%
+  extra_columns = []
+  experiment.alternatives.each do |alternative|
+    extra_info = alternative.extra_info || {}
+    extra_columns += extra_info.keys
+  end
+
+  extra_columns.uniq!
+  summary_texts = {}
+  extra_columns.each do |column|
+    extra_infos = experiment.alternatives.map(&:extra_info).select{|extra_info| extra_info && extra_info[column] }
+    if extra_infos[0][column].kind_of?(Numeric)
+      summary_texts[column] = extra_infos.inject(0){|sum, extra_info| sum += extra_info[column]}
+    else
+      summary_texts[column] = "N/A"
+    end
+  end
+%>
+
 
 <div class="<%= experiment_class %>" data-name="<%= experiment.name %>" data-complete="<%= experiment.has_winner? %>">
   <div class="experiment-header">
@@ -32,6 +51,9 @@
       <th>Non-finished</th>
       <th>Completed</th>
       <th>Conversion Rate</th>
+      <% extra_columns.each do |column| %>
+        <th><%= column %></th>
+      <% end %>
       <th>
         <form>
           <select id="dropdown-<%=experiment.jstring(goal)%>" name="dropdown-<%=experiment.jstring(goal)%>">
@@ -82,6 +104,9 @@
             });
           });
         </script>
+        <% extra_columns.each do |column| %>
+          <td><%= alternative.extra_info && alternative.extra_info[column] %></td>
+        <% end %>
         <td>
           <div class="box-<%=experiment.jstring(goal)%> confidence-<%=experiment.jstring(goal)%>">
             <span title='z-score: <%= round(alternative.z_score(goal), 3) %>'><%= confidence_level(alternative.z_score(goal)) %></span>
@@ -90,7 +115,7 @@
           <div class="box-<%=experiment.jstring(goal)%> probability-<%=experiment.jstring(goal)%>">
             <span title="p_winner: <%= round(alternative.p_winner(goal), 3) %>"><%= number_to_percentage(round(alternative.p_winner(goal), 3)) %>%</span>
           </div>
-        </td>        
+        </td>
         <td>
           <% if experiment.has_winner? %>
             <% if experiment.winner.name == alternative.name %>
@@ -118,6 +143,11 @@
       <td><%= total_unfinished %></td>
       <td><%= total_completed %></td>
       <td>N/A</td>
+      <% extra_columns.each do |column| %>
+        <td>
+          <%= summary_texts[column] %>
+        </td>
+      <% end %>
       <td>N/A</td>
       <td>N/A</td>
     </tr>

data/lib/split/encapsulated_helper.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require "split/helper"
+
 # Split's helper exposes all kinds of methods we don't want to
 # mix into our model classes.
 #

data/lib/split/helper.rb

@@ -76,6 +76,26 @@ module Split
       Split.configuration.db_failover_on_db_error.call(e)
     end
 
+    def ab_record_extra_info(metric_descriptor, key, value = 1)
+      return if exclude_visitor? || Split.configuration.disabled?
+      metric_descriptor, goals = normalize_metric(metric_descriptor)
+      experiments = Metric.possible_experiments(metric_descriptor)
+
+      if experiments.any?
+        experiments.each do |experiment|
+          alternative_name = ab_user[experiment.key]
+
+          if alternative_name
+            alternative = experiment.alternatives.find{|alt| alt.name == alternative_name}
+            alternative.record_extra_info(key, value) if alternative
+          end
+        end
+      end
+    rescue => e
+      raise unless Split.configuration.db_failover
+      Split.configuration.db_failover_on_db_error.call(e)
+    end
+
     def override_present?(experiment_name)
       override_alternative(experiment_name)
     end

data/lib/split/persistence.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
-%w[session_adapter cookie_adapter redis_adapter dual_adapter].each do |f|
-  require "split/persistence/#{f}"
-end
 
 module Split
   module Persistence
+    require 'split/persistence/cookie_adapter'
+    require 'split/persistence/dual_adapter'
+    require 'split/persistence/redis_adapter'
+    require 'split/persistence/session_adapter'
+
     ADAPTERS = {
       :cookie => Split::Persistence::CookieAdapter,
       :session => Split::Persistence::SessionAdapter

data/lib/split/persistence/dual_adapter.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
+
+require 'forwardable'
+
 module Split
   module Persistence
     class DualAdapter

data/lib/split/user.rb

@@ -1,3 +1,5 @@
+require 'forwardable'
+
 module Split
   class User
     extend Forwardable

data/lib/split/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 module Split
-  MAJOR   = 2
-  MINOR   = 2
+  MAJOR   = 3
+  MINOR   = 0
   PATCH   = 0
   VERSION = [MAJOR, MINOR, PATCH].join('.')
 end

data/lib/split/zscore.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Split
-  module Zscore
+  class Zscore
 
     include Math
 

data/spec/algorithms/block_randomization_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+describe Split::Algorithms::BlockRandomization do
+
+  let(:experiment) { Split::Experiment.new 'experiment' }
+  let(:alternative_A) { Split::Alternative.new 'A', 'experiment' }
+  let(:alternative_B) { Split::Alternative.new 'B', 'experiment' }
+  let(:alternative_C) { Split::Alternative.new 'C', 'experiment' }
+
+  before :each do
+    allow(experiment).to receive(:alternatives) { [alternative_A, alternative_B, alternative_C] }
+  end
+
+  it "should return an alternative" do
+    expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment).class).to eq(Split::Alternative)
+  end
+
+  it "should always return the minimum participation option" do
+    allow(alternative_A).to receive(:participant_count) { 1 }
+    allow(alternative_B).to receive(:participant_count) { 1 }
+    allow(alternative_C).to receive(:participant_count) { 0 }
+    expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment)).to eq(alternative_C)
+  end
+
+  it "should return one of the minimum participation options when multiple" do
+    allow(alternative_A).to receive(:participant_count) { 0 }
+    allow(alternative_B).to receive(:participant_count) { 0 }
+    allow(alternative_C).to receive(:participant_count) { 0 }
+    alternative = Split::Algorithms::BlockRandomization.choose_alternative(experiment)
+    expect([alternative_A, alternative_B, alternative_C].include?(alternative)).to be(true)
+  end
+end

data/spec/alternative_spec.rb

@@ -274,4 +274,35 @@ describe Split::Alternative do
       expect(control.z_score(goal2)).to eq('N/A')
     end
   end
+
+  describe "extra_info" do
+    it "reads saved value of recorded_info in redis" do
+      saved_recorded_info = {"key_1" => 1, "key_2" => "2"}
+      Split.redis.hset "#{alternative.experiment_name}:#{alternative.name}", 'recorded_info', saved_recorded_info.to_json
+      extra_info = alternative.extra_info
+
+      expect(extra_info).to eql(saved_recorded_info)
+    end
+  end
+
+  describe "record_extra_info" do
+    it "saves key" do
+      alternative.record_extra_info("signup", 1)
+      expect(alternative.extra_info["signup"]).to eql(1)
+    end
+
+    it "adds value to saved key's value second argument is number" do
+      alternative.record_extra_info("signup", 1)
+      alternative.record_extra_info("signup", 2)
+      expect(alternative.extra_info["signup"]).to eql(3)
+    end
+
+    it "sets saved's key value to the second argument if it's a string" do
+      alternative.record_extra_info("signup", "Value 1")
+      expect(alternative.extra_info["signup"]).to eql("Value 1")
+
+      alternative.record_extra_info("signup", "Value 2")
+      expect(alternative.extra_info["signup"]).to eql("Value 2")
+    end
+  end
 end

data/spec/dashboard_helpers_spec.rb

@@ -24,5 +24,19 @@ describe Split::DashboardHelpers do
       expect(confidence_level(2.58)).to eq('99% confidence')
       expect(confidence_level(3.00)).to eq('99% confidence')
     end
+
+    describe '#round' do
+      it 'can round number strings' do
+        expect(round('3.1415')).to eq BigDecimal.new('3.14')
+      end
+
+      it 'can round number strings for precsion' do
+        expect(round('3.1415', 1)).to eq BigDecimal.new('3.1')
+      end
+
+      it 'can handle invalid number strings' do
+        expect(round('N/A')).to be_zero
+      end
+    end
   end
 end

data/spec/experiment_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 require 'spec_helper'
-require 'split/experiment'
-require 'split/algorithms'
 require 'time'
 
 describe Split::Experiment do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: split
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Andrew Nesbitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-11 00:00:00.000000000 Z
+date: 2017-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -172,11 +172,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- gemfiles/4.1.gemfile
 - gemfiles/4.2.gemfile
 - gemfiles/5.0.gemfile
 - lib/split.rb
-- lib/split/algorithms.rb
+- lib/split/algorithms/block_randomization.rb
 - lib/split/algorithms/weighted_sample.rb
 - lib/split/algorithms/whiplash.rb
 - lib/split/alternative.rb
@@ -198,7 +197,6 @@ files:
 - lib/split/exceptions.rb
 - lib/split/experiment.rb
 - lib/split/experiment_catalog.rb
-- lib/split/extensions.rb
 - lib/split/extensions/string.rb
 - lib/split/goals_collection.rb
 - lib/split/helper.rb
@@ -213,6 +211,7 @@ files:
 - lib/split/user.rb
 - lib/split/version.rb
 - lib/split/zscore.rb
+- spec/algorithms/block_randomization_spec.rb
 - spec/algorithms/weighted_sample_spec.rb
 - spec/algorithms/whiplash_spec.rb
 - spec/alternative_spec.rb
@@ -262,6 +261,7 @@ signing_key:
 specification_version: 4
 summary: Rack based split testing framework
 test_files:
+- spec/algorithms/block_randomization_spec.rb
 - spec/algorithms/weighted_sample_spec.rb
 - spec/algorithms/whiplash_spec.rb
 - spec/alternative_spec.rb

data/gemfiles/4.1.gemfile

@@ -1,9 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "appraisal"
-gem "codeclimate-test-reporter"
-gem "rails", "~> 4.1"
-
-gemspec :path => "../"

data/lib/split/algorithms.rb

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-%w[weighted_sample whiplash].each do |f|
-  require "split/algorithms/#{f}"
-end

data/lib/split/extensions.rb

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-%w[string].each do |f|
-  require "split/extensions/#{f}"
-end