split 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0f98c552d0a03374e9cc58348c9c5df48b576f09
4
- data.tar.gz: 0c7073b3f025cb735a19e3e33da3784e44dc7c9e
3
+ metadata.gz: b0a3a695f8950186c2af7c9fd33b4201f785656a
4
+ data.tar.gz: f24adb5a893e0cac0cbf29c06ec86d9ae692c1a2
5
5
  SHA512:
6
- metadata.gz: a1a5e85d6e9461106bd991def743d9d3482aa196e6b6df029a9179fdc65a4e4c1d5fafd03efdf7b748911d3a86bb7f510e9d934c508650519bfa8b0a8d570ed8
7
- data.tar.gz: 9f7ebba804fca8ba59250cfa81d0cac2c83ec97e652e2f5f456d0a4662043cbc6a3742fa9634ac06ec3316a3eb0e39e0e7b76eeef66331b65aa3c2d8716b8562
6
+ metadata.gz: 41138ef7911a9843cbc3fa81f78b28184087dacfff4d291d820ae32faf343fce0a9552f4d0123d647751ceadf8c1f242d44bd7e476e50a69165a49b13e422e27
7
+ data.tar.gz: c8cfabeb28e143872e7c81f5d3e4dd28f2cb9c8d274f21a31481eaa52725eed06876ef6959ad44aacb84c47f064c84953d2edffd4767951774011037aff28e96
@@ -1,9 +1,8 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 2.3.1
3
+ - 2.4.0
4
4
 
5
5
  gemfile:
6
- - gemfiles/4.1.gemfile
7
6
  - gemfiles/4.2.gemfile
8
7
  - gemfiles/5.0.gemfile
9
8
 
data/Appraisals CHANGED
@@ -1,12 +1,8 @@
1
- appraise "4.1" do
2
- gem "rails", "~> 4.1"
3
- end
4
-
5
1
  appraise "4.2" do
6
2
  gem "rails", "~> 4.2"
7
3
  end
8
4
 
9
5
  appraise "5.0" do
10
6
  gem "rails", "~> 5.0"
11
- gem "sinatra", github: "sinatra/sinatra"
7
+ gem "sinatra", git: "https://github.com/sinatra/sinatra"
12
8
  end
@@ -1,5 +1,23 @@
1
+ ## 3.0.0 (March 30th, 2017)
2
+
3
+ Features:
4
+
5
+ - added block randomization algorithm and specs (@hulleywood, #475)
6
+ - Add ab_record_extra_info to allow record extra info to alternative and display on dashboard. (@tranngocsam, #460)
7
+
8
+ Bugfixes:
9
+
10
+ - Avoid crashing on Ruby 2.4 for numeric strings (@flori, #470)
11
+ - Fix issue where redis isn't required (@tomciopp , #466)
12
+
13
+ Misc:
14
+
15
+ - Avoid variable_size_secure_compare private method (@eliotsykes, #465)
16
+
1
17
  ## 2.2.0 (November 11th, 2016)
2
18
 
19
+ **Backwards incompatible!** Redis keys are renamed. Please make sure all running tests are completed before you upgrade, as they will reset.
20
+
3
21
  Features:
4
22
 
5
23
  - Remove dependency on Redis::Namespace (@bschaeffer, #425)
data/README.md CHANGED
@@ -439,9 +439,9 @@ You may want to password protect that page, you can do so with `Rack::Auth::Basi
439
439
  Split::Dashboard.use Rack::Auth::Basic do |username, password|
440
440
  # Protect against timing attacks:
441
441
  # - Use & (do not use &&) so that it doesn't short circuit.
442
- # - Use `variable_size_secure_compare` to stop length information leaking
443
- ActiveSupport::SecurityUtils.variable_size_secure_compare(username, ENV["SPLIT_USERNAME"]) &
444
- ActiveSupport::SecurityUtils.variable_size_secure_compare(password, ENV["SPLIT_PASSWORD"])
442
+ # - Use digests to stop length information leaking
443
+ ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(username), ::Digest::SHA256.hexdigest(ENV["SPLIT_USERNAME"])) &
444
+ ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(password), ::Digest::SHA256.hexdigest(ENV["SPLIT_PASSWORD"]))
445
445
  end
446
446
 
447
447
  # Apps without activesupport
@@ -793,6 +793,12 @@ It is possible to specify static weights to favor certain alternatives.
793
793
  This algorithm will automatically weight the alternatives based on their relative performance,
794
794
  choosing the better-performing ones more often as trials are completed.
795
795
 
796
+ `Split::Algorithms::BlockRandomization` is an algorithm that ensures equal
797
+ participation across all alternatives. This algorithm will choose the alternative
798
+ with the fewest participants. In the event of multiple minimum participant alternatives
799
+ (i.e. starting a new "Block") the algorithm will choose a random alternative from
800
+ those minimum participant alternatives.
801
+
796
802
  Users may also write their own algorithms. The default algorithm may be specified globally in the configuration file, or on a per experiment basis using the experiments hash of the configuration file.
797
803
 
798
804
  To change the algorithm globally for all experiments, use the following in your initializer:
@@ -5,6 +5,6 @@ source "https://rubygems.org"
5
5
  gem "appraisal"
6
6
  gem "codeclimate-test-reporter"
7
7
  gem "rails", "~> 5.0"
8
- gem "sinatra", :github => "sinatra/sinatra"
8
+ gem "sinatra", :git => "https://github.com/sinatra/sinatra"
9
9
 
10
10
  gemspec :path => "../"
@@ -1,24 +1,25 @@
1
1
  # frozen_string_literal: true
2
- %w[algorithms
3
- alternative
4
- configuration
5
- exceptions
6
- experiment
7
- experiment_catalog
8
- extensions
9
- goals_collection
10
- helper
11
- metric
12
- persistence
13
- encapsulated_helper
14
- redis_interface
15
- trial
16
- user
17
- version
18
- zscore].each do |f|
19
- require "split/#{f}"
20
- end
2
+ require 'redis'
21
3
 
4
+ require 'split/algorithms/block_randomization'
5
+ require 'split/algorithms/weighted_sample'
6
+ require 'split/algorithms/whiplash'
7
+ require 'split/alternative'
8
+ require 'split/configuration'
9
+ require 'split/encapsulated_helper'
10
+ require 'split/exceptions'
11
+ require 'split/experiment'
12
+ require 'split/experiment_catalog'
13
+ require 'split/extensions/string'
14
+ require 'split/goals_collection'
15
+ require 'split/helper'
16
+ require 'split/metric'
17
+ require 'split/persistence'
18
+ require 'split/redis_interface'
19
+ require 'split/trial'
20
+ require 'split/user'
21
+ require 'split/version'
22
+ require 'split/zscore'
22
23
  require 'split/engine' if defined?(Rails)
23
24
 
24
25
  module Split
@@ -0,0 +1,22 @@
1
+ # Selects alternative with minimum count of participants
2
+ # If all counts are even (i.e. all are minimum), samples from all possible alternatives
3
+
4
+ module Split
5
+ module Algorithms
6
+ module BlockRandomization
7
+ class << self
8
+ def choose_alternative(experiment)
9
+ minimum_participant_alternatives(experiment.alternatives).sample
10
+ end
11
+
12
+ private
13
+
14
+ def minimum_participant_alternatives(alternatives)
15
+ alternatives_by_count = alternatives.group_by(&:participant_count)
16
+ min_group = alternatives_by_count.min_by { |k, v| k }
17
+ min_group.last
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -1,15 +1,10 @@
1
1
  # frozen_string_literal: true
2
- require 'split/zscore'
3
-
4
- # TODO - take out require and implement using file paths?
5
-
6
2
  module Split
7
3
  class Alternative
8
4
  attr_accessor :name
9
5
  attr_accessor :experiment_name
10
6
  attr_accessor :weight
11
-
12
- include Zscore
7
+ attr_accessor :recorded_info
13
8
 
14
9
  def initialize(name, experiment_name)
15
10
  @experiment_name = experiment_name
@@ -127,10 +122,37 @@ module Split
127
122
  z_score = Split::Zscore.calculate(p_a, n_a, p_c, n_c)
128
123
  end
129
124
 
125
+ def extra_info
126
+ data = Split.redis.hget(key, 'recorded_info')
127
+ if data && data.length > 1
128
+ begin
129
+ JSON.parse(data)
130
+ rescue
131
+ {}
132
+ end
133
+ else
134
+ {}
135
+ end
136
+ end
137
+
138
+ def record_extra_info(k, value = 1)
139
+ @recorded_info = self.extra_info || {}
140
+
141
+ if value.kind_of?(Numeric)
142
+ @recorded_info[k] ||= 0
143
+ @recorded_info[k] += value
144
+ else
145
+ @recorded_info[k] = value
146
+ end
147
+
148
+ Split.redis.hset key, 'recorded_info', (@recorded_info || {}).to_json
149
+ end
150
+
130
151
  def save
131
152
  Split.redis.hsetnx key, 'participant_count', 0
132
153
  Split.redis.hsetnx key, 'completed_count', 0
133
154
  Split.redis.hsetnx key, 'p_winner', p_winner
155
+ Split.redis.hsetnx key, 'recorded_info', (@recorded_info || {}).to_json
134
156
  end
135
157
 
136
158
  def validate!
@@ -140,7 +162,7 @@ module Split
140
162
  end
141
163
 
142
164
  def reset
143
- Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0
165
+ Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0, 'recorded_info', nil
144
166
  unless goals.empty?
145
167
  goals.each do |g|
146
168
  field = "completed_count:#{g}"
@@ -18,7 +18,11 @@ module Split
18
18
  end
19
19
 
20
20
  def round(number, precision = 2)
21
- BigDecimal.new(number.to_s).round(precision).to_f
21
+ begin
22
+ BigDecimal.new(number.to_s)
23
+ rescue ArgumentError
24
+ BigDecimal.new(0)
25
+ end.round(precision).to_f
22
26
  end
23
27
 
24
28
  def confidence_level(z_score)
@@ -5,6 +5,25 @@
5
5
  <% end %>
6
6
 
7
7
  <% experiment.calc_winning_alternatives %>
8
+ <%
9
+ extra_columns = []
10
+ experiment.alternatives.each do |alternative|
11
+ extra_info = alternative.extra_info || {}
12
+ extra_columns += extra_info.keys
13
+ end
14
+
15
+ extra_columns.uniq!
16
+ summary_texts = {}
17
+ extra_columns.each do |column|
18
+ extra_infos = experiment.alternatives.map(&:extra_info).select{|extra_info| extra_info && extra_info[column] }
19
+ if extra_infos[0][column].kind_of?(Numeric)
20
+ summary_texts[column] = extra_infos.inject(0){|sum, extra_info| sum += extra_info[column]}
21
+ else
22
+ summary_texts[column] = "N/A"
23
+ end
24
+ end
25
+ %>
26
+
8
27
 
9
28
  <div class="<%= experiment_class %>" data-name="<%= experiment.name %>" data-complete="<%= experiment.has_winner? %>">
10
29
  <div class="experiment-header">
@@ -32,6 +51,9 @@
32
51
  <th>Non-finished</th>
33
52
  <th>Completed</th>
34
53
  <th>Conversion Rate</th>
54
+ <% extra_columns.each do |column| %>
55
+ <th><%= column %></th>
56
+ <% end %>
35
57
  <th>
36
58
  <form>
37
59
  <select id="dropdown-<%=experiment.jstring(goal)%>" name="dropdown-<%=experiment.jstring(goal)%>">
@@ -82,6 +104,9 @@
82
104
  });
83
105
  });
84
106
  </script>
107
+ <% extra_columns.each do |column| %>
108
+ <td><%= alternative.extra_info && alternative.extra_info[column] %></td>
109
+ <% end %>
85
110
  <td>
86
111
  <div class="box-<%=experiment.jstring(goal)%> confidence-<%=experiment.jstring(goal)%>">
87
112
  <span title='z-score: <%= round(alternative.z_score(goal), 3) %>'><%= confidence_level(alternative.z_score(goal)) %></span>
@@ -90,7 +115,7 @@
90
115
  <div class="box-<%=experiment.jstring(goal)%> probability-<%=experiment.jstring(goal)%>">
91
116
  <span title="p_winner: <%= round(alternative.p_winner(goal), 3) %>"><%= number_to_percentage(round(alternative.p_winner(goal), 3)) %>%</span>
92
117
  </div>
93
- </td>
118
+ </td>
94
119
  <td>
95
120
  <% if experiment.has_winner? %>
96
121
  <% if experiment.winner.name == alternative.name %>
@@ -118,6 +143,11 @@
118
143
  <td><%= total_unfinished %></td>
119
144
  <td><%= total_completed %></td>
120
145
  <td>N/A</td>
146
+ <% extra_columns.each do |column| %>
147
+ <td>
148
+ <%= summary_texts[column] %>
149
+ </td>
150
+ <% end %>
121
151
  <td>N/A</td>
122
152
  <td>N/A</td>
123
153
  </tr>
@@ -1,4 +1,6 @@
1
1
  # frozen_string_literal: true
2
+ require "split/helper"
3
+
2
4
  # Split's helper exposes all kinds of methods we don't want to
3
5
  # mix into our model classes.
4
6
  #
@@ -76,6 +76,26 @@ module Split
76
76
  Split.configuration.db_failover_on_db_error.call(e)
77
77
  end
78
78
 
79
+ def ab_record_extra_info(metric_descriptor, key, value = 1)
80
+ return if exclude_visitor? || Split.configuration.disabled?
81
+ metric_descriptor, goals = normalize_metric(metric_descriptor)
82
+ experiments = Metric.possible_experiments(metric_descriptor)
83
+
84
+ if experiments.any?
85
+ experiments.each do |experiment|
86
+ alternative_name = ab_user[experiment.key]
87
+
88
+ if alternative_name
89
+ alternative = experiment.alternatives.find{|alt| alt.name == alternative_name}
90
+ alternative.record_extra_info(key, value) if alternative
91
+ end
92
+ end
93
+ end
94
+ rescue => e
95
+ raise unless Split.configuration.db_failover
96
+ Split.configuration.db_failover_on_db_error.call(e)
97
+ end
98
+
79
99
  def override_present?(experiment_name)
80
100
  override_alternative(experiment_name)
81
101
  end
@@ -1,10 +1,12 @@
1
1
  # frozen_string_literal: true
2
- %w[session_adapter cookie_adapter redis_adapter dual_adapter].each do |f|
3
- require "split/persistence/#{f}"
4
- end
5
2
 
6
3
  module Split
7
4
  module Persistence
5
+ require 'split/persistence/cookie_adapter'
6
+ require 'split/persistence/dual_adapter'
7
+ require 'split/persistence/redis_adapter'
8
+ require 'split/persistence/session_adapter'
9
+
8
10
  ADAPTERS = {
9
11
  :cookie => Split::Persistence::CookieAdapter,
10
12
  :session => Split::Persistence::SessionAdapter
@@ -1,4 +1,7 @@
1
1
  # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
2
5
  module Split
3
6
  module Persistence
4
7
  class DualAdapter
@@ -1,3 +1,5 @@
1
+ require 'forwardable'
2
+
1
3
  module Split
2
4
  class User
3
5
  extend Forwardable
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
  module Split
3
- MAJOR = 2
4
- MINOR = 2
3
+ MAJOR = 3
4
+ MINOR = 0
5
5
  PATCH = 0
6
6
  VERSION = [MAJOR, MINOR, PATCH].join('.')
7
7
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
  module Split
3
- module Zscore
3
+ class Zscore
4
4
 
5
5
  include Math
6
6
 
@@ -0,0 +1,32 @@
1
+ require "spec_helper"
2
+
3
+ describe Split::Algorithms::BlockRandomization do
4
+
5
+ let(:experiment) { Split::Experiment.new 'experiment' }
6
+ let(:alternative_A) { Split::Alternative.new 'A', 'experiment' }
7
+ let(:alternative_B) { Split::Alternative.new 'B', 'experiment' }
8
+ let(:alternative_C) { Split::Alternative.new 'C', 'experiment' }
9
+
10
+ before :each do
11
+ allow(experiment).to receive(:alternatives) { [alternative_A, alternative_B, alternative_C] }
12
+ end
13
+
14
+ it "should return an alternative" do
15
+ expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment).class).to eq(Split::Alternative)
16
+ end
17
+
18
+ it "should always return the minimum participation option" do
19
+ allow(alternative_A).to receive(:participant_count) { 1 }
20
+ allow(alternative_B).to receive(:participant_count) { 1 }
21
+ allow(alternative_C).to receive(:participant_count) { 0 }
22
+ expect(Split::Algorithms::BlockRandomization.choose_alternative(experiment)).to eq(alternative_C)
23
+ end
24
+
25
+ it "should return one of the minimum participation options when multiple" do
26
+ allow(alternative_A).to receive(:participant_count) { 0 }
27
+ allow(alternative_B).to receive(:participant_count) { 0 }
28
+ allow(alternative_C).to receive(:participant_count) { 0 }
29
+ alternative = Split::Algorithms::BlockRandomization.choose_alternative(experiment)
30
+ expect([alternative_A, alternative_B, alternative_C].include?(alternative)).to be(true)
31
+ end
32
+ end
@@ -274,4 +274,35 @@ describe Split::Alternative do
274
274
  expect(control.z_score(goal2)).to eq('N/A')
275
275
  end
276
276
  end
277
+
278
+ describe "extra_info" do
279
+ it "reads saved value of recorded_info in redis" do
280
+ saved_recorded_info = {"key_1" => 1, "key_2" => "2"}
281
+ Split.redis.hset "#{alternative.experiment_name}:#{alternative.name}", 'recorded_info', saved_recorded_info.to_json
282
+ extra_info = alternative.extra_info
283
+
284
+ expect(extra_info).to eql(saved_recorded_info)
285
+ end
286
+ end
287
+
288
+ describe "record_extra_info" do
289
+ it "saves key" do
290
+ alternative.record_extra_info("signup", 1)
291
+ expect(alternative.extra_info["signup"]).to eql(1)
292
+ end
293
+
294
+ it "adds value to saved key's value second argument is number" do
295
+ alternative.record_extra_info("signup", 1)
296
+ alternative.record_extra_info("signup", 2)
297
+ expect(alternative.extra_info["signup"]).to eql(3)
298
+ end
299
+
300
+ it "sets saved's key value to the second argument if it's a string" do
301
+ alternative.record_extra_info("signup", "Value 1")
302
+ expect(alternative.extra_info["signup"]).to eql("Value 1")
303
+
304
+ alternative.record_extra_info("signup", "Value 2")
305
+ expect(alternative.extra_info["signup"]).to eql("Value 2")
306
+ end
307
+ end
277
308
  end
@@ -24,5 +24,19 @@ describe Split::DashboardHelpers do
24
24
  expect(confidence_level(2.58)).to eq('99% confidence')
25
25
  expect(confidence_level(3.00)).to eq('99% confidence')
26
26
  end
27
+
28
+ describe '#round' do
29
+ it 'can round number strings' do
30
+ expect(round('3.1415')).to eq BigDecimal.new('3.14')
31
+ end
32
+
33
+ it 'can round number strings for precsion' do
34
+ expect(round('3.1415', 1)).to eq BigDecimal.new('3.1')
35
+ end
36
+
37
+ it 'can handle invalid number strings' do
38
+ expect(round('N/A')).to be_zero
39
+ end
40
+ end
27
41
  end
28
42
  end
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
  require 'spec_helper'
3
- require 'split/experiment'
4
- require 'split/algorithms'
5
3
  require 'time'
6
4
 
7
5
  describe Split::Experiment do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: split
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Nesbitt
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-11 00:00:00.000000000 Z
11
+ date: 2017-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -172,11 +172,10 @@ files:
172
172
  - LICENSE
173
173
  - README.md
174
174
  - Rakefile
175
- - gemfiles/4.1.gemfile
176
175
  - gemfiles/4.2.gemfile
177
176
  - gemfiles/5.0.gemfile
178
177
  - lib/split.rb
179
- - lib/split/algorithms.rb
178
+ - lib/split/algorithms/block_randomization.rb
180
179
  - lib/split/algorithms/weighted_sample.rb
181
180
  - lib/split/algorithms/whiplash.rb
182
181
  - lib/split/alternative.rb
@@ -198,7 +197,6 @@ files:
198
197
  - lib/split/exceptions.rb
199
198
  - lib/split/experiment.rb
200
199
  - lib/split/experiment_catalog.rb
201
- - lib/split/extensions.rb
202
200
  - lib/split/extensions/string.rb
203
201
  - lib/split/goals_collection.rb
204
202
  - lib/split/helper.rb
@@ -213,6 +211,7 @@ files:
213
211
  - lib/split/user.rb
214
212
  - lib/split/version.rb
215
213
  - lib/split/zscore.rb
214
+ - spec/algorithms/block_randomization_spec.rb
216
215
  - spec/algorithms/weighted_sample_spec.rb
217
216
  - spec/algorithms/whiplash_spec.rb
218
217
  - spec/alternative_spec.rb
@@ -262,6 +261,7 @@ signing_key:
262
261
  specification_version: 4
263
262
  summary: Rack based split testing framework
264
263
  test_files:
264
+ - spec/algorithms/block_randomization_spec.rb
265
265
  - spec/algorithms/weighted_sample_spec.rb
266
266
  - spec/algorithms/whiplash_spec.rb
267
267
  - spec/alternative_spec.rb
@@ -1,9 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "https://rubygems.org"
4
-
5
- gem "appraisal"
6
- gem "codeclimate-test-reporter"
7
- gem "rails", "~> 4.1"
8
-
9
- gemspec :path => "../"
@@ -1,4 +0,0 @@
1
- # frozen_string_literal: true
2
- %w[weighted_sample whiplash].each do |f|
3
- require "split/algorithms/#{f}"
4
- end
@@ -1,4 +0,0 @@
1
- # frozen_string_literal: true
2
- %w[string].each do |f|
3
- require "split/extensions/#{f}"
4
- end