split 3.3.0 → 4.0.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.eslintrc +1 -1
- data/.github/FUNDING.yml +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +24 -0
- data/.rspec +1 -0
- data/.rubocop.yml +71 -1044
- data/.rubocop_todo.yml +226 -0
- data/.travis.yml +18 -39
- data/Appraisals +4 -0
- data/CHANGELOG.md +110 -0
- data/CODE_OF_CONDUCT.md +3 -3
- data/Gemfile +2 -0
- data/README.md +58 -23
- data/Rakefile +2 -0
- data/gemfiles/{4.2.gemfile → 6.0.gemfile} +1 -1
- data/lib/split.rb +16 -3
- data/lib/split/algorithms/block_randomization.rb +2 -0
- data/lib/split/algorithms/weighted_sample.rb +2 -1
- data/lib/split/algorithms/whiplash.rb +3 -2
- data/lib/split/alternative.rb +4 -3
- data/lib/split/cache.rb +28 -0
- data/lib/split/combined_experiments_helper.rb +3 -2
- data/lib/split/configuration.rb +15 -14
- data/lib/split/dashboard.rb +19 -1
- data/lib/split/dashboard/helpers.rb +3 -2
- data/lib/split/dashboard/pagination_helpers.rb +4 -4
- data/lib/split/dashboard/paginator.rb +1 -0
- data/lib/split/dashboard/public/dashboard.js +10 -0
- data/lib/split/dashboard/public/style.css +5 -0
- data/lib/split/dashboard/views/_controls.erb +13 -0
- data/lib/split/dashboard/views/layout.erb +1 -1
- data/lib/split/encapsulated_helper.rb +3 -2
- data/lib/split/engine.rb +7 -4
- data/lib/split/exceptions.rb +1 -0
- data/lib/split/experiment.rb +98 -65
- data/lib/split/experiment_catalog.rb +1 -3
- data/lib/split/extensions/string.rb +1 -0
- data/lib/split/goals_collection.rb +2 -0
- data/lib/split/helper.rb +30 -10
- data/lib/split/metric.rb +2 -1
- data/lib/split/persistence.rb +4 -2
- data/lib/split/persistence/cookie_adapter.rb +1 -0
- data/lib/split/persistence/dual_adapter.rb +54 -12
- data/lib/split/persistence/redis_adapter.rb +5 -0
- data/lib/split/persistence/session_adapter.rb +1 -0
- data/lib/split/redis_interface.rb +9 -28
- data/lib/split/trial.rb +25 -17
- data/lib/split/user.rb +19 -3
- data/lib/split/version.rb +2 -4
- data/lib/split/zscore.rb +1 -0
- data/spec/alternative_spec.rb +1 -1
- data/spec/cache_spec.rb +88 -0
- data/spec/configuration_spec.rb +1 -14
- data/spec/dashboard/pagination_helpers_spec.rb +3 -1
- data/spec/dashboard_helpers_spec.rb +2 -2
- data/spec/dashboard_spec.rb +78 -17
- data/spec/encapsulated_helper_spec.rb +2 -2
- data/spec/experiment_spec.rb +116 -12
- data/spec/goals_collection_spec.rb +1 -1
- data/spec/helper_spec.rb +191 -112
- data/spec/persistence/cookie_adapter_spec.rb +1 -1
- data/spec/persistence/dual_adapter_spec.rb +160 -68
- data/spec/persistence/redis_adapter_spec.rb +9 -0
- data/spec/redis_interface_spec.rb +0 -69
- data/spec/spec_helper.rb +5 -6
- data/spec/trial_spec.rb +65 -19
- data/spec/user_spec.rb +28 -0
- data/split.gemspec +9 -9
- metadata +34 -28
data/spec/configuration_spec.rb
CHANGED
@@ -212,20 +212,6 @@ describe Split::Configuration do
|
|
212
212
|
expect(@config.normalized_experiments).to eq({:my_experiment=>{:alternatives=>[{"control_opt"=>0.67}, [{"second_opt"=>0.1}, {"third_opt"=>0.23}]]}})
|
213
213
|
end
|
214
214
|
|
215
|
-
context 'redis_url configuration [DEPRECATED]' do
|
216
|
-
it 'should warn on set and assign to #redis' do
|
217
|
-
expect(@config).to receive(:warn).with(/\[DEPRECATED\]/) { nil }
|
218
|
-
@config.redis_url = 'example_url'
|
219
|
-
expect(@config.redis).to eq('example_url')
|
220
|
-
end
|
221
|
-
|
222
|
-
it 'should warn on get and return #redis' do
|
223
|
-
expect(@config).to receive(:warn).with(/\[DEPRECATED\]/) { nil }
|
224
|
-
@config.redis = 'example_url'
|
225
|
-
expect(@config.redis_url).to eq('example_url')
|
226
|
-
end
|
227
|
-
end
|
228
|
-
|
229
215
|
context "redis configuration" do
|
230
216
|
it "should default to local redis server" do
|
231
217
|
expect(@config.redis).to eq("redis://localhost:6379")
|
@@ -240,6 +226,7 @@ describe Split::Configuration do
|
|
240
226
|
it "should use the ENV variable" do
|
241
227
|
ENV['REDIS_URL'] = "env_redis_url"
|
242
228
|
expect(Split::Configuration.new.redis).to eq("env_redis_url")
|
229
|
+
ENV.delete('REDIS_URL')
|
243
230
|
end
|
244
231
|
end
|
245
232
|
end
|
@@ -10,7 +10,9 @@ describe Split::DashboardPaginationHelpers do
|
|
10
10
|
context 'when params empty' do
|
11
11
|
let(:params) { Hash[] }
|
12
12
|
|
13
|
-
it 'returns 10' do
|
13
|
+
it 'returns the default (10)' do
|
14
|
+
default_per_page = Split.configuration.dashboard_pagination_default_per_page
|
15
|
+
expect(pagination_per).to eql default_per_page
|
14
16
|
expect(pagination_per).to eql 10
|
15
17
|
end
|
16
18
|
end
|
@@ -27,11 +27,11 @@ describe Split::DashboardHelpers do
|
|
27
27
|
|
28
28
|
describe '#round' do
|
29
29
|
it 'can round number strings' do
|
30
|
-
expect(round('3.1415')).to eq BigDecimal
|
30
|
+
expect(round('3.1415')).to eq BigDecimal('3.14')
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'can round number strings for precsion' do
|
34
|
-
expect(round('3.1415', 1)).to eq BigDecimal
|
34
|
+
expect(round('3.1415', 1)).to eq BigDecimal('3.1')
|
35
35
|
end
|
36
36
|
|
37
37
|
it 'can handle invalid number strings' do
|
data/spec/dashboard_spec.rb
CHANGED
@@ -6,8 +6,16 @@ require 'split/dashboard'
|
|
6
6
|
describe Split::Dashboard do
|
7
7
|
include Rack::Test::Methods
|
8
8
|
|
9
|
+
class TestDashboard < Split::Dashboard
|
10
|
+
include Split::Helper
|
11
|
+
|
12
|
+
get '/my_experiment' do
|
13
|
+
ab_test(params[:experiment], 'blue', 'red')
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
9
17
|
def app
|
10
|
-
@app ||=
|
18
|
+
@app ||= TestDashboard
|
11
19
|
end
|
12
20
|
|
13
21
|
def link(color)
|
@@ -29,6 +37,10 @@ describe Split::Dashboard do
|
|
29
37
|
let(:red_link) { link("red") }
|
30
38
|
let(:blue_link) { link("blue") }
|
31
39
|
|
40
|
+
before(:each) do
|
41
|
+
Split.configuration.beta_probability_simulations = 1
|
42
|
+
end
|
43
|
+
|
32
44
|
it "should respond to /" do
|
33
45
|
get '/'
|
34
46
|
expect(last_response).to be_ok
|
@@ -74,17 +86,49 @@ describe Split::Dashboard do
|
|
74
86
|
end
|
75
87
|
|
76
88
|
describe "force alternative" do
|
77
|
-
|
78
|
-
|
79
|
-
|
89
|
+
context "initial version" do
|
90
|
+
let!(:user) do
|
91
|
+
Split::User.new(@app, { experiment.name => 'red' })
|
92
|
+
end
|
80
93
|
|
81
|
-
|
82
|
-
|
94
|
+
before do
|
95
|
+
allow(Split::User).to receive(:new).and_return(user)
|
96
|
+
end
|
97
|
+
|
98
|
+
it "should set current user's alternative" do
|
99
|
+
blue_link.participant_count = 7
|
100
|
+
post "/force_alternative?experiment=#{experiment.name}", alternative: "blue"
|
101
|
+
|
102
|
+
get "/my_experiment?experiment=#{experiment.name}"
|
103
|
+
expect(last_response.body).to include("blue")
|
104
|
+
end
|
105
|
+
|
106
|
+
it "should not modify an existing user" do
|
107
|
+
blue_link.participant_count = 7
|
108
|
+
post "/force_alternative?experiment=#{experiment.name}", alternative: "blue"
|
109
|
+
|
110
|
+
expect(user[experiment.key]).to eq("red")
|
111
|
+
expect(blue_link.participant_count).to eq(7)
|
112
|
+
end
|
83
113
|
end
|
84
114
|
|
85
|
-
|
86
|
-
|
87
|
-
|
115
|
+
context "incremented version" do
|
116
|
+
let!(:user) do
|
117
|
+
experiment.increment_version
|
118
|
+
Split::User.new(@app, { "#{experiment.name}:#{experiment.version}" => 'red' })
|
119
|
+
end
|
120
|
+
|
121
|
+
before do
|
122
|
+
allow(Split::User).to receive(:new).and_return(user)
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should set current user's alternative" do
|
126
|
+
blue_link.participant_count = 7
|
127
|
+
post "/force_alternative?experiment=#{experiment.name}", alternative: "blue"
|
128
|
+
|
129
|
+
get "/my_experiment?experiment=#{experiment.name}"
|
130
|
+
expect(last_response.body).to include("blue")
|
131
|
+
end
|
88
132
|
end
|
89
133
|
end
|
90
134
|
|
@@ -120,7 +164,7 @@ describe Split::Dashboard do
|
|
120
164
|
it "removes winner" do
|
121
165
|
post "/reopen?experiment=#{experiment.name}"
|
122
166
|
|
123
|
-
expect(experiment).to_not have_winner
|
167
|
+
expect(Split::ExperimentCatalog.find(experiment.name)).to_not have_winner
|
124
168
|
end
|
125
169
|
|
126
170
|
it "keeps existing stats" do
|
@@ -135,6 +179,28 @@ describe Split::Dashboard do
|
|
135
179
|
end
|
136
180
|
end
|
137
181
|
|
182
|
+
describe "update cohorting" do
|
183
|
+
it "calls enable of cohorting when action is enable" do
|
184
|
+
post "/update_cohorting?experiment=#{experiment.name}", { "cohorting_action": "enable" }
|
185
|
+
|
186
|
+
expect(experiment.cohorting_disabled?).to eq false
|
187
|
+
end
|
188
|
+
|
189
|
+
it "calls disable of cohorting when action is disable" do
|
190
|
+
post "/update_cohorting?experiment=#{experiment.name}", { "cohorting_action": "disable" }
|
191
|
+
|
192
|
+
expect(experiment.cohorting_disabled?).to eq true
|
193
|
+
end
|
194
|
+
|
195
|
+
it "calls neither enable or disable cohorting when passed invalid action" do
|
196
|
+
previous_value = experiment.cohorting_disabled?
|
197
|
+
|
198
|
+
post "/update_cohorting?experiment=#{experiment.name}", { "cohorting_action": "other" }
|
199
|
+
|
200
|
+
expect(experiment.cohorting_disabled?).to eq previous_value
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
138
204
|
it "should reset an experiment" do
|
139
205
|
red_link.participant_count = 5
|
140
206
|
blue_link.participant_count = 7
|
@@ -167,19 +233,14 @@ describe Split::Dashboard do
|
|
167
233
|
end
|
168
234
|
|
169
235
|
it "should display the start date" do
|
170
|
-
|
171
|
-
expect(Time).to receive(:now).at_least(:once).and_return(experiment_start_time)
|
172
|
-
experiment
|
236
|
+
experiment.start
|
173
237
|
|
174
238
|
get '/'
|
175
239
|
|
176
|
-
expect(last_response.body).to include(
|
240
|
+
expect(last_response.body).to include("<small>#{experiment.start_time.strftime('%Y-%m-%d')}</small>")
|
177
241
|
end
|
178
242
|
|
179
243
|
it "should handle experiments without a start date" do
|
180
|
-
experiment_start_time = Time.parse('2011-07-07')
|
181
|
-
expect(Time).to receive(:now).at_least(:once).and_return(experiment_start_time)
|
182
|
-
|
183
244
|
Split.redis.hdel(:experiment_start_times, experiment.name)
|
184
245
|
|
185
246
|
get '/'
|
@@ -21,7 +21,7 @@ describe Split::EncapsulatedHelper do
|
|
21
21
|
end
|
22
22
|
|
23
23
|
it "calls the block with selected alternative" do
|
24
|
-
expect{|block| ab_test('link_color', 'red', 'red', &block) }.to yield_with_args('red',
|
24
|
+
expect{|block| ab_test('link_color', 'red', 'red', &block) }.to yield_with_args('red', {})
|
25
25
|
end
|
26
26
|
|
27
27
|
context "inside a view" do
|
@@ -33,7 +33,7 @@ describe Split::EncapsulatedHelper do
|
|
33
33
|
static <%= alt %>
|
34
34
|
<% end %>
|
35
35
|
ERB
|
36
|
-
expect(template.result(binding)).to match
|
36
|
+
expect(template.result(binding)).to match(/foo static \d/)
|
37
37
|
end
|
38
38
|
|
39
39
|
end
|
data/spec/experiment_spec.rb
CHANGED
@@ -37,7 +37,7 @@ describe Split::Experiment do
|
|
37
37
|
|
38
38
|
it "should save to redis" do
|
39
39
|
experiment.save
|
40
|
-
expect(Split.redis.exists('basket_text')).to be true
|
40
|
+
expect(Split.redis.exists?('basket_text')).to be true
|
41
41
|
end
|
42
42
|
|
43
43
|
it "should save the start time to redis" do
|
@@ -85,8 +85,8 @@ describe Split::Experiment do
|
|
85
85
|
it "should not create duplicates when saving multiple times" do
|
86
86
|
experiment.save
|
87
87
|
experiment.save
|
88
|
-
expect(Split.redis.exists('basket_text')).to be true
|
89
|
-
expect(Split.redis.lrange('basket_text', 0, -1)).to eq(['Basket', "Cart"])
|
88
|
+
expect(Split.redis.exists?('basket_text')).to be true
|
89
|
+
expect(Split.redis.lrange('basket_text', 0, -1)).to eq(['{"Basket":1}', '{"Cart":1}'])
|
90
90
|
end
|
91
91
|
|
92
92
|
describe 'new record?' do
|
@@ -118,6 +118,23 @@ describe Split::Experiment do
|
|
118
118
|
experiment = Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :resettable => false)
|
119
119
|
expect(experiment.resettable).to be_falsey
|
120
120
|
end
|
121
|
+
|
122
|
+
context 'from configuration' do
|
123
|
+
let(:experiment_name) { :my_experiment }
|
124
|
+
let(:experiments) do
|
125
|
+
{
|
126
|
+
experiment_name => {
|
127
|
+
:alternatives => ['Control Opt', 'Alt one']
|
128
|
+
}
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
before { Split.configuration.experiments = experiments }
|
133
|
+
|
134
|
+
it 'assigns default values to the experiment' do
|
135
|
+
expect(Split::Experiment.new(experiment_name).resettable).to eq(true)
|
136
|
+
end
|
137
|
+
end
|
121
138
|
end
|
122
139
|
|
123
140
|
describe 'persistent configuration' do
|
@@ -134,10 +151,23 @@ describe Split::Experiment do
|
|
134
151
|
|
135
152
|
describe '#metadata' do
|
136
153
|
let(:experiment) { Split::Experiment.new('basket_text', :alternatives => ['Basket', "Cart"], :algorithm => Split::Algorithms::Whiplash, :metadata => meta) }
|
154
|
+
let(:meta) { { a: 'b' }}
|
155
|
+
|
156
|
+
before do
|
157
|
+
experiment.save
|
158
|
+
end
|
159
|
+
|
160
|
+
it "should delete the key when metadata is removed" do
|
161
|
+
experiment.metadata = nil
|
162
|
+
experiment.save
|
163
|
+
|
164
|
+
expect(Split.redis.exists?(experiment.metadata_key)).to be_falsey
|
165
|
+
end
|
166
|
+
|
137
167
|
context 'simple hash' do
|
138
168
|
let(:meta) { { 'basket' => 'a', 'cart' => 'b' } }
|
169
|
+
|
139
170
|
it "should persist metadata in redis" do
|
140
|
-
experiment.save
|
141
171
|
e = Split::ExperimentCatalog.find('basket_text')
|
142
172
|
expect(e).to eq(experiment)
|
143
173
|
expect(e.metadata).to eq(meta)
|
@@ -147,7 +177,6 @@ describe Split::Experiment do
|
|
147
177
|
context 'nested hash' do
|
148
178
|
let(:meta) { { 'basket' => { 'one' => 'two' }, 'cart' => 'b' } }
|
149
179
|
it "should persist metadata in redis" do
|
150
|
-
experiment.save
|
151
180
|
e = Split::ExperimentCatalog.find('basket_text')
|
152
181
|
expect(e).to eq(experiment)
|
153
182
|
expect(e.metadata).to eq(meta)
|
@@ -180,7 +209,7 @@ describe Split::Experiment do
|
|
180
209
|
experiment.save
|
181
210
|
|
182
211
|
experiment.delete
|
183
|
-
expect(Split.redis.exists('link_color')).to be false
|
212
|
+
expect(Split.redis.exists?('link_color')).to be false
|
184
213
|
expect(Split::ExperimentCatalog.find('link_color')).to be_nil
|
185
214
|
end
|
186
215
|
|
@@ -206,19 +235,59 @@ describe Split::Experiment do
|
|
206
235
|
experiment.delete
|
207
236
|
expect(experiment.start_time).to be_nil
|
208
237
|
end
|
209
|
-
end
|
210
238
|
|
239
|
+
it "should default cohorting back to false" do
|
240
|
+
experiment.disable_cohorting
|
241
|
+
expect(experiment.cohorting_disabled?).to eq(true)
|
242
|
+
experiment.delete
|
243
|
+
expect(experiment.cohorting_disabled?).to eq(false)
|
244
|
+
end
|
245
|
+
end
|
211
246
|
|
212
247
|
describe 'winner' do
|
213
248
|
it "should have no winner initially" do
|
214
249
|
expect(experiment.winner).to be_nil
|
215
250
|
end
|
251
|
+
end
|
216
252
|
|
217
|
-
|
253
|
+
describe 'winner=' do
|
254
|
+
it 'should allow you to specify a winner' do
|
218
255
|
experiment.save
|
219
256
|
experiment.winner = 'red'
|
220
257
|
expect(experiment.winner.name).to eq('red')
|
221
258
|
end
|
259
|
+
|
260
|
+
it 'should call the on_experiment_winner_choose hook' do
|
261
|
+
expect(Split.configuration.on_experiment_winner_choose).to receive(:call)
|
262
|
+
experiment.winner = 'green'
|
263
|
+
end
|
264
|
+
|
265
|
+
context 'when has_winner state is memoized' do
|
266
|
+
before { expect(experiment).to_not have_winner }
|
267
|
+
|
268
|
+
it 'should keep has_winner state consistent' do
|
269
|
+
experiment.winner = 'red'
|
270
|
+
expect(experiment).to have_winner
|
271
|
+
end
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
describe 'reset_winner' do
|
276
|
+
before { experiment.winner = 'green' }
|
277
|
+
|
278
|
+
it 'should reset the winner' do
|
279
|
+
experiment.reset_winner
|
280
|
+
expect(experiment.winner).to be_nil
|
281
|
+
end
|
282
|
+
|
283
|
+
context 'when has_winner state is memoized' do
|
284
|
+
before { expect(experiment).to have_winner }
|
285
|
+
|
286
|
+
it 'should keep has_winner state consistent' do
|
287
|
+
experiment.reset_winner
|
288
|
+
expect(experiment).to_not have_winner
|
289
|
+
end
|
290
|
+
end
|
222
291
|
end
|
223
292
|
|
224
293
|
describe 'has_winner?' do
|
@@ -235,6 +304,12 @@ describe Split::Experiment do
|
|
235
304
|
expect(experiment).to_not have_winner
|
236
305
|
end
|
237
306
|
end
|
307
|
+
|
308
|
+
it 'memoizes has_winner state' do
|
309
|
+
expect(experiment).to receive(:winner).once
|
310
|
+
expect(experiment).to_not have_winner
|
311
|
+
expect(experiment).to_not have_winner
|
312
|
+
end
|
238
313
|
end
|
239
314
|
|
240
315
|
describe 'reset' do
|
@@ -335,6 +410,22 @@ describe Split::Experiment do
|
|
335
410
|
end
|
336
411
|
end
|
337
412
|
|
413
|
+
describe "#cohorting_disabled?" do
|
414
|
+
it "returns false when nothing has been configured" do
|
415
|
+
expect(experiment.cohorting_disabled?).to eq false
|
416
|
+
end
|
417
|
+
|
418
|
+
it "returns true when enable_cohorting is performed" do
|
419
|
+
experiment.enable_cohorting
|
420
|
+
expect(experiment.cohorting_disabled?).to eq false
|
421
|
+
end
|
422
|
+
|
423
|
+
it "returns false when nothing has been configured" do
|
424
|
+
experiment.disable_cohorting
|
425
|
+
expect(experiment.cohorting_disabled?).to eq true
|
426
|
+
end
|
427
|
+
end
|
428
|
+
|
338
429
|
describe 'changing an existing experiment' do
|
339
430
|
def same_but_different_alternative
|
340
431
|
Split::ExperimentCatalog.find_or_create('link_color', 'blue', 'yellow', 'orange')
|
@@ -357,6 +448,21 @@ describe Split::Experiment do
|
|
357
448
|
expect(same_experiment_again.version).to eq(1)
|
358
449
|
end
|
359
450
|
|
451
|
+
context "when metadata is changed" do
|
452
|
+
it "should increase version" do
|
453
|
+
experiment.save
|
454
|
+
experiment.metadata = { 'foo' => 'bar' }
|
455
|
+
|
456
|
+
expect { experiment.save }.to change { experiment.version }.by(1)
|
457
|
+
end
|
458
|
+
|
459
|
+
it "does not increase version" do
|
460
|
+
experiment.metadata = nil
|
461
|
+
experiment.save
|
462
|
+
expect { experiment.save }.to change { experiment.version }.by(0)
|
463
|
+
end
|
464
|
+
end
|
465
|
+
|
360
466
|
context 'when experiment configuration is changed' do
|
361
467
|
let(:reset_manually) { false }
|
362
468
|
|
@@ -414,9 +520,7 @@ describe Split::Experiment do
|
|
414
520
|
}
|
415
521
|
|
416
522
|
context "saving experiment" do
|
417
|
-
|
418
|
-
Split::ExperimentCatalog.find_or_create({'link_color' => ["purchase", "refund"]}, 'blue', 'red', 'green')
|
419
|
-
end
|
523
|
+
let(:same_but_different_goals) { Split::ExperimentCatalog.find_or_create({'link_color' => ["purchase", "refund"]}, 'blue', 'red', 'green') }
|
420
524
|
|
421
525
|
before { experiment.save }
|
422
526
|
|
@@ -425,7 +529,7 @@ describe Split::Experiment do
|
|
425
529
|
end
|
426
530
|
|
427
531
|
it "should reset an experiment if it is loaded with different goals" do
|
428
|
-
|
532
|
+
same_but_different_goals
|
429
533
|
expect(Split::ExperimentCatalog.find("link_color").goals).to eq(["purchase", "refund"])
|
430
534
|
end
|
431
535
|
|
data/spec/helper_spec.rb
CHANGED
@@ -183,8 +183,7 @@ describe Split::Helper do
|
|
183
183
|
ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2)
|
184
184
|
experiment = Split::ExperimentCatalog.find('link_color')
|
185
185
|
expect(experiment.alternatives.map(&:name)).to eq(['blue', 'red'])
|
186
|
-
|
187
|
-
# expect(experiment.alternatives.collect{|a| a.weight}).to eq([0.01, 0.2])
|
186
|
+
expect(experiment.alternatives.collect{|a| a.weight}).to match_array([0.01, 0.2])
|
188
187
|
end
|
189
188
|
|
190
189
|
it "should only let a user participate in one experiment at a time" do
|
@@ -230,13 +229,15 @@ describe Split::Helper do
|
|
230
229
|
|
231
230
|
context "when user already has experiment" do
|
232
231
|
let(:mock_user){ Split::User.new(self, {'test_0' => 'test-alt'}) }
|
233
|
-
|
232
|
+
|
233
|
+
before do
|
234
234
|
Split.configure do |config|
|
235
235
|
config.allow_multiple_experiments = 'control'
|
236
236
|
end
|
237
|
+
|
237
238
|
Split::ExperimentCatalog.find_or_initialize('test_0', 'control', 'test-alt').save
|
238
239
|
Split::ExperimentCatalog.find_or_initialize('test_1', 'control', 'test-alt').save
|
239
|
-
|
240
|
+
end
|
240
241
|
|
241
242
|
it "should restore previously selected alternative" do
|
242
243
|
expect(ab_user.active_experiments.size).to eq 1
|
@@ -244,6 +245,16 @@ describe Split::Helper do
|
|
244
245
|
expect(ab_test(:test_0, {'control' => 1}, {"test-alt" => 100})).to eq 'test-alt'
|
245
246
|
end
|
246
247
|
|
248
|
+
it "should select the correct alternatives after experiment resets" do
|
249
|
+
experiment = Split::ExperimentCatalog.find(:test_0)
|
250
|
+
experiment.reset
|
251
|
+
mock_user[experiment.key] = 'test-alt'
|
252
|
+
|
253
|
+
expect(ab_user.active_experiments.size).to eq 1
|
254
|
+
expect(ab_test(:test_0, {'control' => 100}, {"test-alt" => 1})).to eq 'test-alt'
|
255
|
+
expect(ab_test(:test_0, {'control' => 0}, {"test-alt" => 100})).to eq 'test-alt'
|
256
|
+
end
|
257
|
+
|
247
258
|
it "lets override existing choice" do
|
248
259
|
pending "this requires user store reset on first call not depending on whelther it is current trial"
|
249
260
|
@params = { 'ab_test' => { 'test_1' => 'test-alt' } }
|
@@ -266,129 +277,187 @@ describe Split::Helper do
|
|
266
277
|
end
|
267
278
|
|
268
279
|
describe 'metadata' do
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
:
|
273
|
-
|
274
|
-
|
280
|
+
context 'is defined' do
|
281
|
+
before do
|
282
|
+
Split.configuration.experiments = {
|
283
|
+
:my_experiment => {
|
284
|
+
:alternatives => ["one", "two"],
|
285
|
+
:resettable => false,
|
286
|
+
:metadata => { 'one' => 'Meta1', 'two' => 'Meta2' }
|
287
|
+
}
|
275
288
|
}
|
276
|
-
|
277
|
-
end
|
289
|
+
end
|
278
290
|
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
291
|
+
it 'should be passed to helper block' do
|
292
|
+
@params = { 'ab_test' => { 'my_experiment' => 'two' } }
|
293
|
+
expect(ab_test('my_experiment')).to eq 'two'
|
294
|
+
expect(ab_test('my_experiment') do |alternative, meta|
|
295
|
+
meta
|
296
|
+
end).to eq('Meta2')
|
297
|
+
end
|
298
|
+
|
299
|
+
it 'should pass control metadata helper block if library disabled' do
|
300
|
+
Split.configure do |config|
|
301
|
+
config.enabled = false
|
302
|
+
end
|
303
|
+
|
304
|
+
expect(ab_test('my_experiment')).to eq 'one'
|
305
|
+
expect(ab_test('my_experiment') do |_, meta|
|
306
|
+
meta
|
307
|
+
end).to eq('Meta1')
|
308
|
+
end
|
285
309
|
end
|
286
310
|
|
287
|
-
|
288
|
-
|
289
|
-
|
311
|
+
context 'is not defined' do
|
312
|
+
before do
|
313
|
+
Split.configuration.experiments = {
|
314
|
+
:my_experiment => {
|
315
|
+
:alternatives => ["one", "two"],
|
316
|
+
:resettable => false,
|
317
|
+
:metadata => nil
|
318
|
+
}
|
319
|
+
}
|
290
320
|
end
|
291
321
|
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
322
|
+
it 'should be passed to helper block' do
|
323
|
+
expect(ab_test('my_experiment') do |alternative, meta|
|
324
|
+
meta
|
325
|
+
end).to eq({})
|
326
|
+
end
|
327
|
+
|
328
|
+
it 'should pass control metadata helper block if library disabled' do
|
329
|
+
Split.configure do |config|
|
330
|
+
config.enabled = false
|
331
|
+
end
|
332
|
+
|
333
|
+
expect(ab_test('my_experiment') do |_, meta|
|
334
|
+
meta
|
335
|
+
end).to eq({})
|
336
|
+
end
|
296
337
|
end
|
297
338
|
end
|
298
339
|
|
299
340
|
describe 'ab_finished' do
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
341
|
+
context 'for an experiment that the user participates in' do
|
342
|
+
before(:each) do
|
343
|
+
@experiment_name = 'link_color'
|
344
|
+
@alternatives = ['blue', 'red']
|
345
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
346
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
347
|
+
@previous_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
348
|
+
end
|
307
349
|
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
350
|
+
it 'should increment the counter for the completed alternative' do
|
351
|
+
ab_finished(@experiment_name)
|
352
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
353
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
354
|
+
end
|
313
355
|
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
356
|
+
it "should set experiment's finished key if reset is false" do
|
357
|
+
ab_finished(@experiment_name, {:reset => false})
|
358
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
359
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
360
|
+
end
|
319
361
|
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
362
|
+
it 'should not increment the counter if reset is false and the experiment has been already finished' do
|
363
|
+
2.times { ab_finished(@experiment_name, {:reset => false}) }
|
364
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
365
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
366
|
+
end
|
325
367
|
|
326
|
-
|
327
|
-
|
368
|
+
it 'should not increment the counter for an ended experiment' do
|
369
|
+
e = Split::ExperimentCatalog.find_or_create('button_size', 'small', 'big')
|
370
|
+
e.winner = 'small'
|
371
|
+
a = ab_test('button_size', 'small', 'big')
|
372
|
+
expect(a).to eq('small')
|
373
|
+
expect(lambda {
|
374
|
+
ab_finished('button_size')
|
375
|
+
}).not_to change { Split::Alternative.new(a, 'button_size').completed_count }
|
376
|
+
end
|
328
377
|
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
ab_finished('button_size')
|
335
|
-
}).not_to change { Split::Alternative.new('small', 'button_size').completed_count }
|
336
|
-
end
|
378
|
+
it "should clear out the user's participation from their session" do
|
379
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
380
|
+
ab_finished(@experiment_name)
|
381
|
+
expect(ab_user.keys).to be_empty
|
382
|
+
end
|
337
383
|
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
ab_finished('button_size')
|
345
|
-
}).not_to change { Split::Alternative.new(a, 'button_size').completed_count }
|
346
|
-
end
|
384
|
+
it "should not clear out the users session if reset is false" do
|
385
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
386
|
+
ab_finished(@experiment_name, {:reset => false})
|
387
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
388
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
389
|
+
end
|
347
390
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
391
|
+
it "should reset the users session when experiment is not versioned" do
|
392
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
393
|
+
ab_finished(@experiment_name)
|
394
|
+
expect(ab_user.keys).to be_empty
|
395
|
+
end
|
353
396
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
358
|
-
expect(ab_user[@experiment.finished_key]).to eq(true)
|
359
|
-
end
|
397
|
+
it "should reset the users session when experiment is versioned" do
|
398
|
+
@experiment.increment_version
|
399
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
360
400
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
end
|
401
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
402
|
+
ab_finished(@experiment_name)
|
403
|
+
expect(ab_user.keys).to be_empty
|
404
|
+
end
|
366
405
|
|
367
|
-
|
368
|
-
|
369
|
-
|
406
|
+
context "when on_trial_complete is set" do
|
407
|
+
before { Split.configuration.on_trial_complete = :some_method }
|
408
|
+
it "should call the method" do
|
409
|
+
expect(self).to receive(:some_method)
|
410
|
+
ab_finished(@experiment_name)
|
411
|
+
end
|
370
412
|
|
371
|
-
|
372
|
-
|
373
|
-
|
413
|
+
it "should not call the method without alternative" do
|
414
|
+
ab_user[@experiment.key] = nil
|
415
|
+
expect(self).not_to receive(:some_method)
|
416
|
+
ab_finished(@experiment_name)
|
417
|
+
end
|
418
|
+
end
|
374
419
|
end
|
375
420
|
|
376
|
-
|
377
|
-
|
378
|
-
|
421
|
+
context 'for an experiment that the user is excluded from' do
|
422
|
+
before do
|
423
|
+
alternative = ab_test('link_color', 'blue', 'red')
|
424
|
+
expect(Split::Alternative.new(alternative, 'link_color').participant_count).to eq(1)
|
425
|
+
alternative = ab_test('button_size', 'small', 'big')
|
426
|
+
expect(Split::Alternative.new(alternative, 'button_size').participant_count).to eq(0)
|
427
|
+
end
|
428
|
+
|
429
|
+
it 'should not increment the completed counter' do
|
430
|
+
# So, user should be participating in the link_color experiment and
|
431
|
+
# receive the control for button_size. As the user is not participating in
|
432
|
+
# the button size experiment, finishing it should not increase the
|
433
|
+
# completion count for that alternative.
|
434
|
+
expect(lambda {
|
435
|
+
ab_finished('button_size')
|
436
|
+
}).not_to change { Split::Alternative.new('small', 'button_size').completed_count }
|
437
|
+
end
|
379
438
|
end
|
380
439
|
|
381
|
-
context
|
382
|
-
before
|
383
|
-
|
384
|
-
|
385
|
-
|
440
|
+
context 'for an experiment that the user does not participate in' do
|
441
|
+
before do
|
442
|
+
Split::ExperimentCatalog.find_or_create(:not_started_experiment, 'control', 'alt')
|
443
|
+
end
|
444
|
+
it 'should not raise an exception' do
|
445
|
+
expect { ab_finished(:not_started_experiment) }.not_to raise_exception
|
386
446
|
end
|
387
447
|
|
388
|
-
it
|
389
|
-
ab_user[
|
390
|
-
|
391
|
-
|
448
|
+
it 'should not change the user state when reset is false' do
|
449
|
+
expect { ab_finished(:not_started_experiment, reset: false) }.not_to change { ab_user.keys}.from([])
|
450
|
+
end
|
451
|
+
|
452
|
+
it 'should not change the user state when reset is true' do
|
453
|
+
expect(self).not_to receive(:reset!)
|
454
|
+
ab_finished(:not_started_experiment)
|
455
|
+
end
|
456
|
+
|
457
|
+
it 'should not increment the completed counter' do
|
458
|
+
ab_finished(:not_started_experiment)
|
459
|
+
expect(Split::Alternative.new('control', :not_started_experiment).completed_count).to eq(0)
|
460
|
+
expect(Split::Alternative.new('alt', :not_started_experiment).completed_count).to eq(0)
|
392
461
|
end
|
393
462
|
end
|
394
463
|
end
|
@@ -528,6 +597,17 @@ describe Split::Helper do
|
|
528
597
|
expect(active_experiments.first[0]).to eq "link_color"
|
529
598
|
end
|
530
599
|
|
600
|
+
it 'should show versioned tests properly' do
|
601
|
+
10.times { experiment.reset }
|
602
|
+
|
603
|
+
alternative = ab_test(experiment.name, 'blue', 'red')
|
604
|
+
ab_finished(experiment.name, reset: false)
|
605
|
+
|
606
|
+
expect(experiment.version).to eq(10)
|
607
|
+
expect(active_experiments.count).to eq 1
|
608
|
+
expect(active_experiments).to eq({'link_color' => alternative })
|
609
|
+
end
|
610
|
+
|
531
611
|
it 'should show multiple tests' do
|
532
612
|
Split.configure do |config|
|
533
613
|
config.allow_multiple_experiments = true
|
@@ -545,7 +625,7 @@ describe Split::Helper do
|
|
545
625
|
end
|
546
626
|
e = Split::ExperimentCatalog.find_or_create('def', '4', '5', '6')
|
547
627
|
e.winner = '4'
|
548
|
-
|
628
|
+
ab_test('def', '4', '5', '6')
|
549
629
|
another_alternative = ab_test('ghi', '7', '8', '9')
|
550
630
|
expect(active_experiments.count).to eq 1
|
551
631
|
expect(active_experiments.first[0]).to eq "ghi"
|
@@ -564,6 +644,11 @@ describe Split::Helper do
|
|
564
644
|
expect(alternative).to eq experiment.control.name
|
565
645
|
end
|
566
646
|
|
647
|
+
it 'should not create a experiment' do
|
648
|
+
ab_test('link_color', 'blue', 'red')
|
649
|
+
expect(Split::Experiment.new('link_color')).to be_a_new_record
|
650
|
+
end
|
651
|
+
|
567
652
|
it "should not increment the participation count" do
|
568
653
|
|
569
654
|
previous_red_count = Split::Alternative.new('red', 'link_color').participant_count
|
@@ -1016,8 +1101,8 @@ describe Split::Helper do
|
|
1016
1101
|
|
1017
1102
|
it 'should handle multiple experiments correctly' do
|
1018
1103
|
experiment2 = Split::ExperimentCatalog.find_or_create('link_color2', 'blue', 'red')
|
1019
|
-
|
1020
|
-
|
1104
|
+
ab_test('link_color', 'blue', 'red')
|
1105
|
+
ab_test('link_color2', 'blue', 'red')
|
1021
1106
|
ab_finished('link_color2')
|
1022
1107
|
|
1023
1108
|
experiment2.alternatives.each do |alt|
|
@@ -1053,15 +1138,9 @@ describe Split::Helper do
|
|
1053
1138
|
end
|
1054
1139
|
|
1055
1140
|
it "should increment the counter for the specified-goal completed alternative" do
|
1056
|
-
expect(
|
1057
|
-
|
1058
|
-
|
1059
|
-
}).not_to change {
|
1060
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
|
1061
|
-
}
|
1062
|
-
}).to change {
|
1063
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
|
1064
|
-
}.by(1)
|
1141
|
+
expect{ ab_finished({"link_color" => ["purchase"]}) }
|
1142
|
+
.to change{ Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) }.by(0)
|
1143
|
+
.and change{ Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) }.by(1)
|
1065
1144
|
end
|
1066
1145
|
end
|
1067
1146
|
end
|