split 3.3.2 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.eslintrc +1 -1
- data/.github/FUNDING.yml +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +24 -0
- data/.github/dependabot.yml +7 -0
- data/.github/workflows/ci.yml +63 -0
- data/.rspec +1 -0
- data/.rubocop.yml +67 -1043
- data/CHANGELOG.md +121 -0
- data/CODE_OF_CONDUCT.md +3 -3
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +6 -1
- data/README.md +51 -21
- data/Rakefile +6 -5
- data/lib/split/algorithms/block_randomization.rb +7 -6
- data/lib/split/algorithms/weighted_sample.rb +2 -1
- data/lib/split/algorithms/whiplash.rb +17 -18
- data/lib/split/algorithms.rb +14 -0
- data/lib/split/alternative.rb +25 -25
- data/lib/split/cache.rb +27 -0
- data/lib/split/combined_experiments_helper.rb +5 -4
- data/lib/split/configuration.rb +94 -96
- data/lib/split/dashboard/helpers.rb +7 -7
- data/lib/split/dashboard/pagination_helpers.rb +56 -57
- data/lib/split/dashboard/paginator.rb +1 -0
- data/lib/split/dashboard/public/dashboard.js +10 -0
- data/lib/split/dashboard/public/style.css +10 -2
- data/lib/split/dashboard/views/_controls.erb +13 -0
- data/lib/split/dashboard/views/_experiment.erb +2 -1
- data/lib/split/dashboard/views/index.erb +19 -4
- data/lib/split/dashboard/views/layout.erb +1 -1
- data/lib/split/dashboard.rb +46 -21
- data/lib/split/encapsulated_helper.rb +15 -8
- data/lib/split/engine.rb +7 -4
- data/lib/split/exceptions.rb +1 -0
- data/lib/split/experiment.rb +160 -122
- data/lib/split/experiment_catalog.rb +7 -8
- data/lib/split/extensions/string.rb +2 -1
- data/lib/split/goals_collection.rb +10 -10
- data/lib/split/helper.rb +52 -24
- data/lib/split/metric.rb +6 -6
- data/lib/split/persistence/cookie_adapter.rb +47 -44
- data/lib/split/persistence/dual_adapter.rb +53 -12
- data/lib/split/persistence/redis_adapter.rb +8 -4
- data/lib/split/persistence/session_adapter.rb +1 -2
- data/lib/split/persistence.rb +8 -6
- data/lib/split/redis_interface.rb +16 -29
- data/lib/split/trial.rb +44 -35
- data/lib/split/user.rb +30 -15
- data/lib/split/version.rb +2 -4
- data/lib/split/zscore.rb +2 -3
- data/lib/split.rb +35 -28
- data/spec/algorithms/block_randomization_spec.rb +6 -5
- data/spec/algorithms/weighted_sample_spec.rb +6 -5
- data/spec/algorithms/whiplash_spec.rb +4 -5
- data/spec/alternative_spec.rb +35 -36
- data/spec/cache_spec.rb +84 -0
- data/spec/combined_experiments_helper_spec.rb +18 -17
- data/spec/configuration_spec.rb +41 -45
- data/spec/dashboard/pagination_helpers_spec.rb +71 -67
- data/spec/dashboard/paginator_spec.rb +10 -9
- data/spec/dashboard_helpers_spec.rb +19 -18
- data/spec/dashboard_spec.rb +153 -48
- data/spec/encapsulated_helper_spec.rb +47 -23
- data/spec/experiment_catalog_spec.rb +14 -13
- data/spec/experiment_spec.rb +224 -111
- data/spec/goals_collection_spec.rb +18 -16
- data/spec/helper_spec.rb +531 -424
- data/spec/metric_spec.rb +14 -14
- data/spec/persistence/cookie_adapter_spec.rb +26 -11
- data/spec/persistence/dual_adapter_spec.rb +158 -66
- data/spec/persistence/redis_adapter_spec.rb +35 -27
- data/spec/persistence/session_adapter_spec.rb +2 -3
- data/spec/persistence_spec.rb +1 -2
- data/spec/redis_interface_spec.rb +25 -82
- data/spec/spec_helper.rb +38 -24
- data/spec/split_spec.rb +11 -11
- data/spec/support/cookies_mock.rb +1 -2
- data/spec/trial_spec.rb +102 -75
- data/spec/user_spec.rb +69 -27
- data/split.gemspec +26 -23
- metadata +68 -42
- data/.travis.yml +0 -66
- data/Appraisals +0 -19
- data/gemfiles/4.2.gemfile +0 -9
- data/gemfiles/5.0.gemfile +0 -9
- data/gemfiles/5.1.gemfile +0 -9
- data/gemfiles/5.2.gemfile +0 -9
- data/gemfiles/6.0.gemfile +0 -9
data/spec/helper_spec.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
require "spec_helper"
|
|
3
4
|
|
|
4
5
|
# TODO change some of these tests to use Rack::Test
|
|
5
6
|
|
|
@@ -7,156 +8,156 @@ describe Split::Helper do
|
|
|
7
8
|
include Split::Helper
|
|
8
9
|
|
|
9
10
|
let(:experiment) {
|
|
10
|
-
Split::ExperimentCatalog.find_or_create(
|
|
11
|
+
Split::ExperimentCatalog.find_or_create("link_color", "blue", "red")
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
describe "ab_test" do
|
|
14
15
|
it "should not raise an error when passed strings for alternatives" do
|
|
15
|
-
expect
|
|
16
|
+
expect { ab_test("xyz", "1", "2", "3") }.not_to raise_error
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
it "should not raise an error when passed an array for alternatives" do
|
|
19
|
-
expect
|
|
20
|
+
expect { ab_test("xyz", ["1", "2", "3"]) }.not_to raise_error
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
it "should raise the appropriate error when passed integers for alternatives" do
|
|
23
|
-
expect
|
|
24
|
+
expect { ab_test("xyz", 1, 2, 3) }.to raise_error(ArgumentError)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
it "should raise the appropriate error when passed symbols for alternatives" do
|
|
27
|
-
expect
|
|
28
|
+
expect { ab_test("xyz", :a, :b, :c) }.to raise_error(ArgumentError)
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
it "should not raise error when passed an array for goals" do
|
|
31
|
-
expect
|
|
32
|
+
expect { ab_test({ "link_color" => ["purchase", "refund"] }, "blue", "red") }.not_to raise_error
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
it "should not raise error when passed just one goal" do
|
|
35
|
-
expect
|
|
36
|
+
expect { ab_test({ "link_color" => "purchase" }, "blue", "red") }.not_to raise_error
|
|
36
37
|
end
|
|
37
38
|
|
|
38
39
|
it "raises an appropriate error when processing combined expirements" do
|
|
39
40
|
Split.configuration.experiments = {
|
|
40
|
-
:
|
|
41
|
-
:
|
|
42
|
-
:
|
|
43
|
-
:
|
|
41
|
+
combined_exp_1: {
|
|
42
|
+
alternatives: [ { name: "control", percent: 50 }, { name: "test-alt", percent: 50 } ],
|
|
43
|
+
metric: :my_metric,
|
|
44
|
+
combined_experiments: [:combined_exp_1_sub_1]
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
|
-
Split::ExperimentCatalog.find_or_create(
|
|
47
|
-
expect
|
|
47
|
+
Split::ExperimentCatalog.find_or_create("combined_exp_1")
|
|
48
|
+
expect { ab_test("combined_exp_1") }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
|
|
51
|
-
ab_test(
|
|
52
|
-
expect([
|
|
52
|
+
ab_test("link_color", "blue", "red")
|
|
53
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
it "should increment the participation counter after assignment to a new user" do
|
|
56
|
-
previous_red_count = Split::Alternative.new(
|
|
57
|
-
previous_blue_count = Split::Alternative.new(
|
|
57
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
58
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
58
59
|
|
|
59
|
-
ab_test(
|
|
60
|
+
ab_test("link_color", "blue", "red")
|
|
60
61
|
|
|
61
|
-
new_red_count = Split::Alternative.new(
|
|
62
|
-
new_blue_count = Split::Alternative.new(
|
|
62
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
63
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
63
64
|
|
|
64
65
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count + 1)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
|
-
it
|
|
68
|
-
ab_test(
|
|
69
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
70
|
-
expect
|
|
68
|
+
it "should not increment the counter for an experiment that the user is not participating in" do
|
|
69
|
+
ab_test("link_color", "blue", "red")
|
|
70
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
71
|
+
expect {
|
|
71
72
|
# User shouldn't participate in this second experiment
|
|
72
|
-
ab_test(
|
|
73
|
-
}
|
|
73
|
+
ab_test("button_size", "small", "big")
|
|
74
|
+
}.not_to change { e.participant_count }
|
|
74
75
|
end
|
|
75
76
|
|
|
76
|
-
it
|
|
77
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
78
|
-
e.winner =
|
|
79
|
-
expect
|
|
80
|
-
a = ab_test(
|
|
81
|
-
expect(a).to eq(
|
|
82
|
-
}
|
|
77
|
+
it "should not increment the counter for an ended experiment" do
|
|
78
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
79
|
+
e.winner = "small"
|
|
80
|
+
expect {
|
|
81
|
+
a = ab_test("button_size", "small", "big")
|
|
82
|
+
expect(a).to eq("small")
|
|
83
|
+
}.not_to change { e.participant_count }
|
|
83
84
|
end
|
|
84
85
|
|
|
85
|
-
it
|
|
86
|
+
it "should not increment the counter for an not started experiment" do
|
|
86
87
|
expect(Split.configuration).to receive(:start_manually).and_return(true)
|
|
87
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
88
|
-
expect
|
|
89
|
-
a = ab_test(
|
|
90
|
-
expect(a).to eq(
|
|
91
|
-
}
|
|
88
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
89
|
+
expect {
|
|
90
|
+
a = ab_test("button_size", "small", "big")
|
|
91
|
+
expect(a).to eq("small")
|
|
92
|
+
}.not_to change { e.participant_count }
|
|
92
93
|
end
|
|
93
94
|
|
|
94
95
|
it "should return the given alternative for an existing user" do
|
|
95
|
-
expect(ab_test(
|
|
96
|
+
expect(ab_test("link_color", "blue", "red")).to eq ab_test("link_color", "blue", "red")
|
|
96
97
|
end
|
|
97
98
|
|
|
98
|
-
it
|
|
99
|
+
it "should always return the winner if one is present" do
|
|
99
100
|
experiment.winner = "orange"
|
|
100
101
|
|
|
101
|
-
expect(ab_test(
|
|
102
|
+
expect(ab_test("link_color", "blue", "red")).to eq("orange")
|
|
102
103
|
end
|
|
103
104
|
|
|
104
105
|
it "should allow the alternative to be forced by passing it in the params" do
|
|
105
106
|
# ?ab_test[link_color]=blue
|
|
106
|
-
@params = {
|
|
107
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
107
108
|
|
|
108
|
-
alternative = ab_test(
|
|
109
|
-
expect(alternative).to eq(
|
|
109
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
110
|
+
expect(alternative).to eq("blue")
|
|
110
111
|
|
|
111
|
-
alternative = ab_test(
|
|
112
|
-
expect(alternative).to eq(
|
|
112
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
113
|
+
expect(alternative).to eq("blue")
|
|
113
114
|
|
|
114
|
-
@params = {
|
|
115
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
115
116
|
|
|
116
|
-
alternative = ab_test(
|
|
117
|
-
expect(alternative).to eq(
|
|
117
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
118
|
+
expect(alternative).to eq("red")
|
|
118
119
|
|
|
119
|
-
alternative = ab_test(
|
|
120
|
-
expect(alternative).to eq(
|
|
120
|
+
alternative = ab_test("link_color", { "blue" => 5 }, "red" => 1)
|
|
121
|
+
expect(alternative).to eq("red")
|
|
121
122
|
end
|
|
122
123
|
|
|
123
124
|
it "should not allow an arbitrary alternative" do
|
|
124
|
-
@params = {
|
|
125
|
-
alternative = ab_test(
|
|
126
|
-
expect(alternative).to eq(
|
|
125
|
+
@params = { "ab_test" => { "link_color" => "pink" } }
|
|
126
|
+
alternative = ab_test("link_color", "blue")
|
|
127
|
+
expect(alternative).to eq("blue")
|
|
127
128
|
end
|
|
128
129
|
|
|
129
130
|
it "should not store the split when a param forced alternative" do
|
|
130
|
-
@params = {
|
|
131
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
131
132
|
expect(ab_user).not_to receive(:[]=)
|
|
132
|
-
ab_test(
|
|
133
|
+
ab_test("link_color", "blue", "red")
|
|
133
134
|
end
|
|
134
135
|
|
|
135
136
|
it "SPLIT_DISABLE query parameter should also force the alternative (uses control)" do
|
|
136
|
-
@params = {
|
|
137
|
-
alternative = ab_test(
|
|
138
|
-
expect(alternative).to eq(
|
|
139
|
-
alternative = ab_test(
|
|
140
|
-
expect(alternative).to eq(
|
|
141
|
-
alternative = ab_test(
|
|
142
|
-
expect(alternative).to eq(
|
|
143
|
-
alternative = ab_test(
|
|
144
|
-
expect(alternative).to eq(
|
|
137
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
138
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
139
|
+
expect(alternative).to eq("blue")
|
|
140
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
141
|
+
expect(alternative).to eq("blue")
|
|
142
|
+
alternative = ab_test("link_color", "red", "blue")
|
|
143
|
+
expect(alternative).to eq("red")
|
|
144
|
+
alternative = ab_test("link_color", { "red" => 5 }, "blue" => 1)
|
|
145
|
+
expect(alternative).to eq("red")
|
|
145
146
|
end
|
|
146
147
|
|
|
147
148
|
it "should not store the split when Split generically disabled" do
|
|
148
|
-
@params = {
|
|
149
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
149
150
|
expect(ab_user).not_to receive(:[]=)
|
|
150
|
-
ab_test(
|
|
151
|
+
ab_test("link_color", "blue", "red")
|
|
151
152
|
end
|
|
152
153
|
|
|
153
154
|
context "when store_override is set" do
|
|
154
155
|
before { Split.configuration.store_override = true }
|
|
155
156
|
|
|
156
157
|
it "should store the forced alternative" do
|
|
157
|
-
@params = {
|
|
158
|
-
expect(ab_user).to receive(:[]=).with(
|
|
159
|
-
ab_test(
|
|
158
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
159
|
+
expect(ab_user).to receive(:[]=).with("link_color", "blue")
|
|
160
|
+
ab_test("link_color", "blue", "red")
|
|
160
161
|
end
|
|
161
162
|
end
|
|
162
163
|
|
|
@@ -164,36 +165,35 @@ describe Split::Helper do
|
|
|
164
165
|
before { Split.configuration.on_trial_choose = :some_method }
|
|
165
166
|
it "should call the method" do
|
|
166
167
|
expect(self).to receive(:some_method)
|
|
167
|
-
ab_test(
|
|
168
|
+
ab_test("link_color", "blue", "red")
|
|
168
169
|
end
|
|
169
170
|
end
|
|
170
171
|
|
|
171
172
|
it "should allow passing a block" do
|
|
172
|
-
alt = ab_test(
|
|
173
|
-
ret = ab_test(
|
|
173
|
+
alt = ab_test("link_color", "blue", "red")
|
|
174
|
+
ret = ab_test("link_color", "blue", "red") { |alternative| "shared/#{alternative}" }
|
|
174
175
|
expect(ret).to eq("shared/#{alt}")
|
|
175
176
|
end
|
|
176
177
|
|
|
177
178
|
it "should allow the share of visitors see an alternative to be specified" do
|
|
178
|
-
ab_test(
|
|
179
|
-
expect([
|
|
179
|
+
ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })
|
|
180
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
180
181
|
end
|
|
181
182
|
|
|
182
183
|
it "should allow alternative weighting interface as a single hash" do
|
|
183
|
-
ab_test(
|
|
184
|
-
experiment = Split::ExperimentCatalog.find(
|
|
185
|
-
expect(experiment.alternatives.map(&:name)).to eq([
|
|
186
|
-
|
|
187
|
-
# expect(experiment.alternatives.collect{|a| a.weight}).to eq([0.01, 0.2])
|
|
184
|
+
ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)
|
|
185
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
186
|
+
expect(experiment.alternatives.map(&:name)).to eq(["blue", "red"])
|
|
187
|
+
expect(experiment.alternatives.collect { |a| a.weight }).to match_array([0.01, 0.2])
|
|
188
188
|
end
|
|
189
189
|
|
|
190
190
|
it "should only let a user participate in one experiment at a time" do
|
|
191
|
-
link_color = ab_test(
|
|
192
|
-
ab_test(
|
|
193
|
-
expect(ab_user[
|
|
194
|
-
big = Split::Alternative.new(
|
|
191
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
192
|
+
ab_test("button_size", "small", "big")
|
|
193
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
194
|
+
big = Split::Alternative.new("big", "button_size")
|
|
195
195
|
expect(big.participant_count).to eq(0)
|
|
196
|
-
small = Split::Alternative.new(
|
|
196
|
+
small = Split::Alternative.new("small", "button_size")
|
|
197
197
|
expect(small.participant_count).to eq(0)
|
|
198
198
|
end
|
|
199
199
|
|
|
@@ -201,194 +201,262 @@ describe Split::Helper do
|
|
|
201
201
|
Split.configure do |config|
|
|
202
202
|
config.allow_multiple_experiments = true
|
|
203
203
|
end
|
|
204
|
-
link_color = ab_test(
|
|
205
|
-
button_size = ab_test(
|
|
206
|
-
expect(ab_user[
|
|
207
|
-
expect(ab_user[
|
|
208
|
-
button_size_alt = Split::Alternative.new(button_size,
|
|
204
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
205
|
+
button_size = ab_test("button_size", "small", "big")
|
|
206
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
207
|
+
expect(ab_user["button_size"]).to eq(button_size)
|
|
208
|
+
button_size_alt = Split::Alternative.new(button_size, "button_size")
|
|
209
209
|
expect(button_size_alt.participant_count).to eq(1)
|
|
210
210
|
end
|
|
211
211
|
|
|
212
212
|
context "with allow_multiple_experiments = 'control'" do
|
|
213
213
|
it "should let a user participate in many experiment with one non-'control' alternative" do
|
|
214
214
|
Split.configure do |config|
|
|
215
|
-
config.allow_multiple_experiments =
|
|
215
|
+
config.allow_multiple_experiments = "control"
|
|
216
216
|
end
|
|
217
217
|
groups = 100.times.map do |n|
|
|
218
|
-
ab_test("test#{n}".to_sym, {
|
|
218
|
+
ab_test("test#{n}".to_sym, { "control" => (100 - n) }, { "test#{n}-alt" => n })
|
|
219
219
|
end
|
|
220
220
|
|
|
221
221
|
experiments = ab_user.active_experiments
|
|
222
222
|
expect(experiments.size).to be > 1
|
|
223
223
|
|
|
224
|
-
count_control = experiments.values.count { |g| g ==
|
|
224
|
+
count_control = experiments.values.count { |g| g == "control" }
|
|
225
225
|
expect(count_control).to eq(experiments.size - 1)
|
|
226
226
|
|
|
227
|
-
count_alts = groups.count { |g| g !=
|
|
227
|
+
count_alts = groups.count { |g| g != "control" }
|
|
228
228
|
expect(count_alts).to eq(1)
|
|
229
229
|
end
|
|
230
230
|
|
|
231
231
|
context "when user already has experiment" do
|
|
232
|
-
let(:mock_user){ Split::User.new(self, {
|
|
233
|
-
|
|
232
|
+
let(:mock_user) { Split::User.new(self, { "test_0" => "test-alt" }) }
|
|
233
|
+
|
|
234
|
+
before do
|
|
234
235
|
Split.configure do |config|
|
|
235
|
-
config.allow_multiple_experiments =
|
|
236
|
+
config.allow_multiple_experiments = "control"
|
|
236
237
|
end
|
|
237
|
-
|
|
238
|
-
Split::ExperimentCatalog.find_or_initialize(
|
|
239
|
-
|
|
238
|
+
|
|
239
|
+
Split::ExperimentCatalog.find_or_initialize("test_0", "control", "test-alt").save
|
|
240
|
+
Split::ExperimentCatalog.find_or_initialize("test_1", "control", "test-alt").save
|
|
241
|
+
end
|
|
240
242
|
|
|
241
243
|
it "should restore previously selected alternative" do
|
|
242
244
|
expect(ab_user.active_experiments.size).to eq 1
|
|
243
|
-
expect(ab_test(:test_0, {
|
|
244
|
-
expect(ab_test(:test_0, {
|
|
245
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
246
|
+
expect(ab_test(:test_0, { "control" => 1 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
it "should select the correct alternatives after experiment resets" do
|
|
250
|
+
experiment = Split::ExperimentCatalog.find(:test_0)
|
|
251
|
+
experiment.reset
|
|
252
|
+
mock_user[experiment.key] = "test-alt"
|
|
253
|
+
|
|
254
|
+
expect(ab_user.active_experiments.size).to eq 1
|
|
255
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
256
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
245
257
|
end
|
|
246
258
|
|
|
247
259
|
it "lets override existing choice" do
|
|
248
260
|
pending "this requires user store reset on first call not depending on whelther it is current trial"
|
|
249
|
-
@params = {
|
|
261
|
+
@params = { "ab_test" => { "test_1" => "test-alt" } }
|
|
250
262
|
|
|
251
|
-
expect(ab_test(:test_0, {
|
|
252
|
-
expect(ab_test(:test_1, {
|
|
263
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "control"
|
|
264
|
+
expect(ab_test(:test_1, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
253
265
|
end
|
|
254
|
-
|
|
255
266
|
end
|
|
256
|
-
|
|
257
267
|
end
|
|
258
268
|
|
|
259
269
|
it "should not over-write a finished key when an experiment is on a later version" do
|
|
260
270
|
experiment.increment_version
|
|
261
|
-
ab_user = { experiment.key =>
|
|
271
|
+
ab_user = { experiment.key => "blue", experiment.finished_key => true }
|
|
262
272
|
finished_session = ab_user.dup
|
|
263
|
-
ab_test(
|
|
273
|
+
ab_test("link_color", "blue", "red")
|
|
264
274
|
expect(ab_user).to eq(finished_session)
|
|
265
275
|
end
|
|
266
276
|
end
|
|
267
277
|
|
|
268
|
-
describe
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
:
|
|
273
|
-
|
|
274
|
-
|
|
278
|
+
describe "metadata" do
|
|
279
|
+
context "is defined" do
|
|
280
|
+
before do
|
|
281
|
+
Split.configuration.experiments = {
|
|
282
|
+
my_experiment: {
|
|
283
|
+
alternatives: ["one", "two"],
|
|
284
|
+
resettable: false,
|
|
285
|
+
metadata: { "one" => "Meta1", "two" => "Meta2" }
|
|
286
|
+
}
|
|
275
287
|
}
|
|
276
|
-
|
|
277
|
-
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
it "should be passed to helper block" do
|
|
291
|
+
@params = { "ab_test" => { "my_experiment" => "two" } }
|
|
292
|
+
expect(ab_test("my_experiment")).to eq "two"
|
|
293
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
294
|
+
meta
|
|
295
|
+
end).to eq("Meta2")
|
|
296
|
+
end
|
|
278
297
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
298
|
+
it "should pass control metadata helper block if library disabled" do
|
|
299
|
+
Split.configure do |config|
|
|
300
|
+
config.enabled = false
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
expect(ab_test("my_experiment")).to eq "one"
|
|
304
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
305
|
+
meta
|
|
306
|
+
end).to eq("Meta1")
|
|
307
|
+
end
|
|
285
308
|
end
|
|
286
309
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
310
|
+
context "is not defined" do
|
|
311
|
+
before do
|
|
312
|
+
Split.configuration.experiments = {
|
|
313
|
+
my_experiment: {
|
|
314
|
+
alternatives: ["one", "two"],
|
|
315
|
+
resettable: false,
|
|
316
|
+
metadata: nil
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
it "should be passed to helper block" do
|
|
322
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
323
|
+
meta
|
|
324
|
+
end).to eq({})
|
|
290
325
|
end
|
|
291
326
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
327
|
+
it "should pass control metadata helper block if library disabled" do
|
|
328
|
+
Split.configure do |config|
|
|
329
|
+
config.enabled = false
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
333
|
+
meta
|
|
334
|
+
end).to eq({})
|
|
335
|
+
end
|
|
296
336
|
end
|
|
297
337
|
end
|
|
298
338
|
|
|
299
|
-
describe
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
339
|
+
describe "ab_finished" do
|
|
340
|
+
context "for an experiment that the user participates in" do
|
|
341
|
+
before(:each) do
|
|
342
|
+
@experiment_name = "link_color"
|
|
343
|
+
@alternatives = ["blue", "red"]
|
|
344
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
345
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
346
|
+
@previous_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
347
|
+
end
|
|
307
348
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
349
|
+
it "should increment the counter for the completed alternative" do
|
|
350
|
+
ab_finished(@experiment_name)
|
|
351
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
352
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
353
|
+
end
|
|
313
354
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
355
|
+
it "should set experiment's finished key if reset is false" do
|
|
356
|
+
ab_finished(@experiment_name, { reset: false })
|
|
357
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
358
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
359
|
+
end
|
|
319
360
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
361
|
+
it "should not increment the counter if reset is false and the experiment has been already finished" do
|
|
362
|
+
2.times { ab_finished(@experiment_name, { reset: false }) }
|
|
363
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
364
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
365
|
+
end
|
|
325
366
|
|
|
326
|
-
|
|
327
|
-
|
|
367
|
+
it "should not increment the counter for an ended experiment" do
|
|
368
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
369
|
+
e.winner = "small"
|
|
370
|
+
a = ab_test("button_size", "small", "big")
|
|
371
|
+
expect(a).to eq("small")
|
|
372
|
+
expect {
|
|
373
|
+
ab_finished("button_size")
|
|
374
|
+
}.not_to change { Split::Alternative.new(a, "button_size").completed_count }
|
|
375
|
+
end
|
|
328
376
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
ab_finished('button_size')
|
|
335
|
-
}).not_to change { Split::Alternative.new('small', 'button_size').completed_count }
|
|
336
|
-
end
|
|
377
|
+
it "should clear out the user's participation from their session" do
|
|
378
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
379
|
+
ab_finished(@experiment_name)
|
|
380
|
+
expect(ab_user.keys).to be_empty
|
|
381
|
+
end
|
|
337
382
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
ab_finished('button_size')
|
|
345
|
-
}).not_to change { Split::Alternative.new(a, 'button_size').completed_count }
|
|
346
|
-
end
|
|
383
|
+
it "should not clear out the users session if reset is false" do
|
|
384
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
385
|
+
ab_finished(@experiment_name, { reset: false })
|
|
386
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
387
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
388
|
+
end
|
|
347
389
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
390
|
+
it "should reset the users session when experiment is not versioned" do
|
|
391
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
392
|
+
ab_finished(@experiment_name)
|
|
393
|
+
expect(ab_user.keys).to be_empty
|
|
394
|
+
end
|
|
353
395
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
358
|
-
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
359
|
-
end
|
|
396
|
+
it "should reset the users session when experiment is versioned" do
|
|
397
|
+
@experiment.increment_version
|
|
398
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
360
399
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
end
|
|
400
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
401
|
+
ab_finished(@experiment_name)
|
|
402
|
+
expect(ab_user.keys).to be_empty
|
|
403
|
+
end
|
|
366
404
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
405
|
+
context "when on_trial_complete is set" do
|
|
406
|
+
before { Split.configuration.on_trial_complete = :some_method }
|
|
407
|
+
it "should call the method" do
|
|
408
|
+
expect(self).to receive(:some_method)
|
|
409
|
+
ab_finished(@experiment_name)
|
|
410
|
+
end
|
|
370
411
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
412
|
+
it "should not call the method without alternative" do
|
|
413
|
+
ab_user[@experiment.key] = nil
|
|
414
|
+
expect(self).not_to receive(:some_method)
|
|
415
|
+
ab_finished(@experiment_name)
|
|
416
|
+
end
|
|
417
|
+
end
|
|
374
418
|
end
|
|
375
419
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
420
|
+
context "for an experiment that the user is excluded from" do
|
|
421
|
+
before do
|
|
422
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
423
|
+
expect(Split::Alternative.new(alternative, "link_color").participant_count).to eq(1)
|
|
424
|
+
alternative = ab_test("button_size", "small", "big")
|
|
425
|
+
expect(Split::Alternative.new(alternative, "button_size").participant_count).to eq(0)
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
it "should not increment the completed counter" do
|
|
429
|
+
# So, user should be participating in the link_color experiment and
|
|
430
|
+
# receive the control for button_size. As the user is not participating in
|
|
431
|
+
# the button size experiment, finishing it should not increase the
|
|
432
|
+
# completion count for that alternative.
|
|
433
|
+
expect {
|
|
434
|
+
ab_finished("button_size")
|
|
435
|
+
}.not_to change { Split::Alternative.new("small", "button_size").completed_count }
|
|
436
|
+
end
|
|
379
437
|
end
|
|
380
438
|
|
|
381
|
-
context "
|
|
382
|
-
before
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
439
|
+
context "for an experiment that the user does not participate in" do
|
|
440
|
+
before do
|
|
441
|
+
Split::ExperimentCatalog.find_or_create(:not_started_experiment, "control", "alt")
|
|
442
|
+
end
|
|
443
|
+
it "should not raise an exception" do
|
|
444
|
+
expect { ab_finished(:not_started_experiment) }.not_to raise_exception
|
|
386
445
|
end
|
|
387
446
|
|
|
388
|
-
it "should not
|
|
389
|
-
ab_user[
|
|
390
|
-
|
|
391
|
-
|
|
447
|
+
it "should not change the user state when reset is false" do
|
|
448
|
+
expect { ab_finished(:not_started_experiment, reset: false) }.not_to change { ab_user.keys }.from([])
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
it "should not change the user state when reset is true" do
|
|
452
|
+
expect(self).not_to receive(:reset!)
|
|
453
|
+
ab_finished(:not_started_experiment)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
it "should not increment the completed counter" do
|
|
457
|
+
ab_finished(:not_started_experiment)
|
|
458
|
+
expect(Split::Alternative.new("control", :not_started_experiment).completed_count).to eq(0)
|
|
459
|
+
expect(Split::Alternative.new("alt", :not_started_experiment).completed_count).to eq(0)
|
|
392
460
|
end
|
|
393
461
|
end
|
|
394
462
|
end
|
|
@@ -396,9 +464,9 @@ describe Split::Helper do
|
|
|
396
464
|
context "finished with config" do
|
|
397
465
|
it "passes reset option" do
|
|
398
466
|
Split.configuration.experiments = {
|
|
399
|
-
:
|
|
400
|
-
:
|
|
401
|
-
:
|
|
467
|
+
my_experiment: {
|
|
468
|
+
alternatives: ["one", "two"],
|
|
469
|
+
resettable: false,
|
|
402
470
|
}
|
|
403
471
|
}
|
|
404
472
|
alternative = ab_test(:my_experiment)
|
|
@@ -414,11 +482,11 @@ describe Split::Helper do
|
|
|
414
482
|
before { Split.configuration.experiments = {} }
|
|
415
483
|
before { expect(Split::Alternative).to receive(:new).at_least(1).times.and_call_original }
|
|
416
484
|
|
|
417
|
-
def should_finish_experiment(experiment_name, should_finish=true)
|
|
485
|
+
def should_finish_experiment(experiment_name, should_finish = true)
|
|
418
486
|
alts = Split.configuration.experiments[experiment_name][:alternatives]
|
|
419
487
|
experiment = Split::ExperimentCatalog.find_or_create(experiment_name, *alts)
|
|
420
488
|
alt_name = ab_user[experiment.key] = alts.first
|
|
421
|
-
alt = double(
|
|
489
|
+
alt = double("alternative")
|
|
422
490
|
expect(alt).to receive(:name).at_most(1).times.and_return(alt_name)
|
|
423
491
|
expect(Split::Alternative).to receive(:new).at_most(1).times.with(alt_name, experiment_name.to_s).and_return(alt)
|
|
424
492
|
if should_finish
|
|
@@ -430,8 +498,8 @@ describe Split::Helper do
|
|
|
430
498
|
|
|
431
499
|
it "completes the test" do
|
|
432
500
|
Split.configuration.experiments[:my_experiment] = {
|
|
433
|
-
:
|
|
434
|
-
:
|
|
501
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
502
|
+
metric: :my_metric
|
|
435
503
|
}
|
|
436
504
|
should_finish_experiment :my_experiment
|
|
437
505
|
ab_finished :my_metric
|
|
@@ -439,17 +507,17 @@ describe Split::Helper do
|
|
|
439
507
|
|
|
440
508
|
it "completes all relevant tests" do
|
|
441
509
|
Split.configuration.experiments = {
|
|
442
|
-
:
|
|
443
|
-
:
|
|
444
|
-
:
|
|
510
|
+
exp_1: {
|
|
511
|
+
alternatives: [ "1-1", "1-2" ],
|
|
512
|
+
metric: :my_metric
|
|
445
513
|
},
|
|
446
|
-
:
|
|
447
|
-
:
|
|
448
|
-
:
|
|
514
|
+
exp_2: {
|
|
515
|
+
alternatives: [ "2-1", "2-2" ],
|
|
516
|
+
metric: :another_metric
|
|
449
517
|
},
|
|
450
|
-
:
|
|
451
|
-
:
|
|
452
|
-
:
|
|
518
|
+
exp_3: {
|
|
519
|
+
alternatives: [ "3-1", "3-2" ],
|
|
520
|
+
metric: :my_metric
|
|
453
521
|
},
|
|
454
522
|
}
|
|
455
523
|
should_finish_experiment :exp_1
|
|
@@ -460,10 +528,10 @@ describe Split::Helper do
|
|
|
460
528
|
|
|
461
529
|
it "passes reset option" do
|
|
462
530
|
Split.configuration.experiments = {
|
|
463
|
-
:
|
|
464
|
-
:
|
|
465
|
-
:
|
|
466
|
-
:
|
|
531
|
+
my_exp: {
|
|
532
|
+
alternatives: ["one", "two"],
|
|
533
|
+
metric: :my_metric,
|
|
534
|
+
resettable: false,
|
|
467
535
|
}
|
|
468
536
|
}
|
|
469
537
|
alternative_name = ab_test(:my_exp)
|
|
@@ -476,188 +544,233 @@ describe Split::Helper do
|
|
|
476
544
|
|
|
477
545
|
it "passes through options" do
|
|
478
546
|
Split.configuration.experiments = {
|
|
479
|
-
:
|
|
480
|
-
:
|
|
481
|
-
:
|
|
547
|
+
my_exp: {
|
|
548
|
+
alternatives: ["one", "two"],
|
|
549
|
+
metric: :my_metric,
|
|
482
550
|
}
|
|
483
551
|
}
|
|
484
552
|
alternative_name = ab_test(:my_exp)
|
|
485
553
|
exp = Split::ExperimentCatalog.find :my_exp
|
|
486
554
|
|
|
487
|
-
ab_finished :my_metric, :
|
|
555
|
+
ab_finished :my_metric, reset: false
|
|
488
556
|
expect(ab_user[exp.key]).to eq(alternative_name)
|
|
489
557
|
expect(ab_user[exp.finished_key]).to be_truthy
|
|
490
558
|
end
|
|
491
559
|
end
|
|
492
560
|
|
|
493
|
-
describe 'conversions' do
|
|
494
|
-
it 'should return a conversion rate for an alternative' do
|
|
495
|
-
alternative_name = ab_test('link_color', 'blue', 'red')
|
|
496
561
|
|
|
497
|
-
|
|
562
|
+
describe "ab_record_extra_info" do
|
|
563
|
+
context "for an experiment that the user participates in" do
|
|
564
|
+
before(:each) do
|
|
565
|
+
@experiment_name = "link_color"
|
|
566
|
+
@alternatives = ["blue", "red"]
|
|
567
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
568
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
it "records extra data for a given experiment" do
|
|
572
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
573
|
+
|
|
574
|
+
ab_record_extra_info(@experiment_name, "some_data", 10)
|
|
575
|
+
|
|
576
|
+
expect(alternative.extra_info).to eql({ "some_data" => 10 })
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
it "records extra data for a given experiment" do
|
|
580
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
581
|
+
|
|
582
|
+
ab_record_extra_info(@experiment_name, "some_data")
|
|
583
|
+
|
|
584
|
+
expect(alternative.extra_info).to eql({ "some_data" => 1 })
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
it "records extra data for a given experiment" do
|
|
588
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
589
|
+
|
|
590
|
+
ab_record_extra_info(@experiment_name, "some_data", nil)
|
|
591
|
+
|
|
592
|
+
expect(alternative.extra_info).to eql({})
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
describe "conversions" do
|
|
598
|
+
it "should return a conversion rate for an alternative" do
|
|
599
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
600
|
+
|
|
601
|
+
previous_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
498
602
|
expect(previous_convertion_rate).to eq(0.0)
|
|
499
603
|
|
|
500
|
-
ab_finished(
|
|
604
|
+
ab_finished("link_color")
|
|
501
605
|
|
|
502
|
-
new_convertion_rate = Split::Alternative.new(alternative_name,
|
|
606
|
+
new_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
503
607
|
expect(new_convertion_rate).to eq(1.0)
|
|
504
608
|
end
|
|
505
609
|
end
|
|
506
610
|
|
|
507
|
-
describe
|
|
508
|
-
it
|
|
509
|
-
alternative = ab_test(
|
|
611
|
+
describe "active experiments" do
|
|
612
|
+
it "should show an active test" do
|
|
613
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
510
614
|
expect(active_experiments.count).to eq 1
|
|
511
615
|
expect(active_experiments.first[0]).to eq "def"
|
|
512
616
|
expect(active_experiments.first[1]).to eq alternative
|
|
513
617
|
end
|
|
514
618
|
|
|
515
|
-
it
|
|
516
|
-
alternative = ab_test(
|
|
517
|
-
ab_finished(
|
|
619
|
+
it "should show a finished test" do
|
|
620
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
621
|
+
ab_finished("def", { reset: false })
|
|
518
622
|
expect(active_experiments.count).to eq 1
|
|
519
623
|
expect(active_experiments.first[0]).to eq "def"
|
|
520
624
|
expect(active_experiments.first[1]).to eq alternative
|
|
521
625
|
end
|
|
522
626
|
|
|
523
|
-
it
|
|
627
|
+
it "should show an active test when an experiment is on a later version" do
|
|
524
628
|
experiment.reset
|
|
525
629
|
expect(experiment.version).to eq(1)
|
|
526
|
-
ab_test(
|
|
630
|
+
ab_test("link_color", "blue", "red")
|
|
527
631
|
expect(active_experiments.count).to eq 1
|
|
528
632
|
expect(active_experiments.first[0]).to eq "link_color"
|
|
529
633
|
end
|
|
530
634
|
|
|
531
|
-
it
|
|
635
|
+
it "should show versioned tests properly" do
|
|
636
|
+
10.times { experiment.reset }
|
|
637
|
+
|
|
638
|
+
alternative = ab_test(experiment.name, "blue", "red")
|
|
639
|
+
ab_finished(experiment.name, reset: false)
|
|
640
|
+
|
|
641
|
+
expect(experiment.version).to eq(10)
|
|
642
|
+
expect(active_experiments.count).to eq 1
|
|
643
|
+
expect(active_experiments).to eq({ "link_color" => alternative })
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
it "should show multiple tests" do
|
|
532
647
|
Split.configure do |config|
|
|
533
648
|
config.allow_multiple_experiments = true
|
|
534
649
|
end
|
|
535
|
-
alternative = ab_test(
|
|
536
|
-
another_alternative = ab_test(
|
|
650
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
651
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
537
652
|
expect(active_experiments.count).to eq 2
|
|
538
|
-
expect(active_experiments[
|
|
539
|
-
expect(active_experiments[
|
|
653
|
+
expect(active_experiments["def"]).to eq alternative
|
|
654
|
+
expect(active_experiments["ghi"]).to eq another_alternative
|
|
540
655
|
end
|
|
541
656
|
|
|
542
|
-
it
|
|
657
|
+
it "should not show tests with winners" do
|
|
543
658
|
Split.configure do |config|
|
|
544
659
|
config.allow_multiple_experiments = true
|
|
545
660
|
end
|
|
546
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
547
|
-
e.winner =
|
|
548
|
-
|
|
549
|
-
another_alternative = ab_test(
|
|
661
|
+
e = Split::ExperimentCatalog.find_or_create("def", "4", "5", "6")
|
|
662
|
+
e.winner = "4"
|
|
663
|
+
ab_test("def", "4", "5", "6")
|
|
664
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
550
665
|
expect(active_experiments.count).to eq 1
|
|
551
666
|
expect(active_experiments.first[0]).to eq "ghi"
|
|
552
667
|
expect(active_experiments.first[1]).to eq another_alternative
|
|
553
668
|
end
|
|
554
669
|
end
|
|
555
670
|
|
|
556
|
-
describe
|
|
671
|
+
describe "when user is a robot" do
|
|
557
672
|
before(:each) do
|
|
558
|
-
@request =
|
|
673
|
+
@request = build_request(user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)")
|
|
559
674
|
end
|
|
560
675
|
|
|
561
|
-
describe
|
|
562
|
-
it
|
|
563
|
-
alternative = ab_test(
|
|
676
|
+
describe "ab_test" do
|
|
677
|
+
it "should return the control" do
|
|
678
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
564
679
|
expect(alternative).to eq experiment.control.name
|
|
565
680
|
end
|
|
566
681
|
|
|
567
|
-
it
|
|
568
|
-
ab_test(
|
|
569
|
-
expect(Split::Experiment.new(
|
|
682
|
+
it "should not create a experiment" do
|
|
683
|
+
ab_test("link_color", "blue", "red")
|
|
684
|
+
expect(Split::Experiment.new("link_color")).to be_a_new_record
|
|
570
685
|
end
|
|
571
686
|
|
|
572
687
|
it "should not increment the participation count" do
|
|
688
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
689
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
573
690
|
|
|
574
|
-
|
|
575
|
-
previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
|
|
576
|
-
|
|
577
|
-
ab_test('link_color', 'blue', 'red')
|
|
691
|
+
ab_test("link_color", "blue", "red")
|
|
578
692
|
|
|
579
|
-
new_red_count = Split::Alternative.new(
|
|
580
|
-
new_blue_count = Split::Alternative.new(
|
|
693
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
694
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
581
695
|
|
|
582
696
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
583
697
|
end
|
|
584
698
|
end
|
|
585
699
|
|
|
586
|
-
describe
|
|
700
|
+
describe "finished" do
|
|
587
701
|
it "should not increment the completed count" do
|
|
588
|
-
alternative_name = ab_test(
|
|
702
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
589
703
|
|
|
590
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
704
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
591
705
|
|
|
592
|
-
ab_finished(
|
|
706
|
+
ab_finished("link_color")
|
|
593
707
|
|
|
594
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
708
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
595
709
|
|
|
596
710
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
597
711
|
end
|
|
598
712
|
end
|
|
599
713
|
end
|
|
600
714
|
|
|
601
|
-
describe
|
|
715
|
+
describe "when providing custom ignore logic" do
|
|
602
716
|
context "using a proc to configure custom logic" do
|
|
603
|
-
|
|
604
717
|
before(:each) do
|
|
605
718
|
Split.configure do |c|
|
|
606
|
-
c.ignore_filter = proc{|request| true } # ignore everything
|
|
719
|
+
c.ignore_filter = proc { |request| true } # ignore everything
|
|
607
720
|
end
|
|
608
721
|
end
|
|
609
722
|
|
|
610
723
|
it "ignores the ab_test" do
|
|
611
|
-
ab_test(
|
|
724
|
+
ab_test("link_color", "blue", "red")
|
|
612
725
|
|
|
613
|
-
red_count = Split::Alternative.new(
|
|
614
|
-
blue_count = Split::Alternative.new(
|
|
726
|
+
red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
727
|
+
blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
615
728
|
expect((red_count + blue_count)).to be(0)
|
|
616
729
|
end
|
|
617
730
|
end
|
|
618
731
|
end
|
|
619
732
|
|
|
620
733
|
shared_examples_for "a disabled test" do
|
|
621
|
-
describe
|
|
622
|
-
it
|
|
623
|
-
alternative = ab_test(
|
|
734
|
+
describe "ab_test" do
|
|
735
|
+
it "should return the control" do
|
|
736
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
624
737
|
expect(alternative).to eq experiment.control.name
|
|
625
738
|
end
|
|
626
739
|
|
|
627
740
|
it "should not increment the participation count" do
|
|
628
|
-
previous_red_count = Split::Alternative.new(
|
|
629
|
-
previous_blue_count = Split::Alternative.new(
|
|
741
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
742
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
630
743
|
|
|
631
|
-
ab_test(
|
|
744
|
+
ab_test("link_color", "blue", "red")
|
|
632
745
|
|
|
633
|
-
new_red_count = Split::Alternative.new(
|
|
634
|
-
new_blue_count = Split::Alternative.new(
|
|
746
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
747
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
635
748
|
|
|
636
749
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
637
750
|
end
|
|
638
751
|
end
|
|
639
752
|
|
|
640
|
-
describe
|
|
753
|
+
describe "finished" do
|
|
641
754
|
it "should not increment the completed count" do
|
|
642
|
-
alternative_name = ab_test(
|
|
755
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
643
756
|
|
|
644
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
757
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
645
758
|
|
|
646
|
-
ab_finished(
|
|
759
|
+
ab_finished("link_color")
|
|
647
760
|
|
|
648
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
761
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
649
762
|
|
|
650
763
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
651
764
|
end
|
|
652
765
|
end
|
|
653
766
|
end
|
|
654
767
|
|
|
655
|
-
describe
|
|
768
|
+
describe "when ip address is ignored" do
|
|
656
769
|
context "individually" do
|
|
657
770
|
before(:each) do
|
|
658
|
-
@request =
|
|
771
|
+
@request = build_request(ip: "81.19.48.130")
|
|
659
772
|
Split.configure do |c|
|
|
660
|
-
c.ignore_ip_addresses <<
|
|
773
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
661
774
|
end
|
|
662
775
|
end
|
|
663
776
|
|
|
@@ -666,7 +779,7 @@ describe Split::Helper do
|
|
|
666
779
|
|
|
667
780
|
context "for a range" do
|
|
668
781
|
before(:each) do
|
|
669
|
-
@request =
|
|
782
|
+
@request = build_request(ip: "81.19.48.129")
|
|
670
783
|
Split.configure do |c|
|
|
671
784
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
672
785
|
end
|
|
@@ -677,9 +790,9 @@ describe Split::Helper do
|
|
|
677
790
|
|
|
678
791
|
context "using both a range and a specific value" do
|
|
679
792
|
before(:each) do
|
|
680
|
-
@request =
|
|
793
|
+
@request = build_request(ip: "81.19.48.128")
|
|
681
794
|
Split.configure do |c|
|
|
682
|
-
c.ignore_ip_addresses <<
|
|
795
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
683
796
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
684
797
|
end
|
|
685
798
|
end
|
|
@@ -689,119 +802,119 @@ describe Split::Helper do
|
|
|
689
802
|
|
|
690
803
|
context "when ignored other address" do
|
|
691
804
|
before do
|
|
692
|
-
@request =
|
|
805
|
+
@request = build_request(ip: "1.1.1.1")
|
|
693
806
|
Split.configure do |c|
|
|
694
|
-
c.ignore_ip_addresses <<
|
|
807
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
695
808
|
end
|
|
696
809
|
end
|
|
697
810
|
|
|
698
811
|
it "works as usual" do
|
|
699
|
-
alternative_name = ab_test(
|
|
700
|
-
expect{
|
|
701
|
-
ab_finished(
|
|
702
|
-
}.to change(Split::Alternative.new(alternative_name,
|
|
812
|
+
alternative_name = ab_test("link_color", "red", "blue")
|
|
813
|
+
expect {
|
|
814
|
+
ab_finished("link_color")
|
|
815
|
+
}.to change(Split::Alternative.new(alternative_name, "link_color"), :completed_count).by(1)
|
|
703
816
|
end
|
|
704
817
|
end
|
|
705
818
|
end
|
|
706
819
|
|
|
707
|
-
describe
|
|
820
|
+
describe "when user is previewing" do
|
|
708
821
|
before(:each) do
|
|
709
|
-
@request =
|
|
822
|
+
@request = build_request(headers: { "x-purpose" => "preview" })
|
|
710
823
|
end
|
|
711
824
|
|
|
712
825
|
it_behaves_like "a disabled test"
|
|
713
826
|
end
|
|
714
827
|
|
|
715
|
-
describe
|
|
828
|
+
describe "versioned experiments" do
|
|
716
829
|
it "should use version zero if no version is present" do
|
|
717
|
-
alternative_name = ab_test(
|
|
830
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
718
831
|
expect(experiment.version).to eq(0)
|
|
719
|
-
expect(ab_user[
|
|
832
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
720
833
|
end
|
|
721
834
|
|
|
722
835
|
it "should save the version of the experiment to the session" do
|
|
723
836
|
experiment.reset
|
|
724
837
|
expect(experiment.version).to eq(1)
|
|
725
|
-
alternative_name = ab_test(
|
|
726
|
-
expect(ab_user[
|
|
838
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
839
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
727
840
|
end
|
|
728
841
|
|
|
729
842
|
it "should load the experiment even if the version is not 0" do
|
|
730
843
|
experiment.reset
|
|
731
844
|
expect(experiment.version).to eq(1)
|
|
732
|
-
alternative_name = ab_test(
|
|
733
|
-
expect(ab_user[
|
|
734
|
-
return_alternative_name = ab_test(
|
|
845
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
846
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
847
|
+
return_alternative_name = ab_test("link_color", "blue", "red")
|
|
735
848
|
expect(return_alternative_name).to eq(alternative_name)
|
|
736
849
|
end
|
|
737
850
|
|
|
738
851
|
it "should reset the session of a user on an older version of the experiment" do
|
|
739
|
-
alternative_name = ab_test(
|
|
740
|
-
expect(ab_user[
|
|
741
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
852
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
853
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
854
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
742
855
|
expect(alternative.participant_count).to eq(1)
|
|
743
856
|
|
|
744
857
|
experiment.reset
|
|
745
858
|
expect(experiment.version).to eq(1)
|
|
746
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
859
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
747
860
|
expect(alternative.participant_count).to eq(0)
|
|
748
861
|
|
|
749
|
-
new_alternative_name = ab_test(
|
|
750
|
-
expect(ab_user[
|
|
751
|
-
new_alternative = Split::Alternative.new(new_alternative_name,
|
|
862
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
863
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
864
|
+
new_alternative = Split::Alternative.new(new_alternative_name, "link_color")
|
|
752
865
|
expect(new_alternative.participant_count).to eq(1)
|
|
753
866
|
end
|
|
754
867
|
|
|
755
868
|
it "should cleanup old versions of experiments from the session" do
|
|
756
|
-
alternative_name = ab_test(
|
|
757
|
-
expect(ab_user[
|
|
758
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
869
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
870
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
871
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
759
872
|
expect(alternative.participant_count).to eq(1)
|
|
760
873
|
|
|
761
874
|
experiment.reset
|
|
762
875
|
expect(experiment.version).to eq(1)
|
|
763
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
876
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
764
877
|
expect(alternative.participant_count).to eq(0)
|
|
765
878
|
|
|
766
|
-
new_alternative_name = ab_test(
|
|
767
|
-
expect(ab_user[
|
|
879
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
880
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
768
881
|
end
|
|
769
882
|
|
|
770
883
|
it "should only count completion of users on the current version" do
|
|
771
|
-
alternative_name = ab_test(
|
|
772
|
-
expect(ab_user[
|
|
773
|
-
|
|
884
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
885
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
886
|
+
Split::Alternative.new(alternative_name, "link_color")
|
|
774
887
|
|
|
775
888
|
experiment.reset
|
|
776
889
|
expect(experiment.version).to eq(1)
|
|
777
890
|
|
|
778
|
-
ab_finished(
|
|
779
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
891
|
+
ab_finished("link_color")
|
|
892
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
780
893
|
expect(alternative.completed_count).to eq(0)
|
|
781
894
|
end
|
|
782
895
|
end
|
|
783
896
|
|
|
784
|
-
context
|
|
897
|
+
context "when redis is not available" do
|
|
785
898
|
before(:each) do
|
|
786
899
|
expect(Split).to receive(:redis).at_most(5).times.and_raise(Errno::ECONNREFUSED.new)
|
|
787
900
|
end
|
|
788
901
|
|
|
789
|
-
context
|
|
902
|
+
context "and db_failover config option is turned off" do
|
|
790
903
|
before(:each) do
|
|
791
904
|
Split.configure do |config|
|
|
792
905
|
config.db_failover = false
|
|
793
906
|
end
|
|
794
907
|
end
|
|
795
908
|
|
|
796
|
-
describe
|
|
797
|
-
it
|
|
798
|
-
expect
|
|
909
|
+
describe "ab_test" do
|
|
910
|
+
it "should raise an exception" do
|
|
911
|
+
expect { ab_test("link_color", "blue", "red") }.to raise_error(Errno::ECONNREFUSED)
|
|
799
912
|
end
|
|
800
913
|
end
|
|
801
914
|
|
|
802
|
-
describe
|
|
803
|
-
it
|
|
804
|
-
expect
|
|
915
|
+
describe "finished" do
|
|
916
|
+
it "should raise an exception" do
|
|
917
|
+
expect { ab_finished("link_color") }.to raise_error(Errno::ECONNREFUSED)
|
|
805
918
|
end
|
|
806
919
|
end
|
|
807
920
|
|
|
@@ -813,29 +926,29 @@ describe Split::Helper do
|
|
|
813
926
|
end
|
|
814
927
|
|
|
815
928
|
it "should not attempt to connect to redis" do
|
|
816
|
-
expect
|
|
929
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
817
930
|
end
|
|
818
931
|
|
|
819
932
|
it "should return control variable" do
|
|
820
|
-
expect(ab_test(
|
|
821
|
-
expect
|
|
933
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
934
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
822
935
|
end
|
|
823
936
|
end
|
|
824
937
|
end
|
|
825
938
|
|
|
826
|
-
context
|
|
939
|
+
context "and db_failover config option is turned on" do
|
|
827
940
|
before(:each) do
|
|
828
941
|
Split.configure do |config|
|
|
829
942
|
config.db_failover = true
|
|
830
943
|
end
|
|
831
944
|
end
|
|
832
945
|
|
|
833
|
-
describe
|
|
834
|
-
it
|
|
835
|
-
expect
|
|
946
|
+
describe "ab_test" do
|
|
947
|
+
it "should not raise an exception" do
|
|
948
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
836
949
|
end
|
|
837
950
|
|
|
838
|
-
it
|
|
951
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
839
952
|
Split.configure do |config|
|
|
840
953
|
config.db_failover_on_db_error = proc do |error|
|
|
841
954
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -843,43 +956,43 @@ describe Split::Helper do
|
|
|
843
956
|
end
|
|
844
957
|
|
|
845
958
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
846
|
-
ab_test(
|
|
959
|
+
ab_test("link_color", "blue", "red")
|
|
847
960
|
end
|
|
848
961
|
|
|
849
|
-
it
|
|
850
|
-
expect(ab_test(
|
|
851
|
-
expect(ab_test(
|
|
852
|
-
expect(ab_test(
|
|
853
|
-
expect(ab_test(
|
|
962
|
+
it "should always use first alternative" do
|
|
963
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
964
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("blue")
|
|
965
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("blue")
|
|
966
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
854
967
|
"shared/#{alternative}"
|
|
855
|
-
end).to eq(
|
|
968
|
+
end).to eq("shared/blue")
|
|
856
969
|
end
|
|
857
970
|
|
|
858
|
-
context
|
|
971
|
+
context "and db_failover_allow_parameter_override config option is turned on" do
|
|
859
972
|
before(:each) do
|
|
860
973
|
Split.configure do |config|
|
|
861
974
|
config.db_failover_allow_parameter_override = true
|
|
862
975
|
end
|
|
863
976
|
end
|
|
864
977
|
|
|
865
|
-
context
|
|
866
|
-
it
|
|
867
|
-
@params = {
|
|
868
|
-
expect(ab_test(
|
|
869
|
-
expect(ab_test(
|
|
870
|
-
expect(ab_test(
|
|
871
|
-
expect(ab_test(
|
|
872
|
-
expect(ab_test(
|
|
978
|
+
context "and given an override parameter" do
|
|
979
|
+
it "should use given override instead of the first alternative" do
|
|
980
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
981
|
+
expect(ab_test("link_color", "blue", "red")).to eq("red")
|
|
982
|
+
expect(ab_test("link_color", "blue", "red", "green")).to eq("red")
|
|
983
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("red")
|
|
984
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("red")
|
|
985
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
873
986
|
"shared/#{alternative}"
|
|
874
|
-
end).to eq(
|
|
987
|
+
end).to eq("shared/red")
|
|
875
988
|
end
|
|
876
989
|
end
|
|
877
990
|
end
|
|
878
991
|
|
|
879
|
-
context
|
|
992
|
+
context "and preloaded config given" do
|
|
880
993
|
before do
|
|
881
994
|
Split.configuration.experiments[:link_color] = {
|
|
882
|
-
:
|
|
995
|
+
alternatives: [ "blue", "red" ],
|
|
883
996
|
}
|
|
884
997
|
end
|
|
885
998
|
|
|
@@ -889,12 +1002,12 @@ describe Split::Helper do
|
|
|
889
1002
|
end
|
|
890
1003
|
end
|
|
891
1004
|
|
|
892
|
-
describe
|
|
893
|
-
it
|
|
894
|
-
expect
|
|
1005
|
+
describe "finished" do
|
|
1006
|
+
it "should not raise an exception" do
|
|
1007
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
895
1008
|
end
|
|
896
1009
|
|
|
897
|
-
it
|
|
1010
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
898
1011
|
Split.configure do |config|
|
|
899
1012
|
config.db_failover_on_db_error = proc do |error|
|
|
900
1013
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -902,19 +1015,19 @@ describe Split::Helper do
|
|
|
902
1015
|
end
|
|
903
1016
|
|
|
904
1017
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
905
|
-
ab_finished(
|
|
1018
|
+
ab_finished("link_color")
|
|
906
1019
|
end
|
|
907
1020
|
end
|
|
908
1021
|
end
|
|
909
1022
|
end
|
|
910
1023
|
|
|
911
1024
|
context "with preloaded config" do
|
|
912
|
-
before { Split.configuration.experiments = {}}
|
|
1025
|
+
before { Split.configuration.experiments = {} }
|
|
913
1026
|
|
|
914
1027
|
it "pulls options from config file" do
|
|
915
1028
|
Split.configuration.experiments[:my_experiment] = {
|
|
916
|
-
:
|
|
917
|
-
:
|
|
1029
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1030
|
+
goals: ["goal1", "goal2"]
|
|
918
1031
|
}
|
|
919
1032
|
ab_test :my_experiment
|
|
920
1033
|
expect(Split::Experiment.new(:my_experiment).alternatives.map(&:name)).to eq([ "control_opt", "other_opt" ])
|
|
@@ -923,8 +1036,8 @@ describe Split::Helper do
|
|
|
923
1036
|
|
|
924
1037
|
it "can be called multiple times" do
|
|
925
1038
|
Split.configuration.experiments[:my_experiment] = {
|
|
926
|
-
:
|
|
927
|
-
:
|
|
1039
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1040
|
+
goals: ["goal1", "goal2"]
|
|
928
1041
|
}
|
|
929
1042
|
5.times { ab_test :my_experiment }
|
|
930
1043
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -935,8 +1048,8 @@ describe Split::Helper do
|
|
|
935
1048
|
|
|
936
1049
|
it "accepts multiple goals" do
|
|
937
1050
|
Split.configuration.experiments[:my_experiment] = {
|
|
938
|
-
:
|
|
939
|
-
:
|
|
1051
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1052
|
+
goals: [ "goal1", "goal2", "goal3" ]
|
|
940
1053
|
}
|
|
941
1054
|
ab_test :my_experiment
|
|
942
1055
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -945,7 +1058,7 @@ describe Split::Helper do
|
|
|
945
1058
|
|
|
946
1059
|
it "allow specifying goals to be optional" do
|
|
947
1060
|
Split.configuration.experiments[:my_experiment] = {
|
|
948
|
-
:
|
|
1061
|
+
alternatives: [ "control_opt", "other_opt" ]
|
|
949
1062
|
}
|
|
950
1063
|
experiment = Split::Experiment.new(:my_experiment)
|
|
951
1064
|
expect(experiment.goals).to eq([])
|
|
@@ -953,7 +1066,7 @@ describe Split::Helper do
|
|
|
953
1066
|
|
|
954
1067
|
it "accepts multiple alternatives" do
|
|
955
1068
|
Split.configuration.experiments[:my_experiment] = {
|
|
956
|
-
:
|
|
1069
|
+
alternatives: [ "control_opt", "second_opt", "third_opt" ],
|
|
957
1070
|
}
|
|
958
1071
|
ab_test :my_experiment
|
|
959
1072
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -962,68 +1075,68 @@ describe Split::Helper do
|
|
|
962
1075
|
|
|
963
1076
|
it "accepts probability on alternatives" do
|
|
964
1077
|
Split.configuration.experiments[:my_experiment] = {
|
|
965
|
-
:
|
|
966
|
-
{ :
|
|
967
|
-
{ :
|
|
968
|
-
{ :
|
|
1078
|
+
alternatives: [
|
|
1079
|
+
{ name: "control_opt", percent: 67 },
|
|
1080
|
+
{ name: "second_opt", percent: 10 },
|
|
1081
|
+
{ name: "third_opt", percent: 23 },
|
|
969
1082
|
],
|
|
970
1083
|
}
|
|
971
1084
|
ab_test :my_experiment
|
|
972
1085
|
experiment = Split::Experiment.new(:my_experiment)
|
|
973
|
-
expect(experiment.alternatives.collect{|a| [a.name, a.weight]}).to eq([[
|
|
1086
|
+
expect(experiment.alternatives.collect { |a| [a.name, a.weight] }).to eq([["control_opt", 0.67], ["second_opt", 0.1], ["third_opt", 0.23]])
|
|
974
1087
|
end
|
|
975
1088
|
|
|
976
1089
|
it "accepts probability on some alternatives" do
|
|
977
1090
|
Split.configuration.experiments[:my_experiment] = {
|
|
978
|
-
:
|
|
979
|
-
{ :
|
|
1091
|
+
alternatives: [
|
|
1092
|
+
{ name: "control_opt", percent: 34 },
|
|
980
1093
|
"second_opt",
|
|
981
|
-
{ :
|
|
1094
|
+
{ name: "third_opt", percent: 23 },
|
|
982
1095
|
"fourth_opt",
|
|
983
1096
|
],
|
|
984
1097
|
}
|
|
985
1098
|
ab_test :my_experiment
|
|
986
1099
|
experiment = Split::Experiment.new(:my_experiment)
|
|
987
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
988
|
-
expect(names_and_weights).to eq([[
|
|
989
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1100
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1101
|
+
expect(names_and_weights).to eq([["control_opt", 0.34], ["second_opt", 0.215], ["third_opt", 0.23], ["fourth_opt", 0.215]])
|
|
1102
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
990
1103
|
end
|
|
991
1104
|
|
|
992
1105
|
it "allows name param without probability" do
|
|
993
1106
|
Split.configuration.experiments[:my_experiment] = {
|
|
994
|
-
:
|
|
995
|
-
{ :
|
|
1107
|
+
alternatives: [
|
|
1108
|
+
{ name: "control_opt" },
|
|
996
1109
|
"second_opt",
|
|
997
|
-
{ :
|
|
1110
|
+
{ name: "third_opt", percent: 64 },
|
|
998
1111
|
],
|
|
999
1112
|
}
|
|
1000
1113
|
ab_test :my_experiment
|
|
1001
1114
|
experiment = Split::Experiment.new(:my_experiment)
|
|
1002
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
1003
|
-
expect(names_and_weights).to eq([[
|
|
1004
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1115
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1116
|
+
expect(names_and_weights).to eq([["control_opt", 0.18], ["second_opt", 0.18], ["third_opt", 0.64]])
|
|
1117
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
1005
1118
|
end
|
|
1006
1119
|
|
|
1007
1120
|
it "fails gracefully if config is missing experiment" do
|
|
1008
|
-
Split.configuration.experiments = { :
|
|
1009
|
-
expect
|
|
1121
|
+
Split.configuration.experiments = { other_experiment: { foo: "Bar" } }
|
|
1122
|
+
expect { ab_test :my_experiment }.to raise_error(Split::ExperimentNotFound)
|
|
1010
1123
|
end
|
|
1011
1124
|
|
|
1012
1125
|
it "fails gracefully if config is missing" do
|
|
1013
|
-
expect
|
|
1126
|
+
expect { Split.configuration.experiments = nil }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
1014
1127
|
end
|
|
1015
1128
|
|
|
1016
1129
|
it "fails gracefully if config is missing alternatives" do
|
|
1017
|
-
Split.configuration.experiments[:my_experiment] = { :
|
|
1018
|
-
expect
|
|
1130
|
+
Split.configuration.experiments[:my_experiment] = { foo: "Bar" }
|
|
1131
|
+
expect { ab_test :my_experiment }.to raise_error(NoMethodError)
|
|
1019
1132
|
end
|
|
1020
1133
|
end
|
|
1021
1134
|
|
|
1022
|
-
it
|
|
1023
|
-
experiment2 = Split::ExperimentCatalog.find_or_create(
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
ab_finished(
|
|
1135
|
+
it "should handle multiple experiments correctly" do
|
|
1136
|
+
experiment2 = Split::ExperimentCatalog.find_or_create("link_color2", "blue", "red")
|
|
1137
|
+
ab_test("link_color", "blue", "red")
|
|
1138
|
+
ab_test("link_color2", "blue", "red")
|
|
1139
|
+
ab_finished("link_color2")
|
|
1027
1140
|
|
|
1028
1141
|
experiment2.alternatives.each do |alt|
|
|
1029
1142
|
expect(alt.unfinished_count).to eq(0)
|
|
@@ -1032,8 +1145,8 @@ describe Split::Helper do
|
|
|
1032
1145
|
|
|
1033
1146
|
context "with goals" do
|
|
1034
1147
|
before do
|
|
1035
|
-
@experiment = {
|
|
1036
|
-
@alternatives = [
|
|
1148
|
+
@experiment = { "link_color" => ["purchase", "refund"] }
|
|
1149
|
+
@alternatives = ["blue", "red"]
|
|
1037
1150
|
@experiment_name, @goals = normalize_metric(@experiment)
|
|
1038
1151
|
@goal1 = @goals[0]
|
|
1039
1152
|
@goal2 = @goals[1]
|
|
@@ -1047,8 +1160,8 @@ describe Split::Helper do
|
|
|
1047
1160
|
describe "ab_test" do
|
|
1048
1161
|
it "should allow experiment goals interface as a single hash" do
|
|
1049
1162
|
ab_test(@experiment, *@alternatives)
|
|
1050
|
-
experiment = Split::ExperimentCatalog.find(
|
|
1051
|
-
expect(experiment.goals).to eq([
|
|
1163
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
1164
|
+
expect(experiment.goals).to eq(["purchase", "refund"])
|
|
1052
1165
|
end
|
|
1053
1166
|
end
|
|
1054
1167
|
|
|
@@ -1058,15 +1171,9 @@ describe Split::Helper do
|
|
|
1058
1171
|
end
|
|
1059
1172
|
|
|
1060
1173
|
it "should increment the counter for the specified-goal completed alternative" do
|
|
1061
|
-
expect(
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
}).not_to change {
|
|
1065
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
|
|
1066
|
-
}
|
|
1067
|
-
}).to change {
|
|
1068
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
|
|
1069
|
-
}.by(1)
|
|
1174
|
+
expect { ab_finished({ "link_color" => ["purchase"] }) }
|
|
1175
|
+
.to change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) }.by(0)
|
|
1176
|
+
.and change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) }.by(1)
|
|
1070
1177
|
end
|
|
1071
1178
|
end
|
|
1072
1179
|
end
|