split 3.2.0 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.eslintrc +1 -1
- data/.github/FUNDING.yml +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +24 -0
- data/.github/dependabot.yml +7 -0
- data/.github/workflows/ci.yml +63 -0
- data/.rspec +1 -0
- data/.rubocop.yml +67 -1043
- data/CHANGELOG.md +174 -0
- data/CODE_OF_CONDUCT.md +3 -3
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +6 -1
- data/README.md +79 -33
- data/Rakefile +6 -5
- data/lib/split/algorithms/block_randomization.rb +7 -6
- data/lib/split/algorithms/weighted_sample.rb +2 -1
- data/lib/split/algorithms/whiplash.rb +17 -18
- data/lib/split/algorithms.rb +14 -0
- data/lib/split/alternative.rb +25 -25
- data/lib/split/cache.rb +27 -0
- data/lib/split/combined_experiments_helper.rb +6 -5
- data/lib/split/configuration.rb +94 -91
- data/lib/split/dashboard/helpers.rb +9 -9
- data/lib/split/dashboard/pagination_helpers.rb +86 -0
- data/lib/split/dashboard/paginator.rb +17 -0
- data/lib/split/dashboard/public/dashboard.js +10 -0
- data/lib/split/dashboard/public/style.css +19 -2
- data/lib/split/dashboard/views/_controls.erb +13 -0
- data/lib/split/dashboard/views/_experiment.erb +2 -1
- data/lib/split/dashboard/views/index.erb +24 -5
- data/lib/split/dashboard/views/layout.erb +1 -1
- data/lib/split/dashboard.rb +47 -20
- data/lib/split/encapsulated_helper.rb +15 -8
- data/lib/split/engine.rb +7 -4
- data/lib/split/exceptions.rb +1 -0
- data/lib/split/experiment.rb +160 -122
- data/lib/split/experiment_catalog.rb +7 -8
- data/lib/split/extensions/string.rb +2 -1
- data/lib/split/goals_collection.rb +10 -10
- data/lib/split/helper.rb +56 -24
- data/lib/split/metric.rb +6 -6
- data/lib/split/persistence/cookie_adapter.rb +52 -15
- data/lib/split/persistence/dual_adapter.rb +53 -12
- data/lib/split/persistence/redis_adapter.rb +8 -4
- data/lib/split/persistence/session_adapter.rb +1 -2
- data/lib/split/persistence.rb +8 -6
- data/lib/split/redis_interface.rb +16 -31
- data/lib/split/trial.rb +48 -41
- data/lib/split/user.rb +30 -15
- data/lib/split/version.rb +2 -4
- data/lib/split/zscore.rb +2 -3
- data/lib/split.rb +39 -25
- data/spec/algorithms/block_randomization_spec.rb +6 -5
- data/spec/algorithms/weighted_sample_spec.rb +6 -5
- data/spec/algorithms/whiplash_spec.rb +4 -5
- data/spec/alternative_spec.rb +35 -36
- data/spec/cache_spec.rb +84 -0
- data/spec/combined_experiments_helper_spec.rb +18 -17
- data/spec/configuration_spec.rb +41 -45
- data/spec/dashboard/pagination_helpers_spec.rb +202 -0
- data/spec/dashboard/paginator_spec.rb +38 -0
- data/spec/dashboard_helpers_spec.rb +19 -18
- data/spec/dashboard_spec.rb +153 -48
- data/spec/encapsulated_helper_spec.rb +47 -23
- data/spec/experiment_catalog_spec.rb +14 -13
- data/spec/experiment_spec.rb +224 -111
- data/spec/goals_collection_spec.rb +18 -16
- data/spec/helper_spec.rb +539 -419
- data/spec/metric_spec.rb +14 -14
- data/spec/persistence/cookie_adapter_spec.rb +105 -27
- data/spec/persistence/dual_adapter_spec.rb +158 -66
- data/spec/persistence/redis_adapter_spec.rb +35 -27
- data/spec/persistence/session_adapter_spec.rb +2 -3
- data/spec/persistence_spec.rb +1 -2
- data/spec/redis_interface_spec.rb +25 -82
- data/spec/spec_helper.rb +38 -24
- data/spec/split_spec.rb +18 -18
- data/spec/support/cookies_mock.rb +1 -2
- data/spec/trial_spec.rb +117 -70
- data/spec/user_spec.rb +69 -27
- data/split.gemspec +26 -22
- metadata +85 -37
- data/.travis.yml +0 -41
- data/Appraisals +0 -13
- data/gemfiles/4.2.gemfile +0 -9
- data/gemfiles/5.0.gemfile +0 -10
- data/gemfiles/5.1.gemfile +0 -10
data/spec/helper_spec.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
require "spec_helper"
|
|
3
4
|
|
|
4
5
|
# TODO change some of these tests to use Rack::Test
|
|
5
6
|
|
|
@@ -7,156 +8,156 @@ describe Split::Helper do
|
|
|
7
8
|
include Split::Helper
|
|
8
9
|
|
|
9
10
|
let(:experiment) {
|
|
10
|
-
Split::ExperimentCatalog.find_or_create(
|
|
11
|
+
Split::ExperimentCatalog.find_or_create("link_color", "blue", "red")
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
describe "ab_test" do
|
|
14
15
|
it "should not raise an error when passed strings for alternatives" do
|
|
15
|
-
expect
|
|
16
|
+
expect { ab_test("xyz", "1", "2", "3") }.not_to raise_error
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
it "should not raise an error when passed an array for alternatives" do
|
|
19
|
-
expect
|
|
20
|
+
expect { ab_test("xyz", ["1", "2", "3"]) }.not_to raise_error
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
it "should raise the appropriate error when passed integers for alternatives" do
|
|
23
|
-
expect
|
|
24
|
+
expect { ab_test("xyz", 1, 2, 3) }.to raise_error(ArgumentError)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
it "should raise the appropriate error when passed symbols for alternatives" do
|
|
27
|
-
expect
|
|
28
|
+
expect { ab_test("xyz", :a, :b, :c) }.to raise_error(ArgumentError)
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
it "should not raise error when passed an array for goals" do
|
|
31
|
-
expect
|
|
32
|
+
expect { ab_test({ "link_color" => ["purchase", "refund"] }, "blue", "red") }.not_to raise_error
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
it "should not raise error when passed just one goal" do
|
|
35
|
-
expect
|
|
36
|
+
expect { ab_test({ "link_color" => "purchase" }, "blue", "red") }.not_to raise_error
|
|
36
37
|
end
|
|
37
38
|
|
|
38
39
|
it "raises an appropriate error when processing combined expirements" do
|
|
39
40
|
Split.configuration.experiments = {
|
|
40
|
-
:
|
|
41
|
-
:
|
|
42
|
-
:
|
|
43
|
-
:
|
|
41
|
+
combined_exp_1: {
|
|
42
|
+
alternatives: [ { name: "control", percent: 50 }, { name: "test-alt", percent: 50 } ],
|
|
43
|
+
metric: :my_metric,
|
|
44
|
+
combined_experiments: [:combined_exp_1_sub_1]
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
|
-
Split::ExperimentCatalog.find_or_create(
|
|
47
|
-
expect
|
|
47
|
+
Split::ExperimentCatalog.find_or_create("combined_exp_1")
|
|
48
|
+
expect { ab_test("combined_exp_1") }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
|
|
51
|
-
ab_test(
|
|
52
|
-
expect([
|
|
52
|
+
ab_test("link_color", "blue", "red")
|
|
53
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
it "should increment the participation counter after assignment to a new user" do
|
|
56
|
-
previous_red_count = Split::Alternative.new(
|
|
57
|
-
previous_blue_count = Split::Alternative.new(
|
|
57
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
58
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
58
59
|
|
|
59
|
-
ab_test(
|
|
60
|
+
ab_test("link_color", "blue", "red")
|
|
60
61
|
|
|
61
|
-
new_red_count = Split::Alternative.new(
|
|
62
|
-
new_blue_count = Split::Alternative.new(
|
|
62
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
63
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
63
64
|
|
|
64
65
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count + 1)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
|
-
it
|
|
68
|
-
ab_test(
|
|
69
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
70
|
-
expect
|
|
68
|
+
it "should not increment the counter for an experiment that the user is not participating in" do
|
|
69
|
+
ab_test("link_color", "blue", "red")
|
|
70
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
71
|
+
expect {
|
|
71
72
|
# User shouldn't participate in this second experiment
|
|
72
|
-
ab_test(
|
|
73
|
-
}
|
|
73
|
+
ab_test("button_size", "small", "big")
|
|
74
|
+
}.not_to change { e.participant_count }
|
|
74
75
|
end
|
|
75
76
|
|
|
76
|
-
it
|
|
77
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
78
|
-
e.winner =
|
|
79
|
-
expect
|
|
80
|
-
a = ab_test(
|
|
81
|
-
expect(a).to eq(
|
|
82
|
-
}
|
|
77
|
+
it "should not increment the counter for an ended experiment" do
|
|
78
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
79
|
+
e.winner = "small"
|
|
80
|
+
expect {
|
|
81
|
+
a = ab_test("button_size", "small", "big")
|
|
82
|
+
expect(a).to eq("small")
|
|
83
|
+
}.not_to change { e.participant_count }
|
|
83
84
|
end
|
|
84
85
|
|
|
85
|
-
it
|
|
86
|
+
it "should not increment the counter for an not started experiment" do
|
|
86
87
|
expect(Split.configuration).to receive(:start_manually).and_return(true)
|
|
87
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
88
|
-
expect
|
|
89
|
-
a = ab_test(
|
|
90
|
-
expect(a).to eq(
|
|
91
|
-
}
|
|
88
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
89
|
+
expect {
|
|
90
|
+
a = ab_test("button_size", "small", "big")
|
|
91
|
+
expect(a).to eq("small")
|
|
92
|
+
}.not_to change { e.participant_count }
|
|
92
93
|
end
|
|
93
94
|
|
|
94
95
|
it "should return the given alternative for an existing user" do
|
|
95
|
-
expect(ab_test(
|
|
96
|
+
expect(ab_test("link_color", "blue", "red")).to eq ab_test("link_color", "blue", "red")
|
|
96
97
|
end
|
|
97
98
|
|
|
98
|
-
it
|
|
99
|
+
it "should always return the winner if one is present" do
|
|
99
100
|
experiment.winner = "orange"
|
|
100
101
|
|
|
101
|
-
expect(ab_test(
|
|
102
|
+
expect(ab_test("link_color", "blue", "red")).to eq("orange")
|
|
102
103
|
end
|
|
103
104
|
|
|
104
105
|
it "should allow the alternative to be forced by passing it in the params" do
|
|
105
106
|
# ?ab_test[link_color]=blue
|
|
106
|
-
@params = {
|
|
107
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
107
108
|
|
|
108
|
-
alternative = ab_test(
|
|
109
|
-
expect(alternative).to eq(
|
|
109
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
110
|
+
expect(alternative).to eq("blue")
|
|
110
111
|
|
|
111
|
-
alternative = ab_test(
|
|
112
|
-
expect(alternative).to eq(
|
|
112
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
113
|
+
expect(alternative).to eq("blue")
|
|
113
114
|
|
|
114
|
-
@params = {
|
|
115
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
115
116
|
|
|
116
|
-
alternative = ab_test(
|
|
117
|
-
expect(alternative).to eq(
|
|
117
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
118
|
+
expect(alternative).to eq("red")
|
|
118
119
|
|
|
119
|
-
alternative = ab_test(
|
|
120
|
-
expect(alternative).to eq(
|
|
120
|
+
alternative = ab_test("link_color", { "blue" => 5 }, "red" => 1)
|
|
121
|
+
expect(alternative).to eq("red")
|
|
121
122
|
end
|
|
122
123
|
|
|
123
124
|
it "should not allow an arbitrary alternative" do
|
|
124
|
-
@params = {
|
|
125
|
-
alternative = ab_test(
|
|
126
|
-
expect(alternative).to eq(
|
|
125
|
+
@params = { "ab_test" => { "link_color" => "pink" } }
|
|
126
|
+
alternative = ab_test("link_color", "blue")
|
|
127
|
+
expect(alternative).to eq("blue")
|
|
127
128
|
end
|
|
128
129
|
|
|
129
130
|
it "should not store the split when a param forced alternative" do
|
|
130
|
-
@params = {
|
|
131
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
131
132
|
expect(ab_user).not_to receive(:[]=)
|
|
132
|
-
ab_test(
|
|
133
|
+
ab_test("link_color", "blue", "red")
|
|
133
134
|
end
|
|
134
135
|
|
|
135
136
|
it "SPLIT_DISABLE query parameter should also force the alternative (uses control)" do
|
|
136
|
-
@params = {
|
|
137
|
-
alternative = ab_test(
|
|
138
|
-
expect(alternative).to eq(
|
|
139
|
-
alternative = ab_test(
|
|
140
|
-
expect(alternative).to eq(
|
|
141
|
-
alternative = ab_test(
|
|
142
|
-
expect(alternative).to eq(
|
|
143
|
-
alternative = ab_test(
|
|
144
|
-
expect(alternative).to eq(
|
|
137
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
138
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
139
|
+
expect(alternative).to eq("blue")
|
|
140
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
141
|
+
expect(alternative).to eq("blue")
|
|
142
|
+
alternative = ab_test("link_color", "red", "blue")
|
|
143
|
+
expect(alternative).to eq("red")
|
|
144
|
+
alternative = ab_test("link_color", { "red" => 5 }, "blue" => 1)
|
|
145
|
+
expect(alternative).to eq("red")
|
|
145
146
|
end
|
|
146
147
|
|
|
147
148
|
it "should not store the split when Split generically disabled" do
|
|
148
|
-
@params = {
|
|
149
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
149
150
|
expect(ab_user).not_to receive(:[]=)
|
|
150
|
-
ab_test(
|
|
151
|
+
ab_test("link_color", "blue", "red")
|
|
151
152
|
end
|
|
152
153
|
|
|
153
154
|
context "when store_override is set" do
|
|
154
155
|
before { Split.configuration.store_override = true }
|
|
155
156
|
|
|
156
157
|
it "should store the forced alternative" do
|
|
157
|
-
@params = {
|
|
158
|
-
expect(ab_user).to receive(:[]=).with(
|
|
159
|
-
ab_test(
|
|
158
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
159
|
+
expect(ab_user).to receive(:[]=).with("link_color", "blue")
|
|
160
|
+
ab_test("link_color", "blue", "red")
|
|
160
161
|
end
|
|
161
162
|
end
|
|
162
163
|
|
|
@@ -164,36 +165,35 @@ describe Split::Helper do
|
|
|
164
165
|
before { Split.configuration.on_trial_choose = :some_method }
|
|
165
166
|
it "should call the method" do
|
|
166
167
|
expect(self).to receive(:some_method)
|
|
167
|
-
ab_test(
|
|
168
|
+
ab_test("link_color", "blue", "red")
|
|
168
169
|
end
|
|
169
170
|
end
|
|
170
171
|
|
|
171
172
|
it "should allow passing a block" do
|
|
172
|
-
alt = ab_test(
|
|
173
|
-
ret = ab_test(
|
|
173
|
+
alt = ab_test("link_color", "blue", "red")
|
|
174
|
+
ret = ab_test("link_color", "blue", "red") { |alternative| "shared/#{alternative}" }
|
|
174
175
|
expect(ret).to eq("shared/#{alt}")
|
|
175
176
|
end
|
|
176
177
|
|
|
177
178
|
it "should allow the share of visitors see an alternative to be specified" do
|
|
178
|
-
ab_test(
|
|
179
|
-
expect([
|
|
179
|
+
ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })
|
|
180
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
180
181
|
end
|
|
181
182
|
|
|
182
183
|
it "should allow alternative weighting interface as a single hash" do
|
|
183
|
-
ab_test(
|
|
184
|
-
experiment = Split::ExperimentCatalog.find(
|
|
185
|
-
expect(experiment.alternatives.map(&:name)).to eq([
|
|
186
|
-
|
|
187
|
-
# expect(experiment.alternatives.collect{|a| a.weight}).to eq([0.01, 0.2])
|
|
184
|
+
ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)
|
|
185
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
186
|
+
expect(experiment.alternatives.map(&:name)).to eq(["blue", "red"])
|
|
187
|
+
expect(experiment.alternatives.collect { |a| a.weight }).to match_array([0.01, 0.2])
|
|
188
188
|
end
|
|
189
189
|
|
|
190
190
|
it "should only let a user participate in one experiment at a time" do
|
|
191
|
-
link_color = ab_test(
|
|
192
|
-
ab_test(
|
|
193
|
-
expect(ab_user[
|
|
194
|
-
big = Split::Alternative.new(
|
|
191
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
192
|
+
ab_test("button_size", "small", "big")
|
|
193
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
194
|
+
big = Split::Alternative.new("big", "button_size")
|
|
195
195
|
expect(big.participant_count).to eq(0)
|
|
196
|
-
small = Split::Alternative.new(
|
|
196
|
+
small = Split::Alternative.new("small", "button_size")
|
|
197
197
|
expect(small.participant_count).to eq(0)
|
|
198
198
|
end
|
|
199
199
|
|
|
@@ -201,194 +201,262 @@ describe Split::Helper do
|
|
|
201
201
|
Split.configure do |config|
|
|
202
202
|
config.allow_multiple_experiments = true
|
|
203
203
|
end
|
|
204
|
-
link_color = ab_test(
|
|
205
|
-
button_size = ab_test(
|
|
206
|
-
expect(ab_user[
|
|
207
|
-
expect(ab_user[
|
|
208
|
-
button_size_alt = Split::Alternative.new(button_size,
|
|
204
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
205
|
+
button_size = ab_test("button_size", "small", "big")
|
|
206
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
207
|
+
expect(ab_user["button_size"]).to eq(button_size)
|
|
208
|
+
button_size_alt = Split::Alternative.new(button_size, "button_size")
|
|
209
209
|
expect(button_size_alt.participant_count).to eq(1)
|
|
210
210
|
end
|
|
211
211
|
|
|
212
212
|
context "with allow_multiple_experiments = 'control'" do
|
|
213
213
|
it "should let a user participate in many experiment with one non-'control' alternative" do
|
|
214
214
|
Split.configure do |config|
|
|
215
|
-
config.allow_multiple_experiments =
|
|
215
|
+
config.allow_multiple_experiments = "control"
|
|
216
216
|
end
|
|
217
217
|
groups = 100.times.map do |n|
|
|
218
|
-
ab_test("test#{n}".to_sym, {
|
|
218
|
+
ab_test("test#{n}".to_sym, { "control" => (100 - n) }, { "test#{n}-alt" => n })
|
|
219
219
|
end
|
|
220
220
|
|
|
221
221
|
experiments = ab_user.active_experiments
|
|
222
222
|
expect(experiments.size).to be > 1
|
|
223
223
|
|
|
224
|
-
count_control = experiments.values.count { |g| g ==
|
|
224
|
+
count_control = experiments.values.count { |g| g == "control" }
|
|
225
225
|
expect(count_control).to eq(experiments.size - 1)
|
|
226
226
|
|
|
227
|
-
count_alts = groups.count { |g| g !=
|
|
227
|
+
count_alts = groups.count { |g| g != "control" }
|
|
228
228
|
expect(count_alts).to eq(1)
|
|
229
229
|
end
|
|
230
230
|
|
|
231
231
|
context "when user already has experiment" do
|
|
232
|
-
let(:mock_user){ Split::User.new(self, {
|
|
233
|
-
|
|
232
|
+
let(:mock_user) { Split::User.new(self, { "test_0" => "test-alt" }) }
|
|
233
|
+
|
|
234
|
+
before do
|
|
234
235
|
Split.configure do |config|
|
|
235
|
-
config.allow_multiple_experiments =
|
|
236
|
+
config.allow_multiple_experiments = "control"
|
|
236
237
|
end
|
|
237
|
-
|
|
238
|
-
Split::ExperimentCatalog.find_or_initialize(
|
|
239
|
-
|
|
238
|
+
|
|
239
|
+
Split::ExperimentCatalog.find_or_initialize("test_0", "control", "test-alt").save
|
|
240
|
+
Split::ExperimentCatalog.find_or_initialize("test_1", "control", "test-alt").save
|
|
241
|
+
end
|
|
240
242
|
|
|
241
243
|
it "should restore previously selected alternative" do
|
|
242
244
|
expect(ab_user.active_experiments.size).to eq 1
|
|
243
|
-
expect(ab_test(:test_0, {
|
|
244
|
-
expect(ab_test(:test_0, {
|
|
245
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
246
|
+
expect(ab_test(:test_0, { "control" => 1 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
it "should select the correct alternatives after experiment resets" do
|
|
250
|
+
experiment = Split::ExperimentCatalog.find(:test_0)
|
|
251
|
+
experiment.reset
|
|
252
|
+
mock_user[experiment.key] = "test-alt"
|
|
253
|
+
|
|
254
|
+
expect(ab_user.active_experiments.size).to eq 1
|
|
255
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
256
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
245
257
|
end
|
|
246
258
|
|
|
247
259
|
it "lets override existing choice" do
|
|
248
260
|
pending "this requires user store reset on first call not depending on whelther it is current trial"
|
|
249
|
-
@params = {
|
|
261
|
+
@params = { "ab_test" => { "test_1" => "test-alt" } }
|
|
250
262
|
|
|
251
|
-
expect(ab_test(:test_0, {
|
|
252
|
-
expect(ab_test(:test_1, {
|
|
263
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "control"
|
|
264
|
+
expect(ab_test(:test_1, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
253
265
|
end
|
|
254
|
-
|
|
255
266
|
end
|
|
256
|
-
|
|
257
267
|
end
|
|
258
268
|
|
|
259
269
|
it "should not over-write a finished key when an experiment is on a later version" do
|
|
260
270
|
experiment.increment_version
|
|
261
|
-
ab_user = { experiment.key =>
|
|
271
|
+
ab_user = { experiment.key => "blue", experiment.finished_key => true }
|
|
262
272
|
finished_session = ab_user.dup
|
|
263
|
-
ab_test(
|
|
273
|
+
ab_test("link_color", "blue", "red")
|
|
264
274
|
expect(ab_user).to eq(finished_session)
|
|
265
275
|
end
|
|
266
276
|
end
|
|
267
277
|
|
|
268
|
-
describe
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
:
|
|
273
|
-
|
|
274
|
-
|
|
278
|
+
describe "metadata" do
|
|
279
|
+
context "is defined" do
|
|
280
|
+
before do
|
|
281
|
+
Split.configuration.experiments = {
|
|
282
|
+
my_experiment: {
|
|
283
|
+
alternatives: ["one", "two"],
|
|
284
|
+
resettable: false,
|
|
285
|
+
metadata: { "one" => "Meta1", "two" => "Meta2" }
|
|
286
|
+
}
|
|
275
287
|
}
|
|
276
|
-
|
|
277
|
-
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
it "should be passed to helper block" do
|
|
291
|
+
@params = { "ab_test" => { "my_experiment" => "two" } }
|
|
292
|
+
expect(ab_test("my_experiment")).to eq "two"
|
|
293
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
294
|
+
meta
|
|
295
|
+
end).to eq("Meta2")
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
it "should pass control metadata helper block if library disabled" do
|
|
299
|
+
Split.configure do |config|
|
|
300
|
+
config.enabled = false
|
|
301
|
+
end
|
|
278
302
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
end).to eq('Meta1')
|
|
303
|
+
expect(ab_test("my_experiment")).to eq "one"
|
|
304
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
305
|
+
meta
|
|
306
|
+
end).to eq("Meta1")
|
|
307
|
+
end
|
|
285
308
|
end
|
|
286
309
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
310
|
+
context "is not defined" do
|
|
311
|
+
before do
|
|
312
|
+
Split.configuration.experiments = {
|
|
313
|
+
my_experiment: {
|
|
314
|
+
alternatives: ["one", "two"],
|
|
315
|
+
resettable: false,
|
|
316
|
+
metadata: nil
|
|
317
|
+
}
|
|
318
|
+
}
|
|
290
319
|
end
|
|
291
320
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
321
|
+
it "should be passed to helper block" do
|
|
322
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
323
|
+
meta
|
|
324
|
+
end).to eq({})
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
it "should pass control metadata helper block if library disabled" do
|
|
328
|
+
Split.configure do |config|
|
|
329
|
+
config.enabled = false
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
333
|
+
meta
|
|
334
|
+
end).to eq({})
|
|
335
|
+
end
|
|
296
336
|
end
|
|
297
337
|
end
|
|
298
338
|
|
|
299
|
-
describe
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
339
|
+
describe "ab_finished" do
|
|
340
|
+
context "for an experiment that the user participates in" do
|
|
341
|
+
before(:each) do
|
|
342
|
+
@experiment_name = "link_color"
|
|
343
|
+
@alternatives = ["blue", "red"]
|
|
344
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
345
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
346
|
+
@previous_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
347
|
+
end
|
|
307
348
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
349
|
+
it "should increment the counter for the completed alternative" do
|
|
350
|
+
ab_finished(@experiment_name)
|
|
351
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
352
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
353
|
+
end
|
|
313
354
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
355
|
+
it "should set experiment's finished key if reset is false" do
|
|
356
|
+
ab_finished(@experiment_name, { reset: false })
|
|
357
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
358
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
359
|
+
end
|
|
319
360
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
361
|
+
it "should not increment the counter if reset is false and the experiment has been already finished" do
|
|
362
|
+
2.times { ab_finished(@experiment_name, { reset: false }) }
|
|
363
|
+
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
364
|
+
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
365
|
+
end
|
|
325
366
|
|
|
326
|
-
|
|
327
|
-
|
|
367
|
+
it "should not increment the counter for an ended experiment" do
|
|
368
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
369
|
+
e.winner = "small"
|
|
370
|
+
a = ab_test("button_size", "small", "big")
|
|
371
|
+
expect(a).to eq("small")
|
|
372
|
+
expect {
|
|
373
|
+
ab_finished("button_size")
|
|
374
|
+
}.not_to change { Split::Alternative.new(a, "button_size").completed_count }
|
|
375
|
+
end
|
|
328
376
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
ab_finished('button_size')
|
|
335
|
-
}).not_to change { Split::Alternative.new('small', 'button_size').completed_count }
|
|
336
|
-
end
|
|
377
|
+
it "should clear out the user's participation from their session" do
|
|
378
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
379
|
+
ab_finished(@experiment_name)
|
|
380
|
+
expect(ab_user.keys).to be_empty
|
|
381
|
+
end
|
|
337
382
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
ab_finished('button_size')
|
|
345
|
-
}).not_to change { Split::Alternative.new(a, 'button_size').completed_count }
|
|
346
|
-
end
|
|
383
|
+
it "should not clear out the users session if reset is false" do
|
|
384
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
385
|
+
ab_finished(@experiment_name, { reset: false })
|
|
386
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
387
|
+
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
388
|
+
end
|
|
347
389
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
390
|
+
it "should reset the users session when experiment is not versioned" do
|
|
391
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
392
|
+
ab_finished(@experiment_name)
|
|
393
|
+
expect(ab_user.keys).to be_empty
|
|
394
|
+
end
|
|
353
395
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
358
|
-
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
359
|
-
end
|
|
396
|
+
it "should reset the users session when experiment is versioned" do
|
|
397
|
+
@experiment.increment_version
|
|
398
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
360
399
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
end
|
|
400
|
+
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
401
|
+
ab_finished(@experiment_name)
|
|
402
|
+
expect(ab_user.keys).to be_empty
|
|
403
|
+
end
|
|
366
404
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
405
|
+
context "when on_trial_complete is set" do
|
|
406
|
+
before { Split.configuration.on_trial_complete = :some_method }
|
|
407
|
+
it "should call the method" do
|
|
408
|
+
expect(self).to receive(:some_method)
|
|
409
|
+
ab_finished(@experiment_name)
|
|
410
|
+
end
|
|
370
411
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
412
|
+
it "should not call the method without alternative" do
|
|
413
|
+
ab_user[@experiment.key] = nil
|
|
414
|
+
expect(self).not_to receive(:some_method)
|
|
415
|
+
ab_finished(@experiment_name)
|
|
416
|
+
end
|
|
417
|
+
end
|
|
374
418
|
end
|
|
375
419
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
420
|
+
context "for an experiment that the user is excluded from" do
|
|
421
|
+
before do
|
|
422
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
423
|
+
expect(Split::Alternative.new(alternative, "link_color").participant_count).to eq(1)
|
|
424
|
+
alternative = ab_test("button_size", "small", "big")
|
|
425
|
+
expect(Split::Alternative.new(alternative, "button_size").participant_count).to eq(0)
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
it "should not increment the completed counter" do
|
|
429
|
+
# So, user should be participating in the link_color experiment and
|
|
430
|
+
# receive the control for button_size. As the user is not participating in
|
|
431
|
+
# the button size experiment, finishing it should not increase the
|
|
432
|
+
# completion count for that alternative.
|
|
433
|
+
expect {
|
|
434
|
+
ab_finished("button_size")
|
|
435
|
+
}.not_to change { Split::Alternative.new("small", "button_size").completed_count }
|
|
436
|
+
end
|
|
379
437
|
end
|
|
380
438
|
|
|
381
|
-
context "
|
|
382
|
-
before
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
439
|
+
context "for an experiment that the user does not participate in" do
|
|
440
|
+
before do
|
|
441
|
+
Split::ExperimentCatalog.find_or_create(:not_started_experiment, "control", "alt")
|
|
442
|
+
end
|
|
443
|
+
it "should not raise an exception" do
|
|
444
|
+
expect { ab_finished(:not_started_experiment) }.not_to raise_exception
|
|
386
445
|
end
|
|
387
446
|
|
|
388
|
-
it "should not
|
|
389
|
-
ab_user[
|
|
390
|
-
|
|
391
|
-
|
|
447
|
+
it "should not change the user state when reset is false" do
|
|
448
|
+
expect { ab_finished(:not_started_experiment, reset: false) }.not_to change { ab_user.keys }.from([])
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
it "should not change the user state when reset is true" do
|
|
452
|
+
expect(self).not_to receive(:reset!)
|
|
453
|
+
ab_finished(:not_started_experiment)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
it "should not increment the completed counter" do
|
|
457
|
+
ab_finished(:not_started_experiment)
|
|
458
|
+
expect(Split::Alternative.new("control", :not_started_experiment).completed_count).to eq(0)
|
|
459
|
+
expect(Split::Alternative.new("alt", :not_started_experiment).completed_count).to eq(0)
|
|
392
460
|
end
|
|
393
461
|
end
|
|
394
462
|
end
|
|
@@ -396,9 +464,9 @@ describe Split::Helper do
|
|
|
396
464
|
context "finished with config" do
|
|
397
465
|
it "passes reset option" do
|
|
398
466
|
Split.configuration.experiments = {
|
|
399
|
-
:
|
|
400
|
-
:
|
|
401
|
-
:
|
|
467
|
+
my_experiment: {
|
|
468
|
+
alternatives: ["one", "two"],
|
|
469
|
+
resettable: false,
|
|
402
470
|
}
|
|
403
471
|
}
|
|
404
472
|
alternative = ab_test(:my_experiment)
|
|
@@ -414,11 +482,11 @@ describe Split::Helper do
|
|
|
414
482
|
before { Split.configuration.experiments = {} }
|
|
415
483
|
before { expect(Split::Alternative).to receive(:new).at_least(1).times.and_call_original }
|
|
416
484
|
|
|
417
|
-
def should_finish_experiment(experiment_name, should_finish=true)
|
|
485
|
+
def should_finish_experiment(experiment_name, should_finish = true)
|
|
418
486
|
alts = Split.configuration.experiments[experiment_name][:alternatives]
|
|
419
487
|
experiment = Split::ExperimentCatalog.find_or_create(experiment_name, *alts)
|
|
420
488
|
alt_name = ab_user[experiment.key] = alts.first
|
|
421
|
-
alt = double(
|
|
489
|
+
alt = double("alternative")
|
|
422
490
|
expect(alt).to receive(:name).at_most(1).times.and_return(alt_name)
|
|
423
491
|
expect(Split::Alternative).to receive(:new).at_most(1).times.with(alt_name, experiment_name.to_s).and_return(alt)
|
|
424
492
|
if should_finish
|
|
@@ -430,8 +498,8 @@ describe Split::Helper do
|
|
|
430
498
|
|
|
431
499
|
it "completes the test" do
|
|
432
500
|
Split.configuration.experiments[:my_experiment] = {
|
|
433
|
-
:
|
|
434
|
-
:
|
|
501
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
502
|
+
metric: :my_metric
|
|
435
503
|
}
|
|
436
504
|
should_finish_experiment :my_experiment
|
|
437
505
|
ab_finished :my_metric
|
|
@@ -439,17 +507,17 @@ describe Split::Helper do
|
|
|
439
507
|
|
|
440
508
|
it "completes all relevant tests" do
|
|
441
509
|
Split.configuration.experiments = {
|
|
442
|
-
:
|
|
443
|
-
:
|
|
444
|
-
:
|
|
510
|
+
exp_1: {
|
|
511
|
+
alternatives: [ "1-1", "1-2" ],
|
|
512
|
+
metric: :my_metric
|
|
445
513
|
},
|
|
446
|
-
:
|
|
447
|
-
:
|
|
448
|
-
:
|
|
514
|
+
exp_2: {
|
|
515
|
+
alternatives: [ "2-1", "2-2" ],
|
|
516
|
+
metric: :another_metric
|
|
449
517
|
},
|
|
450
|
-
:
|
|
451
|
-
:
|
|
452
|
-
:
|
|
518
|
+
exp_3: {
|
|
519
|
+
alternatives: [ "3-1", "3-2" ],
|
|
520
|
+
metric: :my_metric
|
|
453
521
|
},
|
|
454
522
|
}
|
|
455
523
|
should_finish_experiment :exp_1
|
|
@@ -460,10 +528,10 @@ describe Split::Helper do
|
|
|
460
528
|
|
|
461
529
|
it "passes reset option" do
|
|
462
530
|
Split.configuration.experiments = {
|
|
463
|
-
:
|
|
464
|
-
:
|
|
465
|
-
:
|
|
466
|
-
:
|
|
531
|
+
my_exp: {
|
|
532
|
+
alternatives: ["one", "two"],
|
|
533
|
+
metric: :my_metric,
|
|
534
|
+
resettable: false,
|
|
467
535
|
}
|
|
468
536
|
}
|
|
469
537
|
alternative_name = ab_test(:my_exp)
|
|
@@ -476,183 +544,233 @@ describe Split::Helper do
|
|
|
476
544
|
|
|
477
545
|
it "passes through options" do
|
|
478
546
|
Split.configuration.experiments = {
|
|
479
|
-
:
|
|
480
|
-
:
|
|
481
|
-
:
|
|
547
|
+
my_exp: {
|
|
548
|
+
alternatives: ["one", "two"],
|
|
549
|
+
metric: :my_metric,
|
|
482
550
|
}
|
|
483
551
|
}
|
|
484
552
|
alternative_name = ab_test(:my_exp)
|
|
485
553
|
exp = Split::ExperimentCatalog.find :my_exp
|
|
486
554
|
|
|
487
|
-
ab_finished :my_metric, :
|
|
555
|
+
ab_finished :my_metric, reset: false
|
|
488
556
|
expect(ab_user[exp.key]).to eq(alternative_name)
|
|
489
557
|
expect(ab_user[exp.finished_key]).to be_truthy
|
|
490
558
|
end
|
|
491
559
|
end
|
|
492
560
|
|
|
493
|
-
describe 'conversions' do
|
|
494
|
-
it 'should return a conversion rate for an alternative' do
|
|
495
|
-
alternative_name = ab_test('link_color', 'blue', 'red')
|
|
496
561
|
|
|
497
|
-
|
|
562
|
+
describe "ab_record_extra_info" do
|
|
563
|
+
context "for an experiment that the user participates in" do
|
|
564
|
+
before(:each) do
|
|
565
|
+
@experiment_name = "link_color"
|
|
566
|
+
@alternatives = ["blue", "red"]
|
|
567
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
568
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
it "records extra data for a given experiment" do
|
|
572
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
573
|
+
|
|
574
|
+
ab_record_extra_info(@experiment_name, "some_data", 10)
|
|
575
|
+
|
|
576
|
+
expect(alternative.extra_info).to eql({ "some_data" => 10 })
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
it "records extra data for a given experiment" do
|
|
580
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
581
|
+
|
|
582
|
+
ab_record_extra_info(@experiment_name, "some_data")
|
|
583
|
+
|
|
584
|
+
expect(alternative.extra_info).to eql({ "some_data" => 1 })
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
it "records extra data for a given experiment" do
|
|
588
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
589
|
+
|
|
590
|
+
ab_record_extra_info(@experiment_name, "some_data", nil)
|
|
591
|
+
|
|
592
|
+
expect(alternative.extra_info).to eql({})
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
describe "conversions" do
|
|
598
|
+
it "should return a conversion rate for an alternative" do
|
|
599
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
600
|
+
|
|
601
|
+
previous_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
498
602
|
expect(previous_convertion_rate).to eq(0.0)
|
|
499
603
|
|
|
500
|
-
ab_finished(
|
|
604
|
+
ab_finished("link_color")
|
|
501
605
|
|
|
502
|
-
new_convertion_rate = Split::Alternative.new(alternative_name,
|
|
606
|
+
new_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
503
607
|
expect(new_convertion_rate).to eq(1.0)
|
|
504
608
|
end
|
|
505
609
|
end
|
|
506
610
|
|
|
507
|
-
describe
|
|
508
|
-
it
|
|
509
|
-
alternative = ab_test(
|
|
611
|
+
describe "active experiments" do
|
|
612
|
+
it "should show an active test" do
|
|
613
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
510
614
|
expect(active_experiments.count).to eq 1
|
|
511
615
|
expect(active_experiments.first[0]).to eq "def"
|
|
512
616
|
expect(active_experiments.first[1]).to eq alternative
|
|
513
617
|
end
|
|
514
618
|
|
|
515
|
-
it
|
|
516
|
-
alternative = ab_test(
|
|
517
|
-
ab_finished(
|
|
619
|
+
it "should show a finished test" do
|
|
620
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
621
|
+
ab_finished("def", { reset: false })
|
|
518
622
|
expect(active_experiments.count).to eq 1
|
|
519
623
|
expect(active_experiments.first[0]).to eq "def"
|
|
520
624
|
expect(active_experiments.first[1]).to eq alternative
|
|
521
625
|
end
|
|
522
626
|
|
|
523
|
-
it
|
|
627
|
+
it "should show an active test when an experiment is on a later version" do
|
|
524
628
|
experiment.reset
|
|
525
629
|
expect(experiment.version).to eq(1)
|
|
526
|
-
ab_test(
|
|
630
|
+
ab_test("link_color", "blue", "red")
|
|
527
631
|
expect(active_experiments.count).to eq 1
|
|
528
632
|
expect(active_experiments.first[0]).to eq "link_color"
|
|
529
633
|
end
|
|
530
634
|
|
|
531
|
-
it
|
|
635
|
+
it "should show versioned tests properly" do
|
|
636
|
+
10.times { experiment.reset }
|
|
637
|
+
|
|
638
|
+
alternative = ab_test(experiment.name, "blue", "red")
|
|
639
|
+
ab_finished(experiment.name, reset: false)
|
|
640
|
+
|
|
641
|
+
expect(experiment.version).to eq(10)
|
|
642
|
+
expect(active_experiments.count).to eq 1
|
|
643
|
+
expect(active_experiments).to eq({ "link_color" => alternative })
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
it "should show multiple tests" do
|
|
532
647
|
Split.configure do |config|
|
|
533
648
|
config.allow_multiple_experiments = true
|
|
534
649
|
end
|
|
535
|
-
alternative = ab_test(
|
|
536
|
-
another_alternative = ab_test(
|
|
650
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
651
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
537
652
|
expect(active_experiments.count).to eq 2
|
|
538
|
-
expect(active_experiments[
|
|
539
|
-
expect(active_experiments[
|
|
653
|
+
expect(active_experiments["def"]).to eq alternative
|
|
654
|
+
expect(active_experiments["ghi"]).to eq another_alternative
|
|
540
655
|
end
|
|
541
656
|
|
|
542
|
-
it
|
|
657
|
+
it "should not show tests with winners" do
|
|
543
658
|
Split.configure do |config|
|
|
544
659
|
config.allow_multiple_experiments = true
|
|
545
660
|
end
|
|
546
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
547
|
-
e.winner =
|
|
548
|
-
|
|
549
|
-
another_alternative = ab_test(
|
|
661
|
+
e = Split::ExperimentCatalog.find_or_create("def", "4", "5", "6")
|
|
662
|
+
e.winner = "4"
|
|
663
|
+
ab_test("def", "4", "5", "6")
|
|
664
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
550
665
|
expect(active_experiments.count).to eq 1
|
|
551
666
|
expect(active_experiments.first[0]).to eq "ghi"
|
|
552
667
|
expect(active_experiments.first[1]).to eq another_alternative
|
|
553
668
|
end
|
|
554
669
|
end
|
|
555
670
|
|
|
556
|
-
describe
|
|
671
|
+
describe "when user is a robot" do
|
|
557
672
|
before(:each) do
|
|
558
|
-
@request =
|
|
673
|
+
@request = build_request(user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)")
|
|
559
674
|
end
|
|
560
675
|
|
|
561
|
-
describe
|
|
562
|
-
it
|
|
563
|
-
alternative = ab_test(
|
|
676
|
+
describe "ab_test" do
|
|
677
|
+
it "should return the control" do
|
|
678
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
564
679
|
expect(alternative).to eq experiment.control.name
|
|
565
680
|
end
|
|
566
681
|
|
|
567
|
-
it "should not
|
|
682
|
+
it "should not create a experiment" do
|
|
683
|
+
ab_test("link_color", "blue", "red")
|
|
684
|
+
expect(Split::Experiment.new("link_color")).to be_a_new_record
|
|
685
|
+
end
|
|
568
686
|
|
|
569
|
-
|
|
570
|
-
|
|
687
|
+
it "should not increment the participation count" do
|
|
688
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
689
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
571
690
|
|
|
572
|
-
ab_test(
|
|
691
|
+
ab_test("link_color", "blue", "red")
|
|
573
692
|
|
|
574
|
-
new_red_count = Split::Alternative.new(
|
|
575
|
-
new_blue_count = Split::Alternative.new(
|
|
693
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
694
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
576
695
|
|
|
577
696
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
578
697
|
end
|
|
579
698
|
end
|
|
580
699
|
|
|
581
|
-
describe
|
|
700
|
+
describe "finished" do
|
|
582
701
|
it "should not increment the completed count" do
|
|
583
|
-
alternative_name = ab_test(
|
|
702
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
584
703
|
|
|
585
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
704
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
586
705
|
|
|
587
|
-
ab_finished(
|
|
706
|
+
ab_finished("link_color")
|
|
588
707
|
|
|
589
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
708
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
590
709
|
|
|
591
710
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
592
711
|
end
|
|
593
712
|
end
|
|
594
713
|
end
|
|
595
714
|
|
|
596
|
-
describe
|
|
715
|
+
describe "when providing custom ignore logic" do
|
|
597
716
|
context "using a proc to configure custom logic" do
|
|
598
|
-
|
|
599
717
|
before(:each) do
|
|
600
718
|
Split.configure do |c|
|
|
601
|
-
c.ignore_filter = proc{|request| true } # ignore everything
|
|
719
|
+
c.ignore_filter = proc { |request| true } # ignore everything
|
|
602
720
|
end
|
|
603
721
|
end
|
|
604
722
|
|
|
605
723
|
it "ignores the ab_test" do
|
|
606
|
-
ab_test(
|
|
724
|
+
ab_test("link_color", "blue", "red")
|
|
607
725
|
|
|
608
|
-
red_count = Split::Alternative.new(
|
|
609
|
-
blue_count = Split::Alternative.new(
|
|
726
|
+
red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
727
|
+
blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
610
728
|
expect((red_count + blue_count)).to be(0)
|
|
611
729
|
end
|
|
612
730
|
end
|
|
613
731
|
end
|
|
614
732
|
|
|
615
733
|
shared_examples_for "a disabled test" do
|
|
616
|
-
describe
|
|
617
|
-
it
|
|
618
|
-
alternative = ab_test(
|
|
734
|
+
describe "ab_test" do
|
|
735
|
+
it "should return the control" do
|
|
736
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
619
737
|
expect(alternative).to eq experiment.control.name
|
|
620
738
|
end
|
|
621
739
|
|
|
622
740
|
it "should not increment the participation count" do
|
|
623
|
-
previous_red_count = Split::Alternative.new(
|
|
624
|
-
previous_blue_count = Split::Alternative.new(
|
|
741
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
742
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
625
743
|
|
|
626
|
-
ab_test(
|
|
744
|
+
ab_test("link_color", "blue", "red")
|
|
627
745
|
|
|
628
|
-
new_red_count = Split::Alternative.new(
|
|
629
|
-
new_blue_count = Split::Alternative.new(
|
|
746
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
747
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
630
748
|
|
|
631
749
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
632
750
|
end
|
|
633
751
|
end
|
|
634
752
|
|
|
635
|
-
describe
|
|
753
|
+
describe "finished" do
|
|
636
754
|
it "should not increment the completed count" do
|
|
637
|
-
alternative_name = ab_test(
|
|
755
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
638
756
|
|
|
639
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
757
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
640
758
|
|
|
641
|
-
ab_finished(
|
|
759
|
+
ab_finished("link_color")
|
|
642
760
|
|
|
643
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
761
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
644
762
|
|
|
645
763
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
646
764
|
end
|
|
647
765
|
end
|
|
648
766
|
end
|
|
649
767
|
|
|
650
|
-
describe
|
|
768
|
+
describe "when ip address is ignored" do
|
|
651
769
|
context "individually" do
|
|
652
770
|
before(:each) do
|
|
653
|
-
@request =
|
|
771
|
+
@request = build_request(ip: "81.19.48.130")
|
|
654
772
|
Split.configure do |c|
|
|
655
|
-
c.ignore_ip_addresses <<
|
|
773
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
656
774
|
end
|
|
657
775
|
end
|
|
658
776
|
|
|
@@ -661,7 +779,7 @@ describe Split::Helper do
|
|
|
661
779
|
|
|
662
780
|
context "for a range" do
|
|
663
781
|
before(:each) do
|
|
664
|
-
@request =
|
|
782
|
+
@request = build_request(ip: "81.19.48.129")
|
|
665
783
|
Split.configure do |c|
|
|
666
784
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
667
785
|
end
|
|
@@ -672,9 +790,9 @@ describe Split::Helper do
|
|
|
672
790
|
|
|
673
791
|
context "using both a range and a specific value" do
|
|
674
792
|
before(:each) do
|
|
675
|
-
@request =
|
|
793
|
+
@request = build_request(ip: "81.19.48.128")
|
|
676
794
|
Split.configure do |c|
|
|
677
|
-
c.ignore_ip_addresses <<
|
|
795
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
678
796
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
679
797
|
end
|
|
680
798
|
end
|
|
@@ -684,111 +802,119 @@ describe Split::Helper do
|
|
|
684
802
|
|
|
685
803
|
context "when ignored other address" do
|
|
686
804
|
before do
|
|
687
|
-
@request =
|
|
805
|
+
@request = build_request(ip: "1.1.1.1")
|
|
688
806
|
Split.configure do |c|
|
|
689
|
-
c.ignore_ip_addresses <<
|
|
807
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
690
808
|
end
|
|
691
809
|
end
|
|
692
810
|
|
|
693
811
|
it "works as usual" do
|
|
694
|
-
alternative_name = ab_test(
|
|
695
|
-
expect{
|
|
696
|
-
ab_finished(
|
|
697
|
-
}.to change(Split::Alternative.new(alternative_name,
|
|
812
|
+
alternative_name = ab_test("link_color", "red", "blue")
|
|
813
|
+
expect {
|
|
814
|
+
ab_finished("link_color")
|
|
815
|
+
}.to change(Split::Alternative.new(alternative_name, "link_color"), :completed_count).by(1)
|
|
698
816
|
end
|
|
699
817
|
end
|
|
700
818
|
end
|
|
701
819
|
|
|
702
|
-
describe
|
|
820
|
+
describe "when user is previewing" do
|
|
821
|
+
before(:each) do
|
|
822
|
+
@request = build_request(headers: { "x-purpose" => "preview" })
|
|
823
|
+
end
|
|
824
|
+
|
|
825
|
+
it_behaves_like "a disabled test"
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
describe "versioned experiments" do
|
|
703
829
|
it "should use version zero if no version is present" do
|
|
704
|
-
alternative_name = ab_test(
|
|
830
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
705
831
|
expect(experiment.version).to eq(0)
|
|
706
|
-
expect(ab_user[
|
|
832
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
707
833
|
end
|
|
708
834
|
|
|
709
835
|
it "should save the version of the experiment to the session" do
|
|
710
836
|
experiment.reset
|
|
711
837
|
expect(experiment.version).to eq(1)
|
|
712
|
-
alternative_name = ab_test(
|
|
713
|
-
expect(ab_user[
|
|
838
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
839
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
714
840
|
end
|
|
715
841
|
|
|
716
842
|
it "should load the experiment even if the version is not 0" do
|
|
717
843
|
experiment.reset
|
|
718
844
|
expect(experiment.version).to eq(1)
|
|
719
|
-
alternative_name = ab_test(
|
|
720
|
-
expect(ab_user[
|
|
721
|
-
return_alternative_name = ab_test(
|
|
845
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
846
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
847
|
+
return_alternative_name = ab_test("link_color", "blue", "red")
|
|
722
848
|
expect(return_alternative_name).to eq(alternative_name)
|
|
723
849
|
end
|
|
724
850
|
|
|
725
851
|
it "should reset the session of a user on an older version of the experiment" do
|
|
726
|
-
alternative_name = ab_test(
|
|
727
|
-
expect(ab_user[
|
|
728
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
852
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
853
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
854
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
729
855
|
expect(alternative.participant_count).to eq(1)
|
|
730
856
|
|
|
731
857
|
experiment.reset
|
|
732
858
|
expect(experiment.version).to eq(1)
|
|
733
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
859
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
734
860
|
expect(alternative.participant_count).to eq(0)
|
|
735
861
|
|
|
736
|
-
new_alternative_name = ab_test(
|
|
737
|
-
expect(ab_user[
|
|
738
|
-
new_alternative = Split::Alternative.new(new_alternative_name,
|
|
862
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
863
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
864
|
+
new_alternative = Split::Alternative.new(new_alternative_name, "link_color")
|
|
739
865
|
expect(new_alternative.participant_count).to eq(1)
|
|
740
866
|
end
|
|
741
867
|
|
|
742
868
|
it "should cleanup old versions of experiments from the session" do
|
|
743
|
-
alternative_name = ab_test(
|
|
744
|
-
expect(ab_user[
|
|
745
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
869
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
870
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
871
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
746
872
|
expect(alternative.participant_count).to eq(1)
|
|
747
873
|
|
|
748
874
|
experiment.reset
|
|
749
875
|
expect(experiment.version).to eq(1)
|
|
750
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
876
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
751
877
|
expect(alternative.participant_count).to eq(0)
|
|
752
878
|
|
|
753
|
-
new_alternative_name = ab_test(
|
|
754
|
-
expect(ab_user[
|
|
879
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
880
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
755
881
|
end
|
|
756
882
|
|
|
757
883
|
it "should only count completion of users on the current version" do
|
|
758
|
-
alternative_name = ab_test(
|
|
759
|
-
expect(ab_user[
|
|
760
|
-
|
|
884
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
885
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
886
|
+
Split::Alternative.new(alternative_name, "link_color")
|
|
761
887
|
|
|
762
888
|
experiment.reset
|
|
763
889
|
expect(experiment.version).to eq(1)
|
|
764
890
|
|
|
765
|
-
ab_finished(
|
|
766
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
891
|
+
ab_finished("link_color")
|
|
892
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
767
893
|
expect(alternative.completed_count).to eq(0)
|
|
768
894
|
end
|
|
769
895
|
end
|
|
770
896
|
|
|
771
|
-
context
|
|
897
|
+
context "when redis is not available" do
|
|
772
898
|
before(:each) do
|
|
773
899
|
expect(Split).to receive(:redis).at_most(5).times.and_raise(Errno::ECONNREFUSED.new)
|
|
774
900
|
end
|
|
775
901
|
|
|
776
|
-
context
|
|
902
|
+
context "and db_failover config option is turned off" do
|
|
777
903
|
before(:each) do
|
|
778
904
|
Split.configure do |config|
|
|
779
905
|
config.db_failover = false
|
|
780
906
|
end
|
|
781
907
|
end
|
|
782
908
|
|
|
783
|
-
describe
|
|
784
|
-
it
|
|
785
|
-
expect
|
|
909
|
+
describe "ab_test" do
|
|
910
|
+
it "should raise an exception" do
|
|
911
|
+
expect { ab_test("link_color", "blue", "red") }.to raise_error(Errno::ECONNREFUSED)
|
|
786
912
|
end
|
|
787
913
|
end
|
|
788
914
|
|
|
789
|
-
describe
|
|
790
|
-
it
|
|
791
|
-
expect
|
|
915
|
+
describe "finished" do
|
|
916
|
+
it "should raise an exception" do
|
|
917
|
+
expect { ab_finished("link_color") }.to raise_error(Errno::ECONNREFUSED)
|
|
792
918
|
end
|
|
793
919
|
end
|
|
794
920
|
|
|
@@ -800,29 +926,29 @@ describe Split::Helper do
|
|
|
800
926
|
end
|
|
801
927
|
|
|
802
928
|
it "should not attempt to connect to redis" do
|
|
803
|
-
expect
|
|
929
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
804
930
|
end
|
|
805
931
|
|
|
806
932
|
it "should return control variable" do
|
|
807
|
-
expect(ab_test(
|
|
808
|
-
expect
|
|
933
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
934
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
809
935
|
end
|
|
810
936
|
end
|
|
811
937
|
end
|
|
812
938
|
|
|
813
|
-
context
|
|
939
|
+
context "and db_failover config option is turned on" do
|
|
814
940
|
before(:each) do
|
|
815
941
|
Split.configure do |config|
|
|
816
942
|
config.db_failover = true
|
|
817
943
|
end
|
|
818
944
|
end
|
|
819
945
|
|
|
820
|
-
describe
|
|
821
|
-
it
|
|
822
|
-
expect
|
|
946
|
+
describe "ab_test" do
|
|
947
|
+
it "should not raise an exception" do
|
|
948
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
823
949
|
end
|
|
824
950
|
|
|
825
|
-
it
|
|
951
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
826
952
|
Split.configure do |config|
|
|
827
953
|
config.db_failover_on_db_error = proc do |error|
|
|
828
954
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -830,43 +956,43 @@ describe Split::Helper do
|
|
|
830
956
|
end
|
|
831
957
|
|
|
832
958
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
833
|
-
ab_test(
|
|
959
|
+
ab_test("link_color", "blue", "red")
|
|
834
960
|
end
|
|
835
961
|
|
|
836
|
-
it
|
|
837
|
-
expect(ab_test(
|
|
838
|
-
expect(ab_test(
|
|
839
|
-
expect(ab_test(
|
|
840
|
-
expect(ab_test(
|
|
962
|
+
it "should always use first alternative" do
|
|
963
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
964
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("blue")
|
|
965
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("blue")
|
|
966
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
841
967
|
"shared/#{alternative}"
|
|
842
|
-
end).to eq(
|
|
968
|
+
end).to eq("shared/blue")
|
|
843
969
|
end
|
|
844
970
|
|
|
845
|
-
context
|
|
971
|
+
context "and db_failover_allow_parameter_override config option is turned on" do
|
|
846
972
|
before(:each) do
|
|
847
973
|
Split.configure do |config|
|
|
848
974
|
config.db_failover_allow_parameter_override = true
|
|
849
975
|
end
|
|
850
976
|
end
|
|
851
977
|
|
|
852
|
-
context
|
|
853
|
-
it
|
|
854
|
-
@params = {
|
|
855
|
-
expect(ab_test(
|
|
856
|
-
expect(ab_test(
|
|
857
|
-
expect(ab_test(
|
|
858
|
-
expect(ab_test(
|
|
859
|
-
expect(ab_test(
|
|
978
|
+
context "and given an override parameter" do
|
|
979
|
+
it "should use given override instead of the first alternative" do
|
|
980
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
981
|
+
expect(ab_test("link_color", "blue", "red")).to eq("red")
|
|
982
|
+
expect(ab_test("link_color", "blue", "red", "green")).to eq("red")
|
|
983
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("red")
|
|
984
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("red")
|
|
985
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
860
986
|
"shared/#{alternative}"
|
|
861
|
-
end).to eq(
|
|
987
|
+
end).to eq("shared/red")
|
|
862
988
|
end
|
|
863
989
|
end
|
|
864
990
|
end
|
|
865
991
|
|
|
866
|
-
context
|
|
992
|
+
context "and preloaded config given" do
|
|
867
993
|
before do
|
|
868
994
|
Split.configuration.experiments[:link_color] = {
|
|
869
|
-
:
|
|
995
|
+
alternatives: [ "blue", "red" ],
|
|
870
996
|
}
|
|
871
997
|
end
|
|
872
998
|
|
|
@@ -876,12 +1002,12 @@ describe Split::Helper do
|
|
|
876
1002
|
end
|
|
877
1003
|
end
|
|
878
1004
|
|
|
879
|
-
describe
|
|
880
|
-
it
|
|
881
|
-
expect
|
|
1005
|
+
describe "finished" do
|
|
1006
|
+
it "should not raise an exception" do
|
|
1007
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
882
1008
|
end
|
|
883
1009
|
|
|
884
|
-
it
|
|
1010
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
885
1011
|
Split.configure do |config|
|
|
886
1012
|
config.db_failover_on_db_error = proc do |error|
|
|
887
1013
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -889,19 +1015,19 @@ describe Split::Helper do
|
|
|
889
1015
|
end
|
|
890
1016
|
|
|
891
1017
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
892
|
-
ab_finished(
|
|
1018
|
+
ab_finished("link_color")
|
|
893
1019
|
end
|
|
894
1020
|
end
|
|
895
1021
|
end
|
|
896
1022
|
end
|
|
897
1023
|
|
|
898
1024
|
context "with preloaded config" do
|
|
899
|
-
before { Split.configuration.experiments = {}}
|
|
1025
|
+
before { Split.configuration.experiments = {} }
|
|
900
1026
|
|
|
901
1027
|
it "pulls options from config file" do
|
|
902
1028
|
Split.configuration.experiments[:my_experiment] = {
|
|
903
|
-
:
|
|
904
|
-
:
|
|
1029
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1030
|
+
goals: ["goal1", "goal2"]
|
|
905
1031
|
}
|
|
906
1032
|
ab_test :my_experiment
|
|
907
1033
|
expect(Split::Experiment.new(:my_experiment).alternatives.map(&:name)).to eq([ "control_opt", "other_opt" ])
|
|
@@ -910,8 +1036,8 @@ describe Split::Helper do
|
|
|
910
1036
|
|
|
911
1037
|
it "can be called multiple times" do
|
|
912
1038
|
Split.configuration.experiments[:my_experiment] = {
|
|
913
|
-
:
|
|
914
|
-
:
|
|
1039
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1040
|
+
goals: ["goal1", "goal2"]
|
|
915
1041
|
}
|
|
916
1042
|
5.times { ab_test :my_experiment }
|
|
917
1043
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -922,8 +1048,8 @@ describe Split::Helper do
|
|
|
922
1048
|
|
|
923
1049
|
it "accepts multiple goals" do
|
|
924
1050
|
Split.configuration.experiments[:my_experiment] = {
|
|
925
|
-
:
|
|
926
|
-
:
|
|
1051
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1052
|
+
goals: [ "goal1", "goal2", "goal3" ]
|
|
927
1053
|
}
|
|
928
1054
|
ab_test :my_experiment
|
|
929
1055
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -932,7 +1058,7 @@ describe Split::Helper do
|
|
|
932
1058
|
|
|
933
1059
|
it "allow specifying goals to be optional" do
|
|
934
1060
|
Split.configuration.experiments[:my_experiment] = {
|
|
935
|
-
:
|
|
1061
|
+
alternatives: [ "control_opt", "other_opt" ]
|
|
936
1062
|
}
|
|
937
1063
|
experiment = Split::Experiment.new(:my_experiment)
|
|
938
1064
|
expect(experiment.goals).to eq([])
|
|
@@ -940,7 +1066,7 @@ describe Split::Helper do
|
|
|
940
1066
|
|
|
941
1067
|
it "accepts multiple alternatives" do
|
|
942
1068
|
Split.configuration.experiments[:my_experiment] = {
|
|
943
|
-
:
|
|
1069
|
+
alternatives: [ "control_opt", "second_opt", "third_opt" ],
|
|
944
1070
|
}
|
|
945
1071
|
ab_test :my_experiment
|
|
946
1072
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -949,68 +1075,68 @@ describe Split::Helper do
|
|
|
949
1075
|
|
|
950
1076
|
it "accepts probability on alternatives" do
|
|
951
1077
|
Split.configuration.experiments[:my_experiment] = {
|
|
952
|
-
:
|
|
953
|
-
{ :
|
|
954
|
-
{ :
|
|
955
|
-
{ :
|
|
1078
|
+
alternatives: [
|
|
1079
|
+
{ name: "control_opt", percent: 67 },
|
|
1080
|
+
{ name: "second_opt", percent: 10 },
|
|
1081
|
+
{ name: "third_opt", percent: 23 },
|
|
956
1082
|
],
|
|
957
1083
|
}
|
|
958
1084
|
ab_test :my_experiment
|
|
959
1085
|
experiment = Split::Experiment.new(:my_experiment)
|
|
960
|
-
expect(experiment.alternatives.collect{|a| [a.name, a.weight]}).to eq([[
|
|
1086
|
+
expect(experiment.alternatives.collect { |a| [a.name, a.weight] }).to eq([["control_opt", 0.67], ["second_opt", 0.1], ["third_opt", 0.23]])
|
|
961
1087
|
end
|
|
962
1088
|
|
|
963
1089
|
it "accepts probability on some alternatives" do
|
|
964
1090
|
Split.configuration.experiments[:my_experiment] = {
|
|
965
|
-
:
|
|
966
|
-
{ :
|
|
1091
|
+
alternatives: [
|
|
1092
|
+
{ name: "control_opt", percent: 34 },
|
|
967
1093
|
"second_opt",
|
|
968
|
-
{ :
|
|
1094
|
+
{ name: "third_opt", percent: 23 },
|
|
969
1095
|
"fourth_opt",
|
|
970
1096
|
],
|
|
971
1097
|
}
|
|
972
1098
|
ab_test :my_experiment
|
|
973
1099
|
experiment = Split::Experiment.new(:my_experiment)
|
|
974
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
975
|
-
expect(names_and_weights).to eq([[
|
|
976
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1100
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1101
|
+
expect(names_and_weights).to eq([["control_opt", 0.34], ["second_opt", 0.215], ["third_opt", 0.23], ["fourth_opt", 0.215]])
|
|
1102
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
977
1103
|
end
|
|
978
1104
|
|
|
979
1105
|
it "allows name param without probability" do
|
|
980
1106
|
Split.configuration.experiments[:my_experiment] = {
|
|
981
|
-
:
|
|
982
|
-
{ :
|
|
1107
|
+
alternatives: [
|
|
1108
|
+
{ name: "control_opt" },
|
|
983
1109
|
"second_opt",
|
|
984
|
-
{ :
|
|
1110
|
+
{ name: "third_opt", percent: 64 },
|
|
985
1111
|
],
|
|
986
1112
|
}
|
|
987
1113
|
ab_test :my_experiment
|
|
988
1114
|
experiment = Split::Experiment.new(:my_experiment)
|
|
989
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
990
|
-
expect(names_and_weights).to eq([[
|
|
991
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1115
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1116
|
+
expect(names_and_weights).to eq([["control_opt", 0.18], ["second_opt", 0.18], ["third_opt", 0.64]])
|
|
1117
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
992
1118
|
end
|
|
993
1119
|
|
|
994
1120
|
it "fails gracefully if config is missing experiment" do
|
|
995
|
-
Split.configuration.experiments = { :
|
|
996
|
-
expect
|
|
1121
|
+
Split.configuration.experiments = { other_experiment: { foo: "Bar" } }
|
|
1122
|
+
expect { ab_test :my_experiment }.to raise_error(Split::ExperimentNotFound)
|
|
997
1123
|
end
|
|
998
1124
|
|
|
999
1125
|
it "fails gracefully if config is missing" do
|
|
1000
|
-
expect
|
|
1126
|
+
expect { Split.configuration.experiments = nil }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
1001
1127
|
end
|
|
1002
1128
|
|
|
1003
1129
|
it "fails gracefully if config is missing alternatives" do
|
|
1004
|
-
Split.configuration.experiments[:my_experiment] = { :
|
|
1005
|
-
expect
|
|
1130
|
+
Split.configuration.experiments[:my_experiment] = { foo: "Bar" }
|
|
1131
|
+
expect { ab_test :my_experiment }.to raise_error(NoMethodError)
|
|
1006
1132
|
end
|
|
1007
1133
|
end
|
|
1008
1134
|
|
|
1009
|
-
it
|
|
1010
|
-
experiment2 = Split::ExperimentCatalog.find_or_create(
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
ab_finished(
|
|
1135
|
+
it "should handle multiple experiments correctly" do
|
|
1136
|
+
experiment2 = Split::ExperimentCatalog.find_or_create("link_color2", "blue", "red")
|
|
1137
|
+
ab_test("link_color", "blue", "red")
|
|
1138
|
+
ab_test("link_color2", "blue", "red")
|
|
1139
|
+
ab_finished("link_color2")
|
|
1014
1140
|
|
|
1015
1141
|
experiment2.alternatives.each do |alt|
|
|
1016
1142
|
expect(alt.unfinished_count).to eq(0)
|
|
@@ -1019,8 +1145,8 @@ describe Split::Helper do
|
|
|
1019
1145
|
|
|
1020
1146
|
context "with goals" do
|
|
1021
1147
|
before do
|
|
1022
|
-
@experiment = {
|
|
1023
|
-
@alternatives = [
|
|
1148
|
+
@experiment = { "link_color" => ["purchase", "refund"] }
|
|
1149
|
+
@alternatives = ["blue", "red"]
|
|
1024
1150
|
@experiment_name, @goals = normalize_metric(@experiment)
|
|
1025
1151
|
@goal1 = @goals[0]
|
|
1026
1152
|
@goal2 = @goals[1]
|
|
@@ -1034,8 +1160,8 @@ describe Split::Helper do
|
|
|
1034
1160
|
describe "ab_test" do
|
|
1035
1161
|
it "should allow experiment goals interface as a single hash" do
|
|
1036
1162
|
ab_test(@experiment, *@alternatives)
|
|
1037
|
-
experiment = Split::ExperimentCatalog.find(
|
|
1038
|
-
expect(experiment.goals).to eq([
|
|
1163
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
1164
|
+
expect(experiment.goals).to eq(["purchase", "refund"])
|
|
1039
1165
|
end
|
|
1040
1166
|
end
|
|
1041
1167
|
|
|
@@ -1045,15 +1171,9 @@ describe Split::Helper do
|
|
|
1045
1171
|
end
|
|
1046
1172
|
|
|
1047
1173
|
it "should increment the counter for the specified-goal completed alternative" do
|
|
1048
|
-
expect(
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
}).not_to change {
|
|
1052
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
|
|
1053
|
-
}
|
|
1054
|
-
}).to change {
|
|
1055
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
|
|
1056
|
-
}.by(1)
|
|
1174
|
+
expect { ab_finished({ "link_color" => ["purchase"] }) }
|
|
1175
|
+
.to change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) }.by(0)
|
|
1176
|
+
.and change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) }.by(1)
|
|
1057
1177
|
end
|
|
1058
1178
|
end
|
|
1059
1179
|
end
|