split 3.4.1 → 4.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +1 -0
- data/.github/dependabot.yml +7 -0
- data/.github/workflows/ci.yml +76 -0
- data/.rubocop.yml +177 -4
- data/CHANGELOG.md +87 -0
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +2 -1
- data/README.md +37 -9
- data/Rakefile +5 -5
- data/gemfiles/5.2.gemfile +1 -3
- data/gemfiles/6.0.gemfile +1 -3
- data/gemfiles/{5.0.gemfile → 6.1.gemfile} +2 -4
- data/gemfiles/{5.1.gemfile → 7.0.gemfile} +2 -4
- data/lib/split/algorithms/block_randomization.rb +6 -6
- data/lib/split/algorithms/weighted_sample.rb +2 -1
- data/lib/split/algorithms/whiplash.rb +17 -18
- data/lib/split/algorithms.rb +14 -0
- data/lib/split/alternative.rb +22 -22
- data/lib/split/cache.rb +27 -0
- data/lib/split/combined_experiments_helper.rb +5 -4
- data/lib/split/configuration.rb +89 -94
- data/lib/split/dashboard/helpers.rb +7 -7
- data/lib/split/dashboard/pagination_helpers.rb +54 -54
- data/lib/split/dashboard/paginator.rb +1 -0
- data/lib/split/dashboard/public/dashboard.js +10 -0
- data/lib/split/dashboard/public/style.css +10 -2
- data/lib/split/dashboard/views/_controls.erb +13 -0
- data/lib/split/dashboard/views/_experiment.erb +2 -1
- data/lib/split/dashboard/views/index.erb +19 -4
- data/lib/split/dashboard.rb +42 -21
- data/lib/split/encapsulated_helper.rb +15 -8
- data/lib/split/engine.rb +1 -0
- data/lib/split/exceptions.rb +1 -0
- data/lib/split/experiment.rb +151 -124
- data/lib/split/experiment_catalog.rb +7 -8
- data/lib/split/extensions/string.rb +2 -1
- data/lib/split/goals_collection.rb +9 -10
- data/lib/split/helper.rb +50 -23
- data/lib/split/metric.rb +6 -6
- data/lib/split/persistence/cookie_adapter.rb +46 -44
- data/lib/split/persistence/dual_adapter.rb +7 -8
- data/lib/split/persistence/redis_adapter.rb +8 -4
- data/lib/split/persistence/session_adapter.rb +1 -2
- data/lib/split/persistence.rb +8 -6
- data/lib/split/redis_interface.rb +15 -29
- data/lib/split/trial.rb +43 -34
- data/lib/split/user.rb +25 -14
- data/lib/split/version.rb +2 -4
- data/lib/split/zscore.rb +2 -3
- data/lib/split.rb +34 -27
- data/spec/algorithms/block_randomization_spec.rb +6 -5
- data/spec/algorithms/weighted_sample_spec.rb +6 -5
- data/spec/algorithms/whiplash_spec.rb +4 -5
- data/spec/alternative_spec.rb +35 -36
- data/spec/cache_spec.rb +84 -0
- data/spec/combined_experiments_helper_spec.rb +18 -17
- data/spec/configuration_spec.rb +41 -45
- data/spec/dashboard/pagination_helpers_spec.rb +69 -67
- data/spec/dashboard/paginator_spec.rb +10 -9
- data/spec/dashboard_helpers_spec.rb +19 -18
- data/spec/dashboard_spec.rb +122 -38
- data/spec/encapsulated_helper_spec.rb +46 -22
- data/spec/experiment_catalog_spec.rb +14 -13
- data/spec/experiment_spec.rb +198 -118
- data/spec/goals_collection_spec.rb +18 -16
- data/spec/helper_spec.rb +454 -385
- data/spec/metric_spec.rb +14 -14
- data/spec/persistence/cookie_adapter_spec.rb +26 -11
- data/spec/persistence/dual_adapter_spec.rb +71 -71
- data/spec/persistence/redis_adapter_spec.rb +35 -27
- data/spec/persistence/session_adapter_spec.rb +2 -3
- data/spec/persistence_spec.rb +1 -2
- data/spec/redis_interface_spec.rb +25 -82
- data/spec/spec_helper.rb +35 -24
- data/spec/split_spec.rb +11 -11
- data/spec/support/cookies_mock.rb +1 -2
- data/spec/trial_spec.rb +102 -75
- data/spec/user_spec.rb +60 -29
- data/split.gemspec +22 -21
- metadata +43 -40
- data/.rubocop_todo.yml +0 -679
- data/.travis.yml +0 -60
- data/Appraisals +0 -19
- data/gemfiles/4.2.gemfile +0 -9
data/spec/helper_spec.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
require "spec_helper"
|
|
3
4
|
|
|
4
5
|
# TODO change some of these tests to use Rack::Test
|
|
5
6
|
|
|
@@ -7,156 +8,156 @@ describe Split::Helper do
|
|
|
7
8
|
include Split::Helper
|
|
8
9
|
|
|
9
10
|
let(:experiment) {
|
|
10
|
-
Split::ExperimentCatalog.find_or_create(
|
|
11
|
+
Split::ExperimentCatalog.find_or_create("link_color", "blue", "red")
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
describe "ab_test" do
|
|
14
15
|
it "should not raise an error when passed strings for alternatives" do
|
|
15
|
-
expect
|
|
16
|
+
expect { ab_test("xyz", "1", "2", "3") }.not_to raise_error
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
it "should not raise an error when passed an array for alternatives" do
|
|
19
|
-
expect
|
|
20
|
+
expect { ab_test("xyz", ["1", "2", "3"]) }.not_to raise_error
|
|
20
21
|
end
|
|
21
22
|
|
|
22
23
|
it "should raise the appropriate error when passed integers for alternatives" do
|
|
23
|
-
expect
|
|
24
|
+
expect { ab_test("xyz", 1, 2, 3) }.to raise_error(ArgumentError)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
it "should raise the appropriate error when passed symbols for alternatives" do
|
|
27
|
-
expect
|
|
28
|
+
expect { ab_test("xyz", :a, :b, :c) }.to raise_error(ArgumentError)
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
it "should not raise error when passed an array for goals" do
|
|
31
|
-
expect
|
|
32
|
+
expect { ab_test({ "link_color" => ["purchase", "refund"] }, "blue", "red") }.not_to raise_error
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
it "should not raise error when passed just one goal" do
|
|
35
|
-
expect
|
|
36
|
+
expect { ab_test({ "link_color" => "purchase" }, "blue", "red") }.not_to raise_error
|
|
36
37
|
end
|
|
37
38
|
|
|
38
39
|
it "raises an appropriate error when processing combined expirements" do
|
|
39
40
|
Split.configuration.experiments = {
|
|
40
|
-
:
|
|
41
|
-
:
|
|
42
|
-
:
|
|
43
|
-
:
|
|
41
|
+
combined_exp_1: {
|
|
42
|
+
alternatives: [ { name: "control", percent: 50 }, { name: "test-alt", percent: 50 } ],
|
|
43
|
+
metric: :my_metric,
|
|
44
|
+
combined_experiments: [:combined_exp_1_sub_1]
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
|
-
Split::ExperimentCatalog.find_or_create(
|
|
47
|
-
expect
|
|
47
|
+
Split::ExperimentCatalog.find_or_create("combined_exp_1")
|
|
48
|
+
expect { ab_test("combined_exp_1") }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do
|
|
51
|
-
ab_test(
|
|
52
|
-
expect([
|
|
52
|
+
ab_test("link_color", "blue", "red")
|
|
53
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
it "should increment the participation counter after assignment to a new user" do
|
|
56
|
-
previous_red_count = Split::Alternative.new(
|
|
57
|
-
previous_blue_count = Split::Alternative.new(
|
|
57
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
58
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
58
59
|
|
|
59
|
-
ab_test(
|
|
60
|
+
ab_test("link_color", "blue", "red")
|
|
60
61
|
|
|
61
|
-
new_red_count = Split::Alternative.new(
|
|
62
|
-
new_blue_count = Split::Alternative.new(
|
|
62
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
63
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
63
64
|
|
|
64
65
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count + 1)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
|
-
it
|
|
68
|
-
ab_test(
|
|
69
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
70
|
-
expect
|
|
68
|
+
it "should not increment the counter for an experiment that the user is not participating in" do
|
|
69
|
+
ab_test("link_color", "blue", "red")
|
|
70
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
71
|
+
expect {
|
|
71
72
|
# User shouldn't participate in this second experiment
|
|
72
|
-
ab_test(
|
|
73
|
-
}
|
|
73
|
+
ab_test("button_size", "small", "big")
|
|
74
|
+
}.not_to change { e.participant_count }
|
|
74
75
|
end
|
|
75
76
|
|
|
76
|
-
it
|
|
77
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
78
|
-
e.winner =
|
|
79
|
-
expect
|
|
80
|
-
a = ab_test(
|
|
81
|
-
expect(a).to eq(
|
|
82
|
-
}
|
|
77
|
+
it "should not increment the counter for an ended experiment" do
|
|
78
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
79
|
+
e.winner = "small"
|
|
80
|
+
expect {
|
|
81
|
+
a = ab_test("button_size", "small", "big")
|
|
82
|
+
expect(a).to eq("small")
|
|
83
|
+
}.not_to change { e.participant_count }
|
|
83
84
|
end
|
|
84
85
|
|
|
85
|
-
it
|
|
86
|
+
it "should not increment the counter for an not started experiment" do
|
|
86
87
|
expect(Split.configuration).to receive(:start_manually).and_return(true)
|
|
87
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
88
|
-
expect
|
|
89
|
-
a = ab_test(
|
|
90
|
-
expect(a).to eq(
|
|
91
|
-
}
|
|
88
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
89
|
+
expect {
|
|
90
|
+
a = ab_test("button_size", "small", "big")
|
|
91
|
+
expect(a).to eq("small")
|
|
92
|
+
}.not_to change { e.participant_count }
|
|
92
93
|
end
|
|
93
94
|
|
|
94
95
|
it "should return the given alternative for an existing user" do
|
|
95
|
-
expect(ab_test(
|
|
96
|
+
expect(ab_test("link_color", "blue", "red")).to eq ab_test("link_color", "blue", "red")
|
|
96
97
|
end
|
|
97
98
|
|
|
98
|
-
it
|
|
99
|
+
it "should always return the winner if one is present" do
|
|
99
100
|
experiment.winner = "orange"
|
|
100
101
|
|
|
101
|
-
expect(ab_test(
|
|
102
|
+
expect(ab_test("link_color", "blue", "red")).to eq("orange")
|
|
102
103
|
end
|
|
103
104
|
|
|
104
105
|
it "should allow the alternative to be forced by passing it in the params" do
|
|
105
106
|
# ?ab_test[link_color]=blue
|
|
106
|
-
@params = {
|
|
107
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
107
108
|
|
|
108
|
-
alternative = ab_test(
|
|
109
|
-
expect(alternative).to eq(
|
|
109
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
110
|
+
expect(alternative).to eq("blue")
|
|
110
111
|
|
|
111
|
-
alternative = ab_test(
|
|
112
|
-
expect(alternative).to eq(
|
|
112
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
113
|
+
expect(alternative).to eq("blue")
|
|
113
114
|
|
|
114
|
-
@params = {
|
|
115
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
115
116
|
|
|
116
|
-
alternative = ab_test(
|
|
117
|
-
expect(alternative).to eq(
|
|
117
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
118
|
+
expect(alternative).to eq("red")
|
|
118
119
|
|
|
119
|
-
alternative = ab_test(
|
|
120
|
-
expect(alternative).to eq(
|
|
120
|
+
alternative = ab_test("link_color", { "blue" => 5 }, "red" => 1)
|
|
121
|
+
expect(alternative).to eq("red")
|
|
121
122
|
end
|
|
122
123
|
|
|
123
124
|
it "should not allow an arbitrary alternative" do
|
|
124
|
-
@params = {
|
|
125
|
-
alternative = ab_test(
|
|
126
|
-
expect(alternative).to eq(
|
|
125
|
+
@params = { "ab_test" => { "link_color" => "pink" } }
|
|
126
|
+
alternative = ab_test("link_color", "blue")
|
|
127
|
+
expect(alternative).to eq("blue")
|
|
127
128
|
end
|
|
128
129
|
|
|
129
130
|
it "should not store the split when a param forced alternative" do
|
|
130
|
-
@params = {
|
|
131
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
131
132
|
expect(ab_user).not_to receive(:[]=)
|
|
132
|
-
ab_test(
|
|
133
|
+
ab_test("link_color", "blue", "red")
|
|
133
134
|
end
|
|
134
135
|
|
|
135
136
|
it "SPLIT_DISABLE query parameter should also force the alternative (uses control)" do
|
|
136
|
-
@params = {
|
|
137
|
-
alternative = ab_test(
|
|
138
|
-
expect(alternative).to eq(
|
|
139
|
-
alternative = ab_test(
|
|
140
|
-
expect(alternative).to eq(
|
|
141
|
-
alternative = ab_test(
|
|
142
|
-
expect(alternative).to eq(
|
|
143
|
-
alternative = ab_test(
|
|
144
|
-
expect(alternative).to eq(
|
|
137
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
138
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
139
|
+
expect(alternative).to eq("blue")
|
|
140
|
+
alternative = ab_test("link_color", { "blue" => 1 }, "red" => 5)
|
|
141
|
+
expect(alternative).to eq("blue")
|
|
142
|
+
alternative = ab_test("link_color", "red", "blue")
|
|
143
|
+
expect(alternative).to eq("red")
|
|
144
|
+
alternative = ab_test("link_color", { "red" => 5 }, "blue" => 1)
|
|
145
|
+
expect(alternative).to eq("red")
|
|
145
146
|
end
|
|
146
147
|
|
|
147
148
|
it "should not store the split when Split generically disabled" do
|
|
148
|
-
@params = {
|
|
149
|
+
@params = { "SPLIT_DISABLE" => "true" }
|
|
149
150
|
expect(ab_user).not_to receive(:[]=)
|
|
150
|
-
ab_test(
|
|
151
|
+
ab_test("link_color", "blue", "red")
|
|
151
152
|
end
|
|
152
153
|
|
|
153
154
|
context "when store_override is set" do
|
|
154
155
|
before { Split.configuration.store_override = true }
|
|
155
156
|
|
|
156
157
|
it "should store the forced alternative" do
|
|
157
|
-
@params = {
|
|
158
|
-
expect(ab_user).to receive(:[]=).with(
|
|
159
|
-
ab_test(
|
|
158
|
+
@params = { "ab_test" => { "link_color" => "blue" } }
|
|
159
|
+
expect(ab_user).to receive(:[]=).with("link_color", "blue")
|
|
160
|
+
ab_test("link_color", "blue", "red")
|
|
160
161
|
end
|
|
161
162
|
end
|
|
162
163
|
|
|
@@ -164,35 +165,35 @@ describe Split::Helper do
|
|
|
164
165
|
before { Split.configuration.on_trial_choose = :some_method }
|
|
165
166
|
it "should call the method" do
|
|
166
167
|
expect(self).to receive(:some_method)
|
|
167
|
-
ab_test(
|
|
168
|
+
ab_test("link_color", "blue", "red")
|
|
168
169
|
end
|
|
169
170
|
end
|
|
170
171
|
|
|
171
172
|
it "should allow passing a block" do
|
|
172
|
-
alt = ab_test(
|
|
173
|
-
ret = ab_test(
|
|
173
|
+
alt = ab_test("link_color", "blue", "red")
|
|
174
|
+
ret = ab_test("link_color", "blue", "red") { |alternative| "shared/#{alternative}" }
|
|
174
175
|
expect(ret).to eq("shared/#{alt}")
|
|
175
176
|
end
|
|
176
177
|
|
|
177
178
|
it "should allow the share of visitors see an alternative to be specified" do
|
|
178
|
-
ab_test(
|
|
179
|
-
expect([
|
|
179
|
+
ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })
|
|
180
|
+
expect(["red", "blue"]).to include(ab_user["link_color"])
|
|
180
181
|
end
|
|
181
182
|
|
|
182
183
|
it "should allow alternative weighting interface as a single hash" do
|
|
183
|
-
ab_test(
|
|
184
|
-
experiment = Split::ExperimentCatalog.find(
|
|
185
|
-
expect(experiment.alternatives.map(&:name)).to eq([
|
|
186
|
-
expect(experiment.alternatives.collect{|a| a.weight}).to match_array([0.01, 0.2])
|
|
184
|
+
ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)
|
|
185
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
186
|
+
expect(experiment.alternatives.map(&:name)).to eq(["blue", "red"])
|
|
187
|
+
expect(experiment.alternatives.collect { |a| a.weight }).to match_array([0.01, 0.2])
|
|
187
188
|
end
|
|
188
189
|
|
|
189
190
|
it "should only let a user participate in one experiment at a time" do
|
|
190
|
-
link_color = ab_test(
|
|
191
|
-
ab_test(
|
|
192
|
-
expect(ab_user[
|
|
193
|
-
big = Split::Alternative.new(
|
|
191
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
192
|
+
ab_test("button_size", "small", "big")
|
|
193
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
194
|
+
big = Split::Alternative.new("big", "button_size")
|
|
194
195
|
expect(big.participant_count).to eq(0)
|
|
195
|
-
small = Split::Alternative.new(
|
|
196
|
+
small = Split::Alternative.new("small", "button_size")
|
|
196
197
|
expect(small.participant_count).to eq(0)
|
|
197
198
|
end
|
|
198
199
|
|
|
@@ -200,137 +201,177 @@ describe Split::Helper do
|
|
|
200
201
|
Split.configure do |config|
|
|
201
202
|
config.allow_multiple_experiments = true
|
|
202
203
|
end
|
|
203
|
-
link_color = ab_test(
|
|
204
|
-
button_size = ab_test(
|
|
205
|
-
expect(ab_user[
|
|
206
|
-
expect(ab_user[
|
|
207
|
-
button_size_alt = Split::Alternative.new(button_size,
|
|
204
|
+
link_color = ab_test("link_color", "blue", "red")
|
|
205
|
+
button_size = ab_test("button_size", "small", "big")
|
|
206
|
+
expect(ab_user["link_color"]).to eq(link_color)
|
|
207
|
+
expect(ab_user["button_size"]).to eq(button_size)
|
|
208
|
+
button_size_alt = Split::Alternative.new(button_size, "button_size")
|
|
208
209
|
expect(button_size_alt.participant_count).to eq(1)
|
|
209
210
|
end
|
|
210
211
|
|
|
211
212
|
context "with allow_multiple_experiments = 'control'" do
|
|
212
213
|
it "should let a user participate in many experiment with one non-'control' alternative" do
|
|
213
214
|
Split.configure do |config|
|
|
214
|
-
config.allow_multiple_experiments =
|
|
215
|
+
config.allow_multiple_experiments = "control"
|
|
215
216
|
end
|
|
216
217
|
groups = 100.times.map do |n|
|
|
217
|
-
ab_test("test#{n}".to_sym, {
|
|
218
|
+
ab_test("test#{n}".to_sym, { "control" => (100 - n) }, { "test#{n}-alt" => n })
|
|
218
219
|
end
|
|
219
220
|
|
|
220
221
|
experiments = ab_user.active_experiments
|
|
221
222
|
expect(experiments.size).to be > 1
|
|
222
223
|
|
|
223
|
-
count_control = experiments.values.count { |g| g ==
|
|
224
|
+
count_control = experiments.values.count { |g| g == "control" }
|
|
224
225
|
expect(count_control).to eq(experiments.size - 1)
|
|
225
226
|
|
|
226
|
-
count_alts = groups.count { |g| g !=
|
|
227
|
+
count_alts = groups.count { |g| g != "control" }
|
|
227
228
|
expect(count_alts).to eq(1)
|
|
228
229
|
end
|
|
229
230
|
|
|
230
231
|
context "when user already has experiment" do
|
|
231
|
-
let(:mock_user){ Split::User.new(self, {
|
|
232
|
-
|
|
232
|
+
let(:mock_user) { Split::User.new(self, { "test_0" => "test-alt" }) }
|
|
233
|
+
|
|
234
|
+
before do
|
|
233
235
|
Split.configure do |config|
|
|
234
|
-
config.allow_multiple_experiments =
|
|
236
|
+
config.allow_multiple_experiments = "control"
|
|
235
237
|
end
|
|
236
|
-
|
|
237
|
-
Split::ExperimentCatalog.find_or_initialize(
|
|
238
|
-
|
|
238
|
+
|
|
239
|
+
Split::ExperimentCatalog.find_or_initialize("test_0", "control", "test-alt").save
|
|
240
|
+
Split::ExperimentCatalog.find_or_initialize("test_1", "control", "test-alt").save
|
|
241
|
+
end
|
|
239
242
|
|
|
240
243
|
it "should restore previously selected alternative" do
|
|
241
244
|
expect(ab_user.active_experiments.size).to eq 1
|
|
242
|
-
expect(ab_test(:test_0, {
|
|
243
|
-
expect(ab_test(:test_0, {
|
|
245
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
246
|
+
expect(ab_test(:test_0, { "control" => 1 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
it "should select the correct alternatives after experiment resets" do
|
|
250
|
+
experiment = Split::ExperimentCatalog.find(:test_0)
|
|
251
|
+
experiment.reset
|
|
252
|
+
mock_user[experiment.key] = "test-alt"
|
|
253
|
+
|
|
254
|
+
expect(ab_user.active_experiments.size).to eq 1
|
|
255
|
+
expect(ab_test(:test_0, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
256
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "test-alt"
|
|
244
257
|
end
|
|
245
258
|
|
|
246
259
|
it "lets override existing choice" do
|
|
247
260
|
pending "this requires user store reset on first call not depending on whelther it is current trial"
|
|
248
|
-
@params = {
|
|
261
|
+
@params = { "ab_test" => { "test_1" => "test-alt" } }
|
|
249
262
|
|
|
250
|
-
expect(ab_test(:test_0, {
|
|
251
|
-
expect(ab_test(:test_1, {
|
|
263
|
+
expect(ab_test(:test_0, { "control" => 0 }, { "test-alt" => 100 })).to eq "control"
|
|
264
|
+
expect(ab_test(:test_1, { "control" => 100 }, { "test-alt" => 1 })).to eq "test-alt"
|
|
252
265
|
end
|
|
253
|
-
|
|
254
266
|
end
|
|
255
|
-
|
|
256
267
|
end
|
|
257
268
|
|
|
258
269
|
it "should not over-write a finished key when an experiment is on a later version" do
|
|
259
270
|
experiment.increment_version
|
|
260
|
-
ab_user = { experiment.key =>
|
|
271
|
+
ab_user = { experiment.key => "blue", experiment.finished_key => true }
|
|
261
272
|
finished_session = ab_user.dup
|
|
262
|
-
ab_test(
|
|
273
|
+
ab_test("link_color", "blue", "red")
|
|
263
274
|
expect(ab_user).to eq(finished_session)
|
|
264
275
|
end
|
|
265
276
|
end
|
|
266
277
|
|
|
267
|
-
describe
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
:
|
|
272
|
-
|
|
273
|
-
|
|
278
|
+
describe "metadata" do
|
|
279
|
+
context "is defined" do
|
|
280
|
+
before do
|
|
281
|
+
Split.configuration.experiments = {
|
|
282
|
+
my_experiment: {
|
|
283
|
+
alternatives: ["one", "two"],
|
|
284
|
+
resettable: false,
|
|
285
|
+
metadata: { "one" => "Meta1", "two" => "Meta2" }
|
|
286
|
+
}
|
|
274
287
|
}
|
|
275
|
-
|
|
276
|
-
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
it "should be passed to helper block" do
|
|
291
|
+
@params = { "ab_test" => { "my_experiment" => "two" } }
|
|
292
|
+
expect(ab_test("my_experiment")).to eq "two"
|
|
293
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
294
|
+
meta
|
|
295
|
+
end).to eq("Meta2")
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
it "should pass control metadata helper block if library disabled" do
|
|
299
|
+
Split.configure do |config|
|
|
300
|
+
config.enabled = false
|
|
301
|
+
end
|
|
277
302
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
end).to eq('Meta1')
|
|
303
|
+
expect(ab_test("my_experiment")).to eq "one"
|
|
304
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
305
|
+
meta
|
|
306
|
+
end).to eq("Meta1")
|
|
307
|
+
end
|
|
284
308
|
end
|
|
285
309
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
310
|
+
context "is not defined" do
|
|
311
|
+
before do
|
|
312
|
+
Split.configuration.experiments = {
|
|
313
|
+
my_experiment: {
|
|
314
|
+
alternatives: ["one", "two"],
|
|
315
|
+
resettable: false,
|
|
316
|
+
metadata: nil
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
it "should be passed to helper block" do
|
|
322
|
+
expect(ab_test("my_experiment") do |alternative, meta|
|
|
323
|
+
meta
|
|
324
|
+
end).to eq({})
|
|
289
325
|
end
|
|
290
326
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
327
|
+
it "should pass control metadata helper block if library disabled" do
|
|
328
|
+
Split.configure do |config|
|
|
329
|
+
config.enabled = false
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
expect(ab_test("my_experiment") do |_, meta|
|
|
333
|
+
meta
|
|
334
|
+
end).to eq({})
|
|
335
|
+
end
|
|
295
336
|
end
|
|
296
337
|
end
|
|
297
338
|
|
|
298
|
-
describe
|
|
299
|
-
context
|
|
339
|
+
describe "ab_finished" do
|
|
340
|
+
context "for an experiment that the user participates in" do
|
|
300
341
|
before(:each) do
|
|
301
|
-
@experiment_name =
|
|
302
|
-
@alternatives = [
|
|
342
|
+
@experiment_name = "link_color"
|
|
343
|
+
@alternatives = ["blue", "red"]
|
|
303
344
|
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
304
345
|
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
305
346
|
@previous_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
306
347
|
end
|
|
307
348
|
|
|
308
|
-
it
|
|
349
|
+
it "should increment the counter for the completed alternative" do
|
|
309
350
|
ab_finished(@experiment_name)
|
|
310
351
|
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
311
352
|
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
312
353
|
end
|
|
313
354
|
|
|
314
355
|
it "should set experiment's finished key if reset is false" do
|
|
315
|
-
ab_finished(@experiment_name, {:
|
|
356
|
+
ab_finished(@experiment_name, { reset: false })
|
|
316
357
|
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
317
358
|
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
318
359
|
end
|
|
319
360
|
|
|
320
|
-
it
|
|
321
|
-
2.times { ab_finished(@experiment_name, {:
|
|
361
|
+
it "should not increment the counter if reset is false and the experiment has been already finished" do
|
|
362
|
+
2.times { ab_finished(@experiment_name, { reset: false }) }
|
|
322
363
|
new_completion_count = Split::Alternative.new(@alternative_name, @experiment_name).completed_count
|
|
323
364
|
expect(new_completion_count).to eq(@previous_completion_count + 1)
|
|
324
365
|
end
|
|
325
366
|
|
|
326
|
-
it
|
|
327
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
328
|
-
e.winner =
|
|
329
|
-
a = ab_test(
|
|
330
|
-
expect(a).to eq(
|
|
331
|
-
expect
|
|
332
|
-
ab_finished(
|
|
333
|
-
}
|
|
367
|
+
it "should not increment the counter for an ended experiment" do
|
|
368
|
+
e = Split::ExperimentCatalog.find_or_create("button_size", "small", "big")
|
|
369
|
+
e.winner = "small"
|
|
370
|
+
a = ab_test("button_size", "small", "big")
|
|
371
|
+
expect(a).to eq("small")
|
|
372
|
+
expect {
|
|
373
|
+
ab_finished("button_size")
|
|
374
|
+
}.not_to change { Split::Alternative.new(a, "button_size").completed_count }
|
|
334
375
|
end
|
|
335
376
|
|
|
336
377
|
it "should clear out the user's participation from their session" do
|
|
@@ -341,7 +382,7 @@ describe Split::Helper do
|
|
|
341
382
|
|
|
342
383
|
it "should not clear out the users session if reset is false" do
|
|
343
384
|
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
344
|
-
ab_finished(@experiment_name, {:
|
|
385
|
+
ab_finished(@experiment_name, { reset: false })
|
|
345
386
|
expect(ab_user[@experiment.key]).to eq(@alternative_name)
|
|
346
387
|
expect(ab_user[@experiment.finished_key]).to eq(true)
|
|
347
388
|
end
|
|
@@ -376,46 +417,46 @@ describe Split::Helper do
|
|
|
376
417
|
end
|
|
377
418
|
end
|
|
378
419
|
|
|
379
|
-
context
|
|
420
|
+
context "for an experiment that the user is excluded from" do
|
|
380
421
|
before do
|
|
381
|
-
alternative = ab_test(
|
|
382
|
-
expect(Split::Alternative.new(alternative,
|
|
383
|
-
alternative = ab_test(
|
|
384
|
-
expect(Split::Alternative.new(alternative,
|
|
422
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
423
|
+
expect(Split::Alternative.new(alternative, "link_color").participant_count).to eq(1)
|
|
424
|
+
alternative = ab_test("button_size", "small", "big")
|
|
425
|
+
expect(Split::Alternative.new(alternative, "button_size").participant_count).to eq(0)
|
|
385
426
|
end
|
|
386
427
|
|
|
387
|
-
it
|
|
428
|
+
it "should not increment the completed counter" do
|
|
388
429
|
# So, user should be participating in the link_color experiment and
|
|
389
430
|
# receive the control for button_size. As the user is not participating in
|
|
390
431
|
# the button size experiment, finishing it should not increase the
|
|
391
432
|
# completion count for that alternative.
|
|
392
|
-
expect
|
|
393
|
-
ab_finished(
|
|
394
|
-
}
|
|
433
|
+
expect {
|
|
434
|
+
ab_finished("button_size")
|
|
435
|
+
}.not_to change { Split::Alternative.new("small", "button_size").completed_count }
|
|
395
436
|
end
|
|
396
437
|
end
|
|
397
438
|
|
|
398
|
-
context
|
|
439
|
+
context "for an experiment that the user does not participate in" do
|
|
399
440
|
before do
|
|
400
|
-
Split::ExperimentCatalog.find_or_create(:not_started_experiment,
|
|
441
|
+
Split::ExperimentCatalog.find_or_create(:not_started_experiment, "control", "alt")
|
|
401
442
|
end
|
|
402
|
-
it
|
|
443
|
+
it "should not raise an exception" do
|
|
403
444
|
expect { ab_finished(:not_started_experiment) }.not_to raise_exception
|
|
404
445
|
end
|
|
405
446
|
|
|
406
|
-
it
|
|
407
|
-
expect { ab_finished(:not_started_experiment, reset: false) }.not_to change { ab_user.keys}.from([])
|
|
447
|
+
it "should not change the user state when reset is false" do
|
|
448
|
+
expect { ab_finished(:not_started_experiment, reset: false) }.not_to change { ab_user.keys }.from([])
|
|
408
449
|
end
|
|
409
450
|
|
|
410
|
-
it
|
|
451
|
+
it "should not change the user state when reset is true" do
|
|
411
452
|
expect(self).not_to receive(:reset!)
|
|
412
453
|
ab_finished(:not_started_experiment)
|
|
413
454
|
end
|
|
414
455
|
|
|
415
|
-
it
|
|
456
|
+
it "should not increment the completed counter" do
|
|
416
457
|
ab_finished(:not_started_experiment)
|
|
417
|
-
expect(Split::Alternative.new(
|
|
418
|
-
expect(Split::Alternative.new(
|
|
458
|
+
expect(Split::Alternative.new("control", :not_started_experiment).completed_count).to eq(0)
|
|
459
|
+
expect(Split::Alternative.new("alt", :not_started_experiment).completed_count).to eq(0)
|
|
419
460
|
end
|
|
420
461
|
end
|
|
421
462
|
end
|
|
@@ -423,9 +464,9 @@ describe Split::Helper do
|
|
|
423
464
|
context "finished with config" do
|
|
424
465
|
it "passes reset option" do
|
|
425
466
|
Split.configuration.experiments = {
|
|
426
|
-
:
|
|
427
|
-
:
|
|
428
|
-
:
|
|
467
|
+
my_experiment: {
|
|
468
|
+
alternatives: ["one", "two"],
|
|
469
|
+
resettable: false,
|
|
429
470
|
}
|
|
430
471
|
}
|
|
431
472
|
alternative = ab_test(:my_experiment)
|
|
@@ -441,11 +482,11 @@ describe Split::Helper do
|
|
|
441
482
|
before { Split.configuration.experiments = {} }
|
|
442
483
|
before { expect(Split::Alternative).to receive(:new).at_least(1).times.and_call_original }
|
|
443
484
|
|
|
444
|
-
def should_finish_experiment(experiment_name, should_finish=true)
|
|
485
|
+
def should_finish_experiment(experiment_name, should_finish = true)
|
|
445
486
|
alts = Split.configuration.experiments[experiment_name][:alternatives]
|
|
446
487
|
experiment = Split::ExperimentCatalog.find_or_create(experiment_name, *alts)
|
|
447
488
|
alt_name = ab_user[experiment.key] = alts.first
|
|
448
|
-
alt = double(
|
|
489
|
+
alt = double("alternative")
|
|
449
490
|
expect(alt).to receive(:name).at_most(1).times.and_return(alt_name)
|
|
450
491
|
expect(Split::Alternative).to receive(:new).at_most(1).times.with(alt_name, experiment_name.to_s).and_return(alt)
|
|
451
492
|
if should_finish
|
|
@@ -457,8 +498,8 @@ describe Split::Helper do
|
|
|
457
498
|
|
|
458
499
|
it "completes the test" do
|
|
459
500
|
Split.configuration.experiments[:my_experiment] = {
|
|
460
|
-
:
|
|
461
|
-
:
|
|
501
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
502
|
+
metric: :my_metric
|
|
462
503
|
}
|
|
463
504
|
should_finish_experiment :my_experiment
|
|
464
505
|
ab_finished :my_metric
|
|
@@ -466,17 +507,17 @@ describe Split::Helper do
|
|
|
466
507
|
|
|
467
508
|
it "completes all relevant tests" do
|
|
468
509
|
Split.configuration.experiments = {
|
|
469
|
-
:
|
|
470
|
-
:
|
|
471
|
-
:
|
|
510
|
+
exp_1: {
|
|
511
|
+
alternatives: [ "1-1", "1-2" ],
|
|
512
|
+
metric: :my_metric
|
|
472
513
|
},
|
|
473
|
-
:
|
|
474
|
-
:
|
|
475
|
-
:
|
|
514
|
+
exp_2: {
|
|
515
|
+
alternatives: [ "2-1", "2-2" ],
|
|
516
|
+
metric: :another_metric
|
|
476
517
|
},
|
|
477
|
-
:
|
|
478
|
-
:
|
|
479
|
-
:
|
|
518
|
+
exp_3: {
|
|
519
|
+
alternatives: [ "3-1", "3-2" ],
|
|
520
|
+
metric: :my_metric
|
|
480
521
|
},
|
|
481
522
|
}
|
|
482
523
|
should_finish_experiment :exp_1
|
|
@@ -487,10 +528,10 @@ describe Split::Helper do
|
|
|
487
528
|
|
|
488
529
|
it "passes reset option" do
|
|
489
530
|
Split.configuration.experiments = {
|
|
490
|
-
:
|
|
491
|
-
:
|
|
492
|
-
:
|
|
493
|
-
:
|
|
531
|
+
my_exp: {
|
|
532
|
+
alternatives: ["one", "two"],
|
|
533
|
+
metric: :my_metric,
|
|
534
|
+
resettable: false,
|
|
494
535
|
}
|
|
495
536
|
}
|
|
496
537
|
alternative_name = ab_test(:my_exp)
|
|
@@ -503,199 +544,233 @@ describe Split::Helper do
|
|
|
503
544
|
|
|
504
545
|
it "passes through options" do
|
|
505
546
|
Split.configuration.experiments = {
|
|
506
|
-
:
|
|
507
|
-
:
|
|
508
|
-
:
|
|
547
|
+
my_exp: {
|
|
548
|
+
alternatives: ["one", "two"],
|
|
549
|
+
metric: :my_metric,
|
|
509
550
|
}
|
|
510
551
|
}
|
|
511
552
|
alternative_name = ab_test(:my_exp)
|
|
512
553
|
exp = Split::ExperimentCatalog.find :my_exp
|
|
513
554
|
|
|
514
|
-
ab_finished :my_metric, :
|
|
555
|
+
ab_finished :my_metric, reset: false
|
|
515
556
|
expect(ab_user[exp.key]).to eq(alternative_name)
|
|
516
557
|
expect(ab_user[exp.finished_key]).to be_truthy
|
|
517
558
|
end
|
|
518
559
|
end
|
|
519
560
|
|
|
520
|
-
describe 'conversions' do
|
|
521
|
-
it 'should return a conversion rate for an alternative' do
|
|
522
|
-
alternative_name = ab_test('link_color', 'blue', 'red')
|
|
523
561
|
|
|
524
|
-
|
|
562
|
+
describe "ab_record_extra_info" do
|
|
563
|
+
context "for an experiment that the user participates in" do
|
|
564
|
+
before(:each) do
|
|
565
|
+
@experiment_name = "link_color"
|
|
566
|
+
@alternatives = ["blue", "red"]
|
|
567
|
+
@experiment = Split::ExperimentCatalog.find_or_create(@experiment_name, *@alternatives)
|
|
568
|
+
@alternative_name = ab_test(@experiment_name, *@alternatives)
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
it "records extra data for a given experiment" do
|
|
572
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
573
|
+
|
|
574
|
+
ab_record_extra_info(@experiment_name, "some_data", 10)
|
|
575
|
+
|
|
576
|
+
expect(alternative.extra_info).to eql({ "some_data" => 10 })
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
it "records extra data for a given experiment" do
|
|
580
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
581
|
+
|
|
582
|
+
ab_record_extra_info(@experiment_name, "some_data")
|
|
583
|
+
|
|
584
|
+
expect(alternative.extra_info).to eql({ "some_data" => 1 })
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
it "records extra data for a given experiment" do
|
|
588
|
+
alternative = Split::Alternative.new(@alternative_name, "link_color")
|
|
589
|
+
|
|
590
|
+
ab_record_extra_info(@experiment_name, "some_data", nil)
|
|
591
|
+
|
|
592
|
+
expect(alternative.extra_info).to eql({})
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
describe "conversions" do
|
|
598
|
+
it "should return a conversion rate for an alternative" do
|
|
599
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
600
|
+
|
|
601
|
+
previous_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
525
602
|
expect(previous_convertion_rate).to eq(0.0)
|
|
526
603
|
|
|
527
|
-
ab_finished(
|
|
604
|
+
ab_finished("link_color")
|
|
528
605
|
|
|
529
|
-
new_convertion_rate = Split::Alternative.new(alternative_name,
|
|
606
|
+
new_convertion_rate = Split::Alternative.new(alternative_name, "link_color").conversion_rate
|
|
530
607
|
expect(new_convertion_rate).to eq(1.0)
|
|
531
608
|
end
|
|
532
609
|
end
|
|
533
610
|
|
|
534
|
-
describe
|
|
535
|
-
it
|
|
536
|
-
alternative = ab_test(
|
|
611
|
+
describe "active experiments" do
|
|
612
|
+
it "should show an active test" do
|
|
613
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
537
614
|
expect(active_experiments.count).to eq 1
|
|
538
615
|
expect(active_experiments.first[0]).to eq "def"
|
|
539
616
|
expect(active_experiments.first[1]).to eq alternative
|
|
540
617
|
end
|
|
541
618
|
|
|
542
|
-
it
|
|
543
|
-
alternative = ab_test(
|
|
544
|
-
ab_finished(
|
|
619
|
+
it "should show a finished test" do
|
|
620
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
621
|
+
ab_finished("def", { reset: false })
|
|
545
622
|
expect(active_experiments.count).to eq 1
|
|
546
623
|
expect(active_experiments.first[0]).to eq "def"
|
|
547
624
|
expect(active_experiments.first[1]).to eq alternative
|
|
548
625
|
end
|
|
549
626
|
|
|
550
|
-
it
|
|
627
|
+
it "should show an active test when an experiment is on a later version" do
|
|
551
628
|
experiment.reset
|
|
552
629
|
expect(experiment.version).to eq(1)
|
|
553
|
-
ab_test(
|
|
630
|
+
ab_test("link_color", "blue", "red")
|
|
554
631
|
expect(active_experiments.count).to eq 1
|
|
555
632
|
expect(active_experiments.first[0]).to eq "link_color"
|
|
556
633
|
end
|
|
557
634
|
|
|
558
|
-
it
|
|
635
|
+
it "should show versioned tests properly" do
|
|
559
636
|
10.times { experiment.reset }
|
|
560
637
|
|
|
561
|
-
alternative = ab_test(experiment.name,
|
|
638
|
+
alternative = ab_test(experiment.name, "blue", "red")
|
|
562
639
|
ab_finished(experiment.name, reset: false)
|
|
563
640
|
|
|
564
641
|
expect(experiment.version).to eq(10)
|
|
565
642
|
expect(active_experiments.count).to eq 1
|
|
566
|
-
expect(active_experiments).to eq({
|
|
643
|
+
expect(active_experiments).to eq({ "link_color" => alternative })
|
|
567
644
|
end
|
|
568
645
|
|
|
569
|
-
it
|
|
646
|
+
it "should show multiple tests" do
|
|
570
647
|
Split.configure do |config|
|
|
571
648
|
config.allow_multiple_experiments = true
|
|
572
649
|
end
|
|
573
|
-
alternative = ab_test(
|
|
574
|
-
another_alternative = ab_test(
|
|
650
|
+
alternative = ab_test("def", "4", "5", "6")
|
|
651
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
575
652
|
expect(active_experiments.count).to eq 2
|
|
576
|
-
expect(active_experiments[
|
|
577
|
-
expect(active_experiments[
|
|
653
|
+
expect(active_experiments["def"]).to eq alternative
|
|
654
|
+
expect(active_experiments["ghi"]).to eq another_alternative
|
|
578
655
|
end
|
|
579
656
|
|
|
580
|
-
it
|
|
657
|
+
it "should not show tests with winners" do
|
|
581
658
|
Split.configure do |config|
|
|
582
659
|
config.allow_multiple_experiments = true
|
|
583
660
|
end
|
|
584
|
-
e = Split::ExperimentCatalog.find_or_create(
|
|
585
|
-
e.winner =
|
|
586
|
-
ab_test(
|
|
587
|
-
another_alternative = ab_test(
|
|
661
|
+
e = Split::ExperimentCatalog.find_or_create("def", "4", "5", "6")
|
|
662
|
+
e.winner = "4"
|
|
663
|
+
ab_test("def", "4", "5", "6")
|
|
664
|
+
another_alternative = ab_test("ghi", "7", "8", "9")
|
|
588
665
|
expect(active_experiments.count).to eq 1
|
|
589
666
|
expect(active_experiments.first[0]).to eq "ghi"
|
|
590
667
|
expect(active_experiments.first[1]).to eq another_alternative
|
|
591
668
|
end
|
|
592
669
|
end
|
|
593
670
|
|
|
594
|
-
describe
|
|
671
|
+
describe "when user is a robot" do
|
|
595
672
|
before(:each) do
|
|
596
|
-
@request = OpenStruct.new(:
|
|
673
|
+
@request = OpenStruct.new(user_agent: "Googlebot/2.1 (+http://www.google.com/bot.html)")
|
|
597
674
|
end
|
|
598
675
|
|
|
599
|
-
describe
|
|
600
|
-
it
|
|
601
|
-
alternative = ab_test(
|
|
676
|
+
describe "ab_test" do
|
|
677
|
+
it "should return the control" do
|
|
678
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
602
679
|
expect(alternative).to eq experiment.control.name
|
|
603
680
|
end
|
|
604
681
|
|
|
605
|
-
it
|
|
606
|
-
ab_test(
|
|
607
|
-
expect(Split::Experiment.new(
|
|
682
|
+
it "should not create a experiment" do
|
|
683
|
+
ab_test("link_color", "blue", "red")
|
|
684
|
+
expect(Split::Experiment.new("link_color")).to be_a_new_record
|
|
608
685
|
end
|
|
609
686
|
|
|
610
687
|
it "should not increment the participation count" do
|
|
688
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
689
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
611
690
|
|
|
612
|
-
|
|
613
|
-
previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
|
|
691
|
+
ab_test("link_color", "blue", "red")
|
|
614
692
|
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
new_red_count = Split::Alternative.new('red', 'link_color').participant_count
|
|
618
|
-
new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count
|
|
693
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
694
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
619
695
|
|
|
620
696
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
621
697
|
end
|
|
622
698
|
end
|
|
623
699
|
|
|
624
|
-
describe
|
|
700
|
+
describe "finished" do
|
|
625
701
|
it "should not increment the completed count" do
|
|
626
|
-
alternative_name = ab_test(
|
|
702
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
627
703
|
|
|
628
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
704
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
629
705
|
|
|
630
|
-
ab_finished(
|
|
706
|
+
ab_finished("link_color")
|
|
631
707
|
|
|
632
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
708
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
633
709
|
|
|
634
710
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
635
711
|
end
|
|
636
712
|
end
|
|
637
713
|
end
|
|
638
714
|
|
|
639
|
-
describe
|
|
715
|
+
describe "when providing custom ignore logic" do
|
|
640
716
|
context "using a proc to configure custom logic" do
|
|
641
|
-
|
|
642
717
|
before(:each) do
|
|
643
718
|
Split.configure do |c|
|
|
644
|
-
c.ignore_filter = proc{|request| true } # ignore everything
|
|
719
|
+
c.ignore_filter = proc { |request| true } # ignore everything
|
|
645
720
|
end
|
|
646
721
|
end
|
|
647
722
|
|
|
648
723
|
it "ignores the ab_test" do
|
|
649
|
-
ab_test(
|
|
724
|
+
ab_test("link_color", "blue", "red")
|
|
650
725
|
|
|
651
|
-
red_count = Split::Alternative.new(
|
|
652
|
-
blue_count = Split::Alternative.new(
|
|
726
|
+
red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
727
|
+
blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
653
728
|
expect((red_count + blue_count)).to be(0)
|
|
654
729
|
end
|
|
655
730
|
end
|
|
656
731
|
end
|
|
657
732
|
|
|
658
733
|
shared_examples_for "a disabled test" do
|
|
659
|
-
describe
|
|
660
|
-
it
|
|
661
|
-
alternative = ab_test(
|
|
734
|
+
describe "ab_test" do
|
|
735
|
+
it "should return the control" do
|
|
736
|
+
alternative = ab_test("link_color", "blue", "red")
|
|
662
737
|
expect(alternative).to eq experiment.control.name
|
|
663
738
|
end
|
|
664
739
|
|
|
665
740
|
it "should not increment the participation count" do
|
|
666
|
-
previous_red_count = Split::Alternative.new(
|
|
667
|
-
previous_blue_count = Split::Alternative.new(
|
|
741
|
+
previous_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
742
|
+
previous_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
668
743
|
|
|
669
|
-
ab_test(
|
|
744
|
+
ab_test("link_color", "blue", "red")
|
|
670
745
|
|
|
671
|
-
new_red_count = Split::Alternative.new(
|
|
672
|
-
new_blue_count = Split::Alternative.new(
|
|
746
|
+
new_red_count = Split::Alternative.new("red", "link_color").participant_count
|
|
747
|
+
new_blue_count = Split::Alternative.new("blue", "link_color").participant_count
|
|
673
748
|
|
|
674
749
|
expect((new_red_count + new_blue_count)).to eq(previous_red_count + previous_blue_count)
|
|
675
750
|
end
|
|
676
751
|
end
|
|
677
752
|
|
|
678
|
-
describe
|
|
753
|
+
describe "finished" do
|
|
679
754
|
it "should not increment the completed count" do
|
|
680
|
-
alternative_name = ab_test(
|
|
755
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
681
756
|
|
|
682
|
-
previous_completion_count = Split::Alternative.new(alternative_name,
|
|
757
|
+
previous_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
683
758
|
|
|
684
|
-
ab_finished(
|
|
759
|
+
ab_finished("link_color")
|
|
685
760
|
|
|
686
|
-
new_completion_count = Split::Alternative.new(alternative_name,
|
|
761
|
+
new_completion_count = Split::Alternative.new(alternative_name, "link_color").completed_count
|
|
687
762
|
|
|
688
763
|
expect(new_completion_count).to eq(previous_completion_count)
|
|
689
764
|
end
|
|
690
765
|
end
|
|
691
766
|
end
|
|
692
767
|
|
|
693
|
-
describe
|
|
768
|
+
describe "when ip address is ignored" do
|
|
694
769
|
context "individually" do
|
|
695
770
|
before(:each) do
|
|
696
|
-
@request = OpenStruct.new(:
|
|
771
|
+
@request = OpenStruct.new(ip: "81.19.48.130")
|
|
697
772
|
Split.configure do |c|
|
|
698
|
-
c.ignore_ip_addresses <<
|
|
773
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
699
774
|
end
|
|
700
775
|
end
|
|
701
776
|
|
|
@@ -704,7 +779,7 @@ describe Split::Helper do
|
|
|
704
779
|
|
|
705
780
|
context "for a range" do
|
|
706
781
|
before(:each) do
|
|
707
|
-
@request = OpenStruct.new(:
|
|
782
|
+
@request = OpenStruct.new(ip: "81.19.48.129")
|
|
708
783
|
Split.configure do |c|
|
|
709
784
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
710
785
|
end
|
|
@@ -715,9 +790,9 @@ describe Split::Helper do
|
|
|
715
790
|
|
|
716
791
|
context "using both a range and a specific value" do
|
|
717
792
|
before(:each) do
|
|
718
|
-
@request = OpenStruct.new(:
|
|
793
|
+
@request = OpenStruct.new(ip: "81.19.48.128")
|
|
719
794
|
Split.configure do |c|
|
|
720
|
-
c.ignore_ip_addresses <<
|
|
795
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
721
796
|
c.ignore_ip_addresses << /81\.19\.48\.[0-9]+/
|
|
722
797
|
end
|
|
723
798
|
end
|
|
@@ -727,119 +802,119 @@ describe Split::Helper do
|
|
|
727
802
|
|
|
728
803
|
context "when ignored other address" do
|
|
729
804
|
before do
|
|
730
|
-
@request = OpenStruct.new(:
|
|
805
|
+
@request = OpenStruct.new(ip: "1.1.1.1")
|
|
731
806
|
Split.configure do |c|
|
|
732
|
-
c.ignore_ip_addresses <<
|
|
807
|
+
c.ignore_ip_addresses << "81.19.48.130"
|
|
733
808
|
end
|
|
734
809
|
end
|
|
735
810
|
|
|
736
811
|
it "works as usual" do
|
|
737
|
-
alternative_name = ab_test(
|
|
738
|
-
expect{
|
|
739
|
-
ab_finished(
|
|
740
|
-
}.to change(Split::Alternative.new(alternative_name,
|
|
812
|
+
alternative_name = ab_test("link_color", "red", "blue")
|
|
813
|
+
expect {
|
|
814
|
+
ab_finished("link_color")
|
|
815
|
+
}.to change(Split::Alternative.new(alternative_name, "link_color"), :completed_count).by(1)
|
|
741
816
|
end
|
|
742
817
|
end
|
|
743
818
|
end
|
|
744
819
|
|
|
745
|
-
describe
|
|
820
|
+
describe "when user is previewing" do
|
|
746
821
|
before(:each) do
|
|
747
|
-
@request = OpenStruct.new(headers: {
|
|
822
|
+
@request = OpenStruct.new(headers: { "x-purpose" => "preview" })
|
|
748
823
|
end
|
|
749
824
|
|
|
750
825
|
it_behaves_like "a disabled test"
|
|
751
826
|
end
|
|
752
827
|
|
|
753
|
-
describe
|
|
828
|
+
describe "versioned experiments" do
|
|
754
829
|
it "should use version zero if no version is present" do
|
|
755
|
-
alternative_name = ab_test(
|
|
830
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
756
831
|
expect(experiment.version).to eq(0)
|
|
757
|
-
expect(ab_user[
|
|
832
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
758
833
|
end
|
|
759
834
|
|
|
760
835
|
it "should save the version of the experiment to the session" do
|
|
761
836
|
experiment.reset
|
|
762
837
|
expect(experiment.version).to eq(1)
|
|
763
|
-
alternative_name = ab_test(
|
|
764
|
-
expect(ab_user[
|
|
838
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
839
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
765
840
|
end
|
|
766
841
|
|
|
767
842
|
it "should load the experiment even if the version is not 0" do
|
|
768
843
|
experiment.reset
|
|
769
844
|
expect(experiment.version).to eq(1)
|
|
770
|
-
alternative_name = ab_test(
|
|
771
|
-
expect(ab_user[
|
|
772
|
-
return_alternative_name = ab_test(
|
|
845
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
846
|
+
expect(ab_user["link_color:1"]).to eq(alternative_name)
|
|
847
|
+
return_alternative_name = ab_test("link_color", "blue", "red")
|
|
773
848
|
expect(return_alternative_name).to eq(alternative_name)
|
|
774
849
|
end
|
|
775
850
|
|
|
776
851
|
it "should reset the session of a user on an older version of the experiment" do
|
|
777
|
-
alternative_name = ab_test(
|
|
778
|
-
expect(ab_user[
|
|
779
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
852
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
853
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
854
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
780
855
|
expect(alternative.participant_count).to eq(1)
|
|
781
856
|
|
|
782
857
|
experiment.reset
|
|
783
858
|
expect(experiment.version).to eq(1)
|
|
784
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
859
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
785
860
|
expect(alternative.participant_count).to eq(0)
|
|
786
861
|
|
|
787
|
-
new_alternative_name = ab_test(
|
|
788
|
-
expect(ab_user[
|
|
789
|
-
new_alternative = Split::Alternative.new(new_alternative_name,
|
|
862
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
863
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
864
|
+
new_alternative = Split::Alternative.new(new_alternative_name, "link_color")
|
|
790
865
|
expect(new_alternative.participant_count).to eq(1)
|
|
791
866
|
end
|
|
792
867
|
|
|
793
868
|
it "should cleanup old versions of experiments from the session" do
|
|
794
|
-
alternative_name = ab_test(
|
|
795
|
-
expect(ab_user[
|
|
796
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
869
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
870
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
871
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
797
872
|
expect(alternative.participant_count).to eq(1)
|
|
798
873
|
|
|
799
874
|
experiment.reset
|
|
800
875
|
expect(experiment.version).to eq(1)
|
|
801
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
876
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
802
877
|
expect(alternative.participant_count).to eq(0)
|
|
803
878
|
|
|
804
|
-
new_alternative_name = ab_test(
|
|
805
|
-
expect(ab_user[
|
|
879
|
+
new_alternative_name = ab_test("link_color", "blue", "red")
|
|
880
|
+
expect(ab_user["link_color:1"]).to eq(new_alternative_name)
|
|
806
881
|
end
|
|
807
882
|
|
|
808
883
|
it "should only count completion of users on the current version" do
|
|
809
|
-
alternative_name = ab_test(
|
|
810
|
-
expect(ab_user[
|
|
811
|
-
|
|
884
|
+
alternative_name = ab_test("link_color", "blue", "red")
|
|
885
|
+
expect(ab_user["link_color"]).to eq(alternative_name)
|
|
886
|
+
Split::Alternative.new(alternative_name, "link_color")
|
|
812
887
|
|
|
813
888
|
experiment.reset
|
|
814
889
|
expect(experiment.version).to eq(1)
|
|
815
890
|
|
|
816
|
-
ab_finished(
|
|
817
|
-
alternative = Split::Alternative.new(alternative_name,
|
|
891
|
+
ab_finished("link_color")
|
|
892
|
+
alternative = Split::Alternative.new(alternative_name, "link_color")
|
|
818
893
|
expect(alternative.completed_count).to eq(0)
|
|
819
894
|
end
|
|
820
895
|
end
|
|
821
896
|
|
|
822
|
-
context
|
|
897
|
+
context "when redis is not available" do
|
|
823
898
|
before(:each) do
|
|
824
899
|
expect(Split).to receive(:redis).at_most(5).times.and_raise(Errno::ECONNREFUSED.new)
|
|
825
900
|
end
|
|
826
901
|
|
|
827
|
-
context
|
|
902
|
+
context "and db_failover config option is turned off" do
|
|
828
903
|
before(:each) do
|
|
829
904
|
Split.configure do |config|
|
|
830
905
|
config.db_failover = false
|
|
831
906
|
end
|
|
832
907
|
end
|
|
833
908
|
|
|
834
|
-
describe
|
|
835
|
-
it
|
|
836
|
-
expect
|
|
909
|
+
describe "ab_test" do
|
|
910
|
+
it "should raise an exception" do
|
|
911
|
+
expect { ab_test("link_color", "blue", "red") }.to raise_error(Errno::ECONNREFUSED)
|
|
837
912
|
end
|
|
838
913
|
end
|
|
839
914
|
|
|
840
|
-
describe
|
|
841
|
-
it
|
|
842
|
-
expect
|
|
915
|
+
describe "finished" do
|
|
916
|
+
it "should raise an exception" do
|
|
917
|
+
expect { ab_finished("link_color") }.to raise_error(Errno::ECONNREFUSED)
|
|
843
918
|
end
|
|
844
919
|
end
|
|
845
920
|
|
|
@@ -851,29 +926,29 @@ describe Split::Helper do
|
|
|
851
926
|
end
|
|
852
927
|
|
|
853
928
|
it "should not attempt to connect to redis" do
|
|
854
|
-
expect
|
|
929
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
855
930
|
end
|
|
856
931
|
|
|
857
932
|
it "should return control variable" do
|
|
858
|
-
expect(ab_test(
|
|
859
|
-
expect
|
|
933
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
934
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
860
935
|
end
|
|
861
936
|
end
|
|
862
937
|
end
|
|
863
938
|
|
|
864
|
-
context
|
|
939
|
+
context "and db_failover config option is turned on" do
|
|
865
940
|
before(:each) do
|
|
866
941
|
Split.configure do |config|
|
|
867
942
|
config.db_failover = true
|
|
868
943
|
end
|
|
869
944
|
end
|
|
870
945
|
|
|
871
|
-
describe
|
|
872
|
-
it
|
|
873
|
-
expect
|
|
946
|
+
describe "ab_test" do
|
|
947
|
+
it "should not raise an exception" do
|
|
948
|
+
expect { ab_test("link_color", "blue", "red") }.not_to raise_error
|
|
874
949
|
end
|
|
875
950
|
|
|
876
|
-
it
|
|
951
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
877
952
|
Split.configure do |config|
|
|
878
953
|
config.db_failover_on_db_error = proc do |error|
|
|
879
954
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -881,43 +956,43 @@ describe Split::Helper do
|
|
|
881
956
|
end
|
|
882
957
|
|
|
883
958
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
884
|
-
ab_test(
|
|
959
|
+
ab_test("link_color", "blue", "red")
|
|
885
960
|
end
|
|
886
961
|
|
|
887
|
-
it
|
|
888
|
-
expect(ab_test(
|
|
889
|
-
expect(ab_test(
|
|
890
|
-
expect(ab_test(
|
|
891
|
-
expect(ab_test(
|
|
962
|
+
it "should always use first alternative" do
|
|
963
|
+
expect(ab_test("link_color", "blue", "red")).to eq("blue")
|
|
964
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("blue")
|
|
965
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("blue")
|
|
966
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
892
967
|
"shared/#{alternative}"
|
|
893
|
-
end).to eq(
|
|
968
|
+
end).to eq("shared/blue")
|
|
894
969
|
end
|
|
895
970
|
|
|
896
|
-
context
|
|
971
|
+
context "and db_failover_allow_parameter_override config option is turned on" do
|
|
897
972
|
before(:each) do
|
|
898
973
|
Split.configure do |config|
|
|
899
974
|
config.db_failover_allow_parameter_override = true
|
|
900
975
|
end
|
|
901
976
|
end
|
|
902
977
|
|
|
903
|
-
context
|
|
904
|
-
it
|
|
905
|
-
@params = {
|
|
906
|
-
expect(ab_test(
|
|
907
|
-
expect(ab_test(
|
|
908
|
-
expect(ab_test(
|
|
909
|
-
expect(ab_test(
|
|
910
|
-
expect(ab_test(
|
|
978
|
+
context "and given an override parameter" do
|
|
979
|
+
it "should use given override instead of the first alternative" do
|
|
980
|
+
@params = { "ab_test" => { "link_color" => "red" } }
|
|
981
|
+
expect(ab_test("link_color", "blue", "red")).to eq("red")
|
|
982
|
+
expect(ab_test("link_color", "blue", "red", "green")).to eq("red")
|
|
983
|
+
expect(ab_test("link_color", { "blue" => 0.01 }, "red" => 0.2)).to eq("red")
|
|
984
|
+
expect(ab_test("link_color", { "blue" => 0.8 }, { "red" => 20 })).to eq("red")
|
|
985
|
+
expect(ab_test("link_color", "blue", "red") do |alternative|
|
|
911
986
|
"shared/#{alternative}"
|
|
912
|
-
end).to eq(
|
|
987
|
+
end).to eq("shared/red")
|
|
913
988
|
end
|
|
914
989
|
end
|
|
915
990
|
end
|
|
916
991
|
|
|
917
|
-
context
|
|
992
|
+
context "and preloaded config given" do
|
|
918
993
|
before do
|
|
919
994
|
Split.configuration.experiments[:link_color] = {
|
|
920
|
-
:
|
|
995
|
+
alternatives: [ "blue", "red" ],
|
|
921
996
|
}
|
|
922
997
|
end
|
|
923
998
|
|
|
@@ -927,12 +1002,12 @@ describe Split::Helper do
|
|
|
927
1002
|
end
|
|
928
1003
|
end
|
|
929
1004
|
|
|
930
|
-
describe
|
|
931
|
-
it
|
|
932
|
-
expect
|
|
1005
|
+
describe "finished" do
|
|
1006
|
+
it "should not raise an exception" do
|
|
1007
|
+
expect { ab_finished("link_color") }.not_to raise_error
|
|
933
1008
|
end
|
|
934
1009
|
|
|
935
|
-
it
|
|
1010
|
+
it "should call db_failover_on_db_error proc with error as parameter" do
|
|
936
1011
|
Split.configure do |config|
|
|
937
1012
|
config.db_failover_on_db_error = proc do |error|
|
|
938
1013
|
expect(error).to be_a(Errno::ECONNREFUSED)
|
|
@@ -940,19 +1015,19 @@ describe Split::Helper do
|
|
|
940
1015
|
end
|
|
941
1016
|
|
|
942
1017
|
expect(Split.configuration.db_failover_on_db_error).to receive(:call).and_call_original
|
|
943
|
-
ab_finished(
|
|
1018
|
+
ab_finished("link_color")
|
|
944
1019
|
end
|
|
945
1020
|
end
|
|
946
1021
|
end
|
|
947
1022
|
end
|
|
948
1023
|
|
|
949
1024
|
context "with preloaded config" do
|
|
950
|
-
before { Split.configuration.experiments = {}}
|
|
1025
|
+
before { Split.configuration.experiments = {} }
|
|
951
1026
|
|
|
952
1027
|
it "pulls options from config file" do
|
|
953
1028
|
Split.configuration.experiments[:my_experiment] = {
|
|
954
|
-
:
|
|
955
|
-
:
|
|
1029
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1030
|
+
goals: ["goal1", "goal2"]
|
|
956
1031
|
}
|
|
957
1032
|
ab_test :my_experiment
|
|
958
1033
|
expect(Split::Experiment.new(:my_experiment).alternatives.map(&:name)).to eq([ "control_opt", "other_opt" ])
|
|
@@ -961,8 +1036,8 @@ describe Split::Helper do
|
|
|
961
1036
|
|
|
962
1037
|
it "can be called multiple times" do
|
|
963
1038
|
Split.configuration.experiments[:my_experiment] = {
|
|
964
|
-
:
|
|
965
|
-
:
|
|
1039
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1040
|
+
goals: ["goal1", "goal2"]
|
|
966
1041
|
}
|
|
967
1042
|
5.times { ab_test :my_experiment }
|
|
968
1043
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -973,8 +1048,8 @@ describe Split::Helper do
|
|
|
973
1048
|
|
|
974
1049
|
it "accepts multiple goals" do
|
|
975
1050
|
Split.configuration.experiments[:my_experiment] = {
|
|
976
|
-
:
|
|
977
|
-
:
|
|
1051
|
+
alternatives: [ "control_opt", "other_opt" ],
|
|
1052
|
+
goals: [ "goal1", "goal2", "goal3" ]
|
|
978
1053
|
}
|
|
979
1054
|
ab_test :my_experiment
|
|
980
1055
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -983,7 +1058,7 @@ describe Split::Helper do
|
|
|
983
1058
|
|
|
984
1059
|
it "allow specifying goals to be optional" do
|
|
985
1060
|
Split.configuration.experiments[:my_experiment] = {
|
|
986
|
-
:
|
|
1061
|
+
alternatives: [ "control_opt", "other_opt" ]
|
|
987
1062
|
}
|
|
988
1063
|
experiment = Split::Experiment.new(:my_experiment)
|
|
989
1064
|
expect(experiment.goals).to eq([])
|
|
@@ -991,7 +1066,7 @@ describe Split::Helper do
|
|
|
991
1066
|
|
|
992
1067
|
it "accepts multiple alternatives" do
|
|
993
1068
|
Split.configuration.experiments[:my_experiment] = {
|
|
994
|
-
:
|
|
1069
|
+
alternatives: [ "control_opt", "second_opt", "third_opt" ],
|
|
995
1070
|
}
|
|
996
1071
|
ab_test :my_experiment
|
|
997
1072
|
experiment = Split::Experiment.new(:my_experiment)
|
|
@@ -1000,68 +1075,68 @@ describe Split::Helper do
|
|
|
1000
1075
|
|
|
1001
1076
|
it "accepts probability on alternatives" do
|
|
1002
1077
|
Split.configuration.experiments[:my_experiment] = {
|
|
1003
|
-
:
|
|
1004
|
-
{ :
|
|
1005
|
-
{ :
|
|
1006
|
-
{ :
|
|
1078
|
+
alternatives: [
|
|
1079
|
+
{ name: "control_opt", percent: 67 },
|
|
1080
|
+
{ name: "second_opt", percent: 10 },
|
|
1081
|
+
{ name: "third_opt", percent: 23 },
|
|
1007
1082
|
],
|
|
1008
1083
|
}
|
|
1009
1084
|
ab_test :my_experiment
|
|
1010
1085
|
experiment = Split::Experiment.new(:my_experiment)
|
|
1011
|
-
expect(experiment.alternatives.collect{|a| [a.name, a.weight]}).to eq([[
|
|
1086
|
+
expect(experiment.alternatives.collect { |a| [a.name, a.weight] }).to eq([["control_opt", 0.67], ["second_opt", 0.1], ["third_opt", 0.23]])
|
|
1012
1087
|
end
|
|
1013
1088
|
|
|
1014
1089
|
it "accepts probability on some alternatives" do
|
|
1015
1090
|
Split.configuration.experiments[:my_experiment] = {
|
|
1016
|
-
:
|
|
1017
|
-
{ :
|
|
1091
|
+
alternatives: [
|
|
1092
|
+
{ name: "control_opt", percent: 34 },
|
|
1018
1093
|
"second_opt",
|
|
1019
|
-
{ :
|
|
1094
|
+
{ name: "third_opt", percent: 23 },
|
|
1020
1095
|
"fourth_opt",
|
|
1021
1096
|
],
|
|
1022
1097
|
}
|
|
1023
1098
|
ab_test :my_experiment
|
|
1024
1099
|
experiment = Split::Experiment.new(:my_experiment)
|
|
1025
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
1026
|
-
expect(names_and_weights).to eq([[
|
|
1027
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1100
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1101
|
+
expect(names_and_weights).to eq([["control_opt", 0.34], ["second_opt", 0.215], ["third_opt", 0.23], ["fourth_opt", 0.215]])
|
|
1102
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
1028
1103
|
end
|
|
1029
1104
|
|
|
1030
1105
|
it "allows name param without probability" do
|
|
1031
1106
|
Split.configuration.experiments[:my_experiment] = {
|
|
1032
|
-
:
|
|
1033
|
-
{ :
|
|
1107
|
+
alternatives: [
|
|
1108
|
+
{ name: "control_opt" },
|
|
1034
1109
|
"second_opt",
|
|
1035
|
-
{ :
|
|
1110
|
+
{ name: "third_opt", percent: 64 },
|
|
1036
1111
|
],
|
|
1037
1112
|
}
|
|
1038
1113
|
ab_test :my_experiment
|
|
1039
1114
|
experiment = Split::Experiment.new(:my_experiment)
|
|
1040
|
-
names_and_weights = experiment.alternatives.collect{|a| [a.name, a.weight]}
|
|
1041
|
-
expect(names_and_weights).to eq([[
|
|
1042
|
-
expect(names_and_weights.inject(0){|sum, nw| sum + nw[1]}).to eq(1.0)
|
|
1115
|
+
names_and_weights = experiment.alternatives.collect { |a| [a.name, a.weight] }
|
|
1116
|
+
expect(names_and_weights).to eq([["control_opt", 0.18], ["second_opt", 0.18], ["third_opt", 0.64]])
|
|
1117
|
+
expect(names_and_weights.inject(0) { |sum, nw| sum + nw[1] }).to eq(1.0)
|
|
1043
1118
|
end
|
|
1044
1119
|
|
|
1045
1120
|
it "fails gracefully if config is missing experiment" do
|
|
1046
|
-
Split.configuration.experiments = { :
|
|
1047
|
-
expect
|
|
1121
|
+
Split.configuration.experiments = { other_experiment: { foo: "Bar" } }
|
|
1122
|
+
expect { ab_test :my_experiment }.to raise_error(Split::ExperimentNotFound)
|
|
1048
1123
|
end
|
|
1049
1124
|
|
|
1050
1125
|
it "fails gracefully if config is missing" do
|
|
1051
|
-
expect
|
|
1126
|
+
expect { Split.configuration.experiments = nil }.to raise_error(Split::InvalidExperimentsFormatError)
|
|
1052
1127
|
end
|
|
1053
1128
|
|
|
1054
1129
|
it "fails gracefully if config is missing alternatives" do
|
|
1055
|
-
Split.configuration.experiments[:my_experiment] = { :
|
|
1056
|
-
expect
|
|
1130
|
+
Split.configuration.experiments[:my_experiment] = { foo: "Bar" }
|
|
1131
|
+
expect { ab_test :my_experiment }.to raise_error(NoMethodError)
|
|
1057
1132
|
end
|
|
1058
1133
|
end
|
|
1059
1134
|
|
|
1060
|
-
it
|
|
1061
|
-
experiment2 = Split::ExperimentCatalog.find_or_create(
|
|
1062
|
-
ab_test(
|
|
1063
|
-
ab_test(
|
|
1064
|
-
ab_finished(
|
|
1135
|
+
it "should handle multiple experiments correctly" do
|
|
1136
|
+
experiment2 = Split::ExperimentCatalog.find_or_create("link_color2", "blue", "red")
|
|
1137
|
+
ab_test("link_color", "blue", "red")
|
|
1138
|
+
ab_test("link_color2", "blue", "red")
|
|
1139
|
+
ab_finished("link_color2")
|
|
1065
1140
|
|
|
1066
1141
|
experiment2.alternatives.each do |alt|
|
|
1067
1142
|
expect(alt.unfinished_count).to eq(0)
|
|
@@ -1070,8 +1145,8 @@ describe Split::Helper do
|
|
|
1070
1145
|
|
|
1071
1146
|
context "with goals" do
|
|
1072
1147
|
before do
|
|
1073
|
-
@experiment = {
|
|
1074
|
-
@alternatives = [
|
|
1148
|
+
@experiment = { "link_color" => ["purchase", "refund"] }
|
|
1149
|
+
@alternatives = ["blue", "red"]
|
|
1075
1150
|
@experiment_name, @goals = normalize_metric(@experiment)
|
|
1076
1151
|
@goal1 = @goals[0]
|
|
1077
1152
|
@goal2 = @goals[1]
|
|
@@ -1085,8 +1160,8 @@ describe Split::Helper do
|
|
|
1085
1160
|
describe "ab_test" do
|
|
1086
1161
|
it "should allow experiment goals interface as a single hash" do
|
|
1087
1162
|
ab_test(@experiment, *@alternatives)
|
|
1088
|
-
experiment = Split::ExperimentCatalog.find(
|
|
1089
|
-
expect(experiment.goals).to eq([
|
|
1163
|
+
experiment = Split::ExperimentCatalog.find("link_color")
|
|
1164
|
+
expect(experiment.goals).to eq(["purchase", "refund"])
|
|
1090
1165
|
end
|
|
1091
1166
|
end
|
|
1092
1167
|
|
|
@@ -1096,15 +1171,9 @@ describe Split::Helper do
|
|
|
1096
1171
|
end
|
|
1097
1172
|
|
|
1098
1173
|
it "should increment the counter for the specified-goal completed alternative" do
|
|
1099
|
-
expect(
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
}).not_to change {
|
|
1103
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2)
|
|
1104
|
-
}
|
|
1105
|
-
}).to change {
|
|
1106
|
-
Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1)
|
|
1107
|
-
}.by(1)
|
|
1174
|
+
expect { ab_finished({ "link_color" => ["purchase"] }) }
|
|
1175
|
+
.to change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal2) }.by(0)
|
|
1176
|
+
.and change { Split::Alternative.new(@alternative_name, @experiment_name).completed_count(@goal1) }.by(1)
|
|
1108
1177
|
end
|
|
1109
1178
|
end
|
|
1110
1179
|
end
|