vanity 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +34 -19
- data/README.rdoc +19 -15
- data/lib/vanity/commands/report.rb +11 -4
- data/lib/vanity/experiment/ab_test.rb +149 -108
- data/lib/vanity/experiment/base.rb +42 -53
- data/lib/vanity/playground.rb +48 -24
- data/lib/vanity/rails.rb +2 -1
- data/lib/vanity/rails/dashboard.rb +15 -0
- data/lib/vanity/rails/helpers.rb +19 -32
- data/lib/vanity/rails/testing.rb +3 -1
- data/lib/vanity/templates/_ab_test.erb +7 -5
- data/lib/vanity/templates/_experiment.erb +5 -0
- data/lib/vanity/templates/_experiments.erb +2 -7
- data/lib/vanity/templates/_report.erb +14 -14
- data/lib/vanity/templates/vanity.css +13 -0
- data/test/ab_test_test.rb +147 -110
- data/test/experiment_test.rb +15 -22
- data/test/experiments/age_and_zipcode.rb +17 -2
- data/test/experiments/null_abc.rb +1 -1
- data/test/playground_test.rb +53 -31
- data/test/rails_test.rb +1 -1
- data/test/test_helper.rb +2 -0
- data/vanity.gemspec +2 -2
- metadata +7 -6
- data/lib/vanity/rails/console.rb +0 -14
- data/lib/vanity/templates/_vanity.css +0 -13
@@ -1,12 +1,7 @@
|
|
1
1
|
<ul class="experiments">
|
2
2
|
<% experiments.sort_by(&:created_at).each do |experiment| %>
|
3
|
-
<li class="experiment" id="experiment_<%=
|
4
|
-
|
5
|
-
<p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
|
6
|
-
<%= render Vanity.template("ab_test"), experiment: experiment %>
|
7
|
-
<p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
|
8
|
-
<%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
|
9
|
-
</p>
|
3
|
+
<li class="experiment <%= experiment.type %>" id="experiment_<%=h experiment.id.to_s %>">
|
4
|
+
<%= render Vanity.template("experiment"), experiment: experiment %>
|
10
5
|
</li>
|
11
6
|
<% end %>
|
12
7
|
</ul>
|
@@ -1,16 +1,16 @@
|
|
1
1
|
<html>
|
2
|
-
<head>
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
</head>
|
10
|
-
<body>
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
</body>
|
2
|
+
<head>
|
3
|
+
<title>Experiments (<%= Time.now %>)</title>
|
4
|
+
<style>
|
5
|
+
.vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
|
6
|
+
.vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
|
7
|
+
<%= File.read(Vanity.template("vanity.css")) %>
|
8
|
+
</style>
|
9
|
+
</head>
|
10
|
+
<body>
|
11
|
+
<div class="vanity">
|
12
|
+
<h1>Experiments</h1>
|
13
|
+
<%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
|
14
|
+
</div>
|
15
|
+
</body>
|
16
16
|
</html>
|
@@ -0,0 +1,13 @@
|
|
1
|
+
.vanity .experiments { list-style: none; margin: 0; padding: 0 }
|
2
|
+
.vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
|
3
|
+
.vanity .experiment .description { padding: .5em; margin: 0 }
|
4
|
+
.vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
|
5
|
+
.vanity .experiment .meta { color: #444; font-style: italic }
|
6
|
+
|
7
|
+
.vanity .ab_test table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
|
8
|
+
.vanity .ab_test td { padding: .5em; border-top: 1px solid #ccc }
|
9
|
+
.vanity .ab_test .choice td { font-weight: bold; background: #f0f0f8 }
|
10
|
+
.vanity .ab_test caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
|
11
|
+
.vanity .ab_test td.option { width: 5em; white-space: nowrap; overflow: hidden }
|
12
|
+
.vanity .ab_test td.value { width: 8em; white-space: nowrap; overflow: hidden }
|
13
|
+
.vanity .ab_test td.action { width: 6em; overflow: hidden; text-align: center }
|
data/test/ab_test_test.rb
CHANGED
@@ -16,8 +16,8 @@ class AbTestController < ActionController::Base
|
|
16
16
|
render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
20
|
-
|
19
|
+
def track
|
20
|
+
track! :simple
|
21
21
|
render text: ""
|
22
22
|
end
|
23
23
|
end
|
@@ -28,29 +28,24 @@ class AbTestTest < ActionController::TestCase
|
|
28
28
|
|
29
29
|
# -- Experiment definition --
|
30
30
|
|
31
|
-
def test_uses_ab_test_when_type_is_ab_test
|
32
|
-
experiment(:ab, type: :ab_test) { }
|
33
|
-
assert_instance_of Vanity::Experiment::AbTest, experiment(:ab)
|
34
|
-
end
|
35
|
-
|
36
31
|
def test_requires_at_least_two_alternatives_per_experiment
|
37
32
|
assert_raises RuntimeError do
|
38
|
-
|
33
|
+
ab_test :none do
|
39
34
|
alternatives []
|
40
35
|
end
|
41
36
|
end
|
42
37
|
assert_raises RuntimeError do
|
43
|
-
|
38
|
+
ab_test :one do
|
44
39
|
alternatives "foo"
|
45
40
|
end
|
46
41
|
end
|
47
|
-
|
42
|
+
ab_test :two do
|
48
43
|
alternatives "foo", "bar"
|
49
44
|
end
|
50
45
|
end
|
51
46
|
|
52
47
|
def test_returning_alternative_by_value
|
53
|
-
|
48
|
+
ab_test :abcd do
|
54
49
|
alternatives :a, :b, :c, :d
|
55
50
|
end
|
56
51
|
assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
|
@@ -58,7 +53,7 @@ class AbTestTest < ActionController::TestCase
|
|
58
53
|
end
|
59
54
|
|
60
55
|
def test_alternative_name
|
61
|
-
|
56
|
+
ab_test :abcd do
|
62
57
|
alternatives :a, :b
|
63
58
|
end
|
64
59
|
assert_equal "option A", experiment(:abcd).alternative(:a).name
|
@@ -69,7 +64,7 @@ class AbTestTest < ActionController::TestCase
|
|
69
64
|
# -- Running experiment --
|
70
65
|
|
71
66
|
def test_returns_the_same_alternative_consistently
|
72
|
-
|
67
|
+
ab_test :foobar do
|
73
68
|
alternatives "foo", "bar"
|
74
69
|
identify { "6e98ec" }
|
75
70
|
end
|
@@ -81,7 +76,7 @@ class AbTestTest < ActionController::TestCase
|
|
81
76
|
end
|
82
77
|
|
83
78
|
def test_returns_different_alternatives_for_each_participant
|
84
|
-
|
79
|
+
ab_test :foobar do
|
85
80
|
alternatives "foo", "bar"
|
86
81
|
identify { rand }
|
87
82
|
end
|
@@ -92,7 +87,7 @@ class AbTestTest < ActionController::TestCase
|
|
92
87
|
|
93
88
|
def test_records_all_participants_in_each_alternative
|
94
89
|
ids = (Array.new(200) { |i| i } * 5).shuffle
|
95
|
-
|
90
|
+
ab_test :foobar do
|
96
91
|
alternatives "foo", "bar"
|
97
92
|
identify { ids.pop }
|
98
93
|
end
|
@@ -104,13 +99,13 @@ class AbTestTest < ActionController::TestCase
|
|
104
99
|
|
105
100
|
def test_records_each_converted_participant_only_once
|
106
101
|
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
107
|
-
|
102
|
+
ab_test :foobar do
|
108
103
|
alternatives "foo", "bar"
|
109
104
|
identify { ids.pop }
|
110
105
|
end
|
111
106
|
500.times do
|
112
107
|
experiment(:foobar).choose
|
113
|
-
experiment(:foobar).
|
108
|
+
experiment(:foobar).track!
|
114
109
|
end
|
115
110
|
alts = experiment(:foobar).alternatives
|
116
111
|
assert_equal 100, alts.map(&:converted).sum
|
@@ -118,31 +113,31 @@ class AbTestTest < ActionController::TestCase
|
|
118
113
|
|
119
114
|
def test_records_conversion_only_for_participants
|
120
115
|
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
121
|
-
|
116
|
+
ab_test :foobar do
|
122
117
|
alternatives "foo", "bar"
|
123
118
|
identify { ids.pop }
|
124
119
|
end
|
125
120
|
500.times do
|
126
121
|
experiment(:foobar).choose
|
127
|
-
experiment(:foobar).
|
128
|
-
experiment(:foobar).
|
122
|
+
experiment(:foobar).track!
|
123
|
+
experiment(:foobar).track!
|
129
124
|
end
|
130
125
|
alts = experiment(:foobar).alternatives
|
131
126
|
assert_equal 100, alts.map(&:converted).sum
|
132
127
|
end
|
133
128
|
|
134
|
-
def
|
135
|
-
|
129
|
+
def test_destroy_experiment
|
130
|
+
ab_test :simple do
|
136
131
|
identify { "me" }
|
137
132
|
complete_if { alternatives.map(&:converted).sum >= 1 }
|
138
133
|
outcome_is { alternative(true) }
|
139
134
|
end
|
140
135
|
experiment(:simple).choose
|
141
|
-
experiment(:simple).
|
136
|
+
experiment(:simple).track!
|
142
137
|
refute experiment(:simple).active?
|
143
138
|
assert_equal true, experiment(:simple).outcome.value
|
144
139
|
|
145
|
-
experiment(:simple).
|
140
|
+
experiment(:simple).destroy
|
146
141
|
assert experiment(:simple).active?
|
147
142
|
assert_nil experiment(:simple).outcome
|
148
143
|
assert_nil experiment(:simple).completed_at
|
@@ -155,13 +150,13 @@ class AbTestTest < ActionController::TestCase
|
|
155
150
|
# -- A/B helper methods --
|
156
151
|
|
157
152
|
def test_fail_if_no_experiment
|
158
|
-
assert_raise
|
153
|
+
assert_raise LoadError do
|
159
154
|
get :test_render
|
160
155
|
end
|
161
156
|
end
|
162
157
|
|
163
158
|
def test_ab_test_chooses_in_render
|
164
|
-
|
159
|
+
ab_test(:simple) { }
|
165
160
|
responses = Array.new(100) do
|
166
161
|
@controller = nil ; setup_controller_request_and_response
|
167
162
|
get :test_render
|
@@ -171,7 +166,7 @@ class AbTestTest < ActionController::TestCase
|
|
171
166
|
end
|
172
167
|
|
173
168
|
def test_ab_test_chooses_view_helper
|
174
|
-
|
169
|
+
ab_test(:simple) { }
|
175
170
|
responses = Array.new(100) do
|
176
171
|
@controller = nil ; setup_controller_request_and_response
|
177
172
|
get :test_view
|
@@ -181,7 +176,7 @@ class AbTestTest < ActionController::TestCase
|
|
181
176
|
end
|
182
177
|
|
183
178
|
def test_ab_test_with_capture
|
184
|
-
|
179
|
+
ab_test(:simple) { }
|
185
180
|
responses = Array.new(100) do
|
186
181
|
@controller = nil ; setup_controller_request_and_response
|
187
182
|
get :test_capture
|
@@ -190,11 +185,11 @@ class AbTestTest < ActionController::TestCase
|
|
190
185
|
assert_equal %w{false true}, responses.map(&:strip).uniq.sort
|
191
186
|
end
|
192
187
|
|
193
|
-
def
|
194
|
-
|
188
|
+
def test_ab_test_track
|
189
|
+
ab_test(:simple) { }
|
195
190
|
responses = Array.new(100) do
|
196
191
|
@controller.send(:cookies).clear
|
197
|
-
get :
|
192
|
+
get :track
|
198
193
|
@response.body
|
199
194
|
end
|
200
195
|
end
|
@@ -203,7 +198,7 @@ class AbTestTest < ActionController::TestCase
|
|
203
198
|
# -- Testing with tests --
|
204
199
|
|
205
200
|
def test_with_given_choice
|
206
|
-
|
201
|
+
ab_test(:simple) { alternatives :a, :b, :c }
|
207
202
|
100.times do |i|
|
208
203
|
@controller = nil ; setup_controller_request_and_response
|
209
204
|
experiment(:simple).chooses(:b)
|
@@ -213,35 +208,50 @@ class AbTestTest < ActionController::TestCase
|
|
213
208
|
end
|
214
209
|
|
215
210
|
def test_which_chooses_non_existent_alternative
|
216
|
-
|
211
|
+
ab_test(:simple) { }
|
217
212
|
assert_raises ArgumentError do
|
218
213
|
experiment(:simple).chooses(404)
|
219
214
|
end
|
220
215
|
end
|
221
216
|
|
217
|
+
def test_chooses_cleared_with_nil
|
218
|
+
ab_test :simple do
|
219
|
+
identify { rand }
|
220
|
+
alternatives :a, :b, :c
|
221
|
+
end
|
222
|
+
responses = Array.new(100) { |i|
|
223
|
+
@controller = nil ; setup_controller_request_and_response
|
224
|
+
experiment(:simple).chooses(:b)
|
225
|
+
experiment(:simple).chooses(nil)
|
226
|
+
get :test_render
|
227
|
+
@response.body
|
228
|
+
}
|
229
|
+
assert responses.uniq.size == 3
|
230
|
+
end
|
231
|
+
|
222
232
|
|
223
233
|
# -- Scoring --
|
224
234
|
|
225
235
|
def test_scoring
|
226
|
-
|
236
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
227
237
|
# participating, conversions, rate, z-score
|
228
238
|
# Control: 182 35 19.23% N/A
|
229
|
-
182.times { |i| experiment(:abcd).
|
230
|
-
35.times { |i| experiment(:abcd).
|
239
|
+
182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
|
240
|
+
35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
|
231
241
|
# Treatment A: 180 45 25.00% 1.33
|
232
|
-
180.times { |i| experiment(:abcd).
|
233
|
-
45.times { |i| experiment(:abcd).
|
242
|
+
180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
243
|
+
45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
234
244
|
# treatment B: 189 28 14.81% -1.13
|
235
|
-
189.times { |i| experiment(:abcd).
|
236
|
-
28.times { |i| experiment(:abcd).
|
245
|
+
189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
|
246
|
+
28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
|
237
247
|
# treatment C: 188 61 32.45% 2.94
|
238
|
-
188.times { |i| experiment(:abcd).
|
239
|
-
61.times { |i| experiment(:abcd).
|
248
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
249
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
240
250
|
|
241
251
|
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
242
252
|
assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
|
243
|
-
|
244
|
-
assert_equal [90, 0, 99, 90],
|
253
|
+
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
254
|
+
assert_equal [90, 0, 99, 90], probabilities
|
245
255
|
|
246
256
|
diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
|
247
257
|
assert_equal [30, 69, nil, 119], diff
|
@@ -253,9 +263,9 @@ class AbTestTest < ActionController::TestCase
|
|
253
263
|
end
|
254
264
|
|
255
265
|
def test_scoring_with_no_performers
|
256
|
-
|
266
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
257
267
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
258
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
268
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
259
269
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
260
270
|
assert_nil experiment(:abcd).score.best
|
261
271
|
assert_nil experiment(:abcd).score.choice
|
@@ -263,11 +273,11 @@ class AbTestTest < ActionController::TestCase
|
|
263
273
|
end
|
264
274
|
|
265
275
|
def test_scoring_with_one_performer
|
266
|
-
|
267
|
-
10.times { |i| experiment(:abcd).
|
268
|
-
8.times { |i| experiment(:abcd).
|
276
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
277
|
+
10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
278
|
+
8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
269
279
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
270
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
280
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
271
281
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
272
282
|
assert 1, experiment(:abcd).score.best.id
|
273
283
|
assert_nil experiment(:abcd).score.choice
|
@@ -276,16 +286,16 @@ class AbTestTest < ActionController::TestCase
|
|
276
286
|
end
|
277
287
|
|
278
288
|
def test_scoring_with_some_performers
|
279
|
-
|
280
|
-
10.times { |i| experiment(:abcd).
|
281
|
-
8.times { |i| experiment(:abcd).
|
282
|
-
12.times { |i| experiment(:abcd).
|
283
|
-
5.times { |i| experiment(:abcd).
|
289
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
290
|
+
10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
291
|
+
8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
292
|
+
12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
293
|
+
5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
284
294
|
|
285
295
|
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
286
296
|
assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
|
287
|
-
|
288
|
-
assert_equal [0, 95, 0, 0],
|
297
|
+
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
298
|
+
assert_equal [0, 95, 0, 0], probabilities
|
289
299
|
diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
|
290
300
|
assert_equal [nil, 92, nil, nil], diff
|
291
301
|
assert_equal 1, experiment(:abcd).score.best.id
|
@@ -294,26 +304,39 @@ class AbTestTest < ActionController::TestCase
|
|
294
304
|
assert_equal 3, experiment(:abcd).score.least.id
|
295
305
|
end
|
296
306
|
|
307
|
+
def test_scoring_with_different_probability
|
308
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
309
|
+
10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
310
|
+
8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
311
|
+
12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
312
|
+
5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
313
|
+
|
314
|
+
assert_equal 1, experiment(:abcd).score(90).choice.id
|
315
|
+
assert_equal 1, experiment(:abcd).score(95).choice.id
|
316
|
+
assert_nil experiment(:abcd).score(99).choice
|
317
|
+
end
|
318
|
+
|
297
319
|
|
298
320
|
# -- Conclusion --
|
299
321
|
|
300
322
|
def test_conclusion
|
301
|
-
|
323
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
302
324
|
# participating, conversions, rate, z-score
|
303
325
|
# Control: 182 35 19.23% N/A
|
304
|
-
182.times { |i| experiment(:abcd).
|
305
|
-
35.times { |i| experiment(:abcd).
|
326
|
+
182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
|
327
|
+
35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
|
306
328
|
# Treatment A: 180 45 25.00% 1.33
|
307
|
-
180.times { |i| experiment(:abcd).
|
308
|
-
45.times { |i| experiment(:abcd).
|
329
|
+
180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
330
|
+
45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
309
331
|
# treatment B: 189 28 14.81% -1.13
|
310
|
-
189.times { |i| experiment(:abcd).
|
311
|
-
28.times { |i| experiment(:abcd).
|
332
|
+
189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
|
333
|
+
28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
|
312
334
|
# treatment C: 188 61 32.45% 2.94
|
313
|
-
188.times { |i| experiment(:abcd).
|
314
|
-
61.times { |i| experiment(:abcd).
|
335
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
336
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
315
337
|
|
316
338
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
339
|
+
There are 739 participants in this experiment.
|
317
340
|
The best choice is option D: it converted at 32.4% (30% better than option B).
|
318
341
|
With 90% probability this result is statistically significant.
|
319
342
|
Option B converted at 25.0%.
|
@@ -324,15 +347,16 @@ Option D selected as the best alternative.
|
|
324
347
|
end
|
325
348
|
|
326
349
|
def test_conclusion_with_some_performers
|
327
|
-
|
350
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
328
351
|
# Treatment A: 180 45 25.00% 1.33
|
329
|
-
180.times { |i| experiment(:abcd).
|
330
|
-
45.times { |i| experiment(:abcd).
|
352
|
+
180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
353
|
+
45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
331
354
|
# treatment C: 188 61 32.45% 2.94
|
332
|
-
188.times { |i| experiment(:abcd).
|
333
|
-
61.times { |i| experiment(:abcd).
|
355
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
356
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
334
357
|
|
335
358
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
359
|
+
There are 368 participants in this experiment.
|
336
360
|
The best choice is option D: it converted at 32.4% (30% better than option B).
|
337
361
|
With 90% probability this result is statistically significant.
|
338
362
|
Option B converted at 25.0%.
|
@@ -343,15 +367,16 @@ Option D selected as the best alternative.
|
|
343
367
|
end
|
344
368
|
|
345
369
|
def test_conclusion_without_clear_winner
|
346
|
-
|
370
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
347
371
|
# Treatment A: 180 45 25.00% 1.33
|
348
|
-
180.times { |i| experiment(:abcd).
|
349
|
-
58.times { |i| experiment(:abcd).
|
372
|
+
180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
373
|
+
58.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
350
374
|
# treatment C: 188 61 32.45% 2.94
|
351
|
-
188.times { |i| experiment(:abcd).
|
352
|
-
61.times { |i| experiment(:abcd).
|
375
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
376
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
353
377
|
|
354
378
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
379
|
+
There are 368 participants in this experiment.
|
355
380
|
The best choice is option D: it converted at 32.4% (1% better than option B).
|
356
381
|
This result is not statistically significant, suggest you continue this experiment.
|
357
382
|
Option B converted at 32.2%.
|
@@ -361,15 +386,16 @@ Option C did not convert.
|
|
361
386
|
end
|
362
387
|
|
363
388
|
def test_conclusion_without_close_performers
|
364
|
-
|
389
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
365
390
|
# Treatment A: 180 45 25.00% 1.33
|
366
|
-
186.times { |i| experiment(:abcd).
|
367
|
-
60.times { |i| experiment(:abcd).
|
391
|
+
186.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
392
|
+
60.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
368
393
|
# treatment C: 188 61 32.45% 2.94
|
369
|
-
188.times { |i| experiment(:abcd).
|
370
|
-
61.times { |i| experiment(:abcd).
|
394
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
395
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
371
396
|
|
372
397
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
398
|
+
There are 374 participants in this experiment.
|
373
399
|
The best choice is option D: it converted at 32.4%.
|
374
400
|
This result is not statistically significant, suggest you continue this experiment.
|
375
401
|
Option B converted at 32.3%.
|
@@ -379,15 +405,16 @@ Option C did not convert.
|
|
379
405
|
end
|
380
406
|
|
381
407
|
def test_conclusion_without_equal_performers
|
382
|
-
|
408
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
383
409
|
# Treatment A: 180 45 25.00% 1.33
|
384
|
-
188.times { |i| experiment(:abcd).
|
385
|
-
61.times { |i| experiment(:abcd).
|
410
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
411
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
386
412
|
# treatment C: 188 61 32.45% 2.94
|
387
|
-
188.times { |i| experiment(:abcd).
|
388
|
-
61.times { |i| experiment(:abcd).
|
413
|
+
188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
|
414
|
+
61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
|
389
415
|
|
390
416
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
417
|
+
There are 376 participants in this experiment.
|
391
418
|
Option D converted at 32.4%.
|
392
419
|
Option B converted at 32.4%.
|
393
420
|
Option A did not convert.
|
@@ -396,24 +423,30 @@ Option C did not convert.
|
|
396
423
|
end
|
397
424
|
|
398
425
|
def test_conclusion_with_one_performers
|
399
|
-
|
426
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
400
427
|
# Treatment A: 180 45 25.00% 1.33
|
401
|
-
180.times { |i| experiment(:abcd).
|
402
|
-
45.times { |i| experiment(:abcd).
|
428
|
+
180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
|
429
|
+
45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
|
403
430
|
|
404
|
-
assert_equal
|
431
|
+
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
432
|
+
There are 180 participants in this experiment.
|
433
|
+
This experiment did not run long enough to find a clear winner.
|
434
|
+
TEXT
|
405
435
|
end
|
406
436
|
|
407
437
|
def test_conclusion_with_no_performers
|
408
|
-
|
409
|
-
assert_equal
|
438
|
+
ab_test(:abcd) { alternatives :a, :b, :c, :d }
|
439
|
+
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
440
|
+
There are no participants in this experiment yet.
|
441
|
+
This experiment did not run long enough to find a clear winner.
|
442
|
+
TEXT
|
410
443
|
end
|
411
444
|
|
412
445
|
|
413
446
|
# -- Completion --
|
414
447
|
|
415
448
|
def test_completion_if
|
416
|
-
|
449
|
+
ab_test :simple do
|
417
450
|
identify { rand }
|
418
451
|
complete_if { true }
|
419
452
|
end
|
@@ -422,7 +455,7 @@ Option C did not convert.
|
|
422
455
|
end
|
423
456
|
|
424
457
|
def test_completion_if_fails
|
425
|
-
|
458
|
+
ab_test :simple do
|
426
459
|
identify { rand }
|
427
460
|
complete_if { fail }
|
428
461
|
end
|
@@ -432,7 +465,7 @@ Option C did not convert.
|
|
432
465
|
|
433
466
|
def test_completion
|
434
467
|
ids = Array.new(100) { |i| i.to_s }.shuffle
|
435
|
-
|
468
|
+
ab_test :simple do
|
436
469
|
identify { ids.pop }
|
437
470
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
438
471
|
end
|
@@ -447,7 +480,7 @@ Option C did not convert.
|
|
447
480
|
|
448
481
|
def test_ab_methods_after_completion
|
449
482
|
ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
|
450
|
-
|
483
|
+
ab_test :simple do
|
451
484
|
identify { ids.pop }
|
452
485
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
453
486
|
outcome_is { alternatives[1] }
|
@@ -456,7 +489,7 @@ Option C did not convert.
|
|
456
489
|
results = Set.new
|
457
490
|
100.times do
|
458
491
|
results << experiment(:simple).choose
|
459
|
-
experiment(:simple).
|
492
|
+
experiment(:simple).track!
|
460
493
|
end
|
461
494
|
assert results.include?(true) && results.include?(false)
|
462
495
|
refute experiment(:simple).active?
|
@@ -464,7 +497,7 @@ Option C did not convert.
|
|
464
497
|
# Test that we always get the same choice (true)
|
465
498
|
100.times do
|
466
499
|
assert_equal true, experiment(:simple).choose
|
467
|
-
experiment(:simple).
|
500
|
+
experiment(:simple).track!
|
468
501
|
end
|
469
502
|
# We don't get to count the 100 participant's conversion, but that's ok.
|
470
503
|
assert_equal 99, experiment(:simple).alternatives.map(&:converted).sum
|
@@ -475,7 +508,7 @@ Option C did not convert.
|
|
475
508
|
# -- Outcome --
|
476
509
|
|
477
510
|
def test_completion_outcome
|
478
|
-
|
511
|
+
ab_test :quick do
|
479
512
|
outcome_is { alternatives[1] }
|
480
513
|
end
|
481
514
|
experiment(:quick).complete!
|
@@ -483,7 +516,7 @@ Option C did not convert.
|
|
483
516
|
end
|
484
517
|
|
485
518
|
def test_outcome_is_returns_nil
|
486
|
-
|
519
|
+
ab_test :quick do
|
487
520
|
outcome_is { nil }
|
488
521
|
end
|
489
522
|
experiment(:quick).complete!
|
@@ -491,7 +524,7 @@ Option C did not convert.
|
|
491
524
|
end
|
492
525
|
|
493
526
|
def test_outcome_is_returns_something_else
|
494
|
-
|
527
|
+
ab_test :quick do
|
495
528
|
outcome_is { "error" }
|
496
529
|
end
|
497
530
|
experiment(:quick).complete!
|
@@ -499,7 +532,7 @@ Option C did not convert.
|
|
499
532
|
end
|
500
533
|
|
501
534
|
def test_outcome_is_fails
|
502
|
-
|
535
|
+
ab_test :quick do
|
503
536
|
outcome_is { fail }
|
504
537
|
end
|
505
538
|
experiment(:quick).complete!
|
@@ -507,29 +540,33 @@ Option C did not convert.
|
|
507
540
|
end
|
508
541
|
|
509
542
|
def test_outcome_choosing_best_alternative
|
510
|
-
|
543
|
+
ab_test :quick do
|
511
544
|
end
|
512
|
-
2.times { |i| experiment(:quick).
|
513
|
-
10.times { |i| experiment(:quick).
|
545
|
+
2.times { |i| experiment(:quick).send(:count_participant, i, false) }
|
546
|
+
10.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
|
514
547
|
experiment(:quick).complete!
|
515
548
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
516
549
|
end
|
517
550
|
|
518
551
|
def test_outcome_only_performing_alternative
|
519
|
-
|
552
|
+
ab_test :quick do
|
520
553
|
end
|
521
|
-
2.times
|
554
|
+
2.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
|
522
555
|
experiment(:quick).complete!
|
523
556
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
524
557
|
end
|
525
558
|
|
526
559
|
def test_outcome_choosing_equal_alternatives
|
527
|
-
|
560
|
+
ab_test :quick do
|
528
561
|
end
|
529
|
-
8.times { |i| experiment(:quick).
|
530
|
-
8.times { |i| experiment(:quick).
|
562
|
+
8.times { |i| experiment(:quick).send(:count_participant, i, false).send(:count_conversion, i, false) }
|
563
|
+
8.times { |i| experiment(:quick).send(:count_participant, i, true). send(:count_conversion, i, true) }
|
531
564
|
experiment(:quick).complete!
|
532
565
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
533
566
|
end
|
534
567
|
|
568
|
+
|
569
|
+
def ab_test(name, &block)
|
570
|
+
Vanity.playground.define name, :ab_test, &block
|
571
|
+
end
|
535
572
|
end
|