vanity 0.3.1 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (26) hide show
  1. data/CHANGELOG +34 -19
  2. data/README.rdoc +19 -15
  3. data/lib/vanity/commands/report.rb +11 -4
  4. data/lib/vanity/experiment/ab_test.rb +149 -108
  5. data/lib/vanity/experiment/base.rb +42 -53
  6. data/lib/vanity/playground.rb +48 -24
  7. data/lib/vanity/rails.rb +2 -1
  8. data/lib/vanity/rails/dashboard.rb +15 -0
  9. data/lib/vanity/rails/helpers.rb +19 -32
  10. data/lib/vanity/rails/testing.rb +3 -1
  11. data/lib/vanity/templates/_ab_test.erb +7 -5
  12. data/lib/vanity/templates/_experiment.erb +5 -0
  13. data/lib/vanity/templates/_experiments.erb +2 -7
  14. data/lib/vanity/templates/_report.erb +14 -14
  15. data/lib/vanity/templates/vanity.css +13 -0
  16. data/test/ab_test_test.rb +147 -110
  17. data/test/experiment_test.rb +15 -22
  18. data/test/experiments/age_and_zipcode.rb +17 -2
  19. data/test/experiments/null_abc.rb +1 -1
  20. data/test/playground_test.rb +53 -31
  21. data/test/rails_test.rb +1 -1
  22. data/test/test_helper.rb +2 -0
  23. data/vanity.gemspec +2 -2
  24. metadata +7 -6
  25. data/lib/vanity/rails/console.rb +0 -14
  26. data/lib/vanity/templates/_vanity.css +0 -13
data/lib/vanity/templates/_experiments.erb CHANGED
@@ -1,12 +1,7 @@
1
1
  <ul class="experiments">
2
2
  <% experiments.sort_by(&:created_at).each do |experiment| %>
3
- <li class="experiment" id="experiment_<%= CGI.escape experiment.id.to_s %>">
4
- <h2 class="<%= experiment.type %>"><%= CGI.escape_html experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
5
- <p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
6
- <%= render Vanity.template("ab_test"), experiment: experiment %>
7
- <p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
8
- <%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
9
- </p>
3
+ <li class="experiment <%= experiment.type %>" id="experiment_<%=h experiment.id.to_s %>">
4
+ <%= render Vanity.template("experiment"), experiment: experiment %>
10
5
  </li>
11
6
  <% end %>
12
7
  </ul>
data/lib/vanity/templates/_report.erb CHANGED
@@ -1,16 +1,16 @@
1
1
  <html>
2
- <head>
3
- <title>Experiments (<%= Time.now %>)</title>
4
- <style>
5
- .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
6
- .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
7
- </style>
8
- <%= render Vanity.template("vanity.css") %>
9
- </head>
10
- <body>
11
- <div class="vanity">
12
- <h1>Experiments</h1>
13
- <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
14
- </div>
15
- </body>
2
+ <head>
3
+ <title>Experiments (<%= Time.now %>)</title>
4
+ <style>
5
+ .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
6
+ .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
7
+ <%= File.read(Vanity.template("vanity.css")) %>
8
+ </style>
9
+ </head>
10
+ <body>
11
+ <div class="vanity">
12
+ <h1>Experiments</h1>
13
+ <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
14
+ </div>
15
+ </body>
16
16
  </html>
data/lib/vanity/templates/vanity.css ADDED
@@ -0,0 +1,13 @@
1
+ .vanity .experiments { list-style: none; margin: 0; padding: 0 }
2
+ .vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
3
+ .vanity .experiment .description { padding: .5em; margin: 0 }
4
+ .vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
5
+ .vanity .experiment .meta { color: #444; font-style: italic }
6
+
7
+ .vanity .ab_test table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
8
+ .vanity .ab_test td { padding: .5em; border-top: 1px solid #ccc }
9
+ .vanity .ab_test .choice td { font-weight: bold; background: #f0f0f8 }
10
+ .vanity .ab_test caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
11
+ .vanity .ab_test td.option { width: 5em; white-space: nowrap; overflow: hidden }
12
+ .vanity .ab_test td.value { width: 8em; white-space: nowrap; overflow: hidden }
13
+ .vanity .ab_test td.action { width: 6em; overflow: hidden; text-align: center }
data/test/ab_test_test.rb CHANGED
@@ -16,8 +16,8 @@ class AbTestController < ActionController::Base
16
16
  render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
17
17
  end
18
18
 
19
- def goal
20
- ab_goal! :simple
19
+ def track
20
+ track! :simple
21
21
  render text: ""
22
22
  end
23
23
  end
@@ -28,29 +28,24 @@ class AbTestTest < ActionController::TestCase
28
28
 
29
29
  # -- Experiment definition --
30
30
 
31
- def test_uses_ab_test_when_type_is_ab_test
32
- experiment(:ab, type: :ab_test) { }
33
- assert_instance_of Vanity::Experiment::AbTest, experiment(:ab)
34
- end
35
-
36
31
  def test_requires_at_least_two_alternatives_per_experiment
37
32
  assert_raises RuntimeError do
38
- experiment :none, type: :ab_test do
33
+ ab_test :none do
39
34
  alternatives []
40
35
  end
41
36
  end
42
37
  assert_raises RuntimeError do
43
- experiment :one, type: :ab_test do
38
+ ab_test :one do
44
39
  alternatives "foo"
45
40
  end
46
41
  end
47
- experiment :two, type: :ab_test do
42
+ ab_test :two do
48
43
  alternatives "foo", "bar"
49
44
  end
50
45
  end
51
46
 
52
47
  def test_returning_alternative_by_value
53
- experiment :abcd do
48
+ ab_test :abcd do
54
49
  alternatives :a, :b, :c, :d
55
50
  end
56
51
  assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
@@ -58,7 +53,7 @@ class AbTestTest < ActionController::TestCase
58
53
  end
59
54
 
60
55
  def test_alternative_name
61
- experiment :abcd do
56
+ ab_test :abcd do
62
57
  alternatives :a, :b
63
58
  end
64
59
  assert_equal "option A", experiment(:abcd).alternative(:a).name
@@ -69,7 +64,7 @@ class AbTestTest < ActionController::TestCase
69
64
  # -- Running experiment --
70
65
 
71
66
  def test_returns_the_same_alternative_consistently
72
- experiment :foobar do
67
+ ab_test :foobar do
73
68
  alternatives "foo", "bar"
74
69
  identify { "6e98ec" }
75
70
  end
@@ -81,7 +76,7 @@ class AbTestTest < ActionController::TestCase
81
76
  end
82
77
 
83
78
  def test_returns_different_alternatives_for_each_participant
84
- experiment :foobar do
79
+ ab_test :foobar do
85
80
  alternatives "foo", "bar"
86
81
  identify { rand }
87
82
  end
@@ -92,7 +87,7 @@ class AbTestTest < ActionController::TestCase
92
87
 
93
88
  def test_records_all_participants_in_each_alternative
94
89
  ids = (Array.new(200) { |i| i } * 5).shuffle
95
- experiment :foobar do
90
+ ab_test :foobar do
96
91
  alternatives "foo", "bar"
97
92
  identify { ids.pop }
98
93
  end
@@ -104,13 +99,13 @@ class AbTestTest < ActionController::TestCase
104
99
 
105
100
  def test_records_each_converted_participant_only_once
106
101
  ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
107
- experiment :foobar do
102
+ ab_test :foobar do
108
103
  alternatives "foo", "bar"
109
104
  identify { ids.pop }
110
105
  end
111
106
  500.times do
112
107
  experiment(:foobar).choose
113
- experiment(:foobar).conversion!
108
+ experiment(:foobar).track!
114
109
  end
115
110
  alts = experiment(:foobar).alternatives
116
111
  assert_equal 100, alts.map(&:converted).sum
@@ -118,31 +113,31 @@ class AbTestTest < ActionController::TestCase
118
113
 
119
114
  def test_records_conversion_only_for_participants
120
115
  ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
121
- experiment :foobar do
116
+ ab_test :foobar do
122
117
  alternatives "foo", "bar"
123
118
  identify { ids.pop }
124
119
  end
125
120
  500.times do
126
121
  experiment(:foobar).choose
127
- experiment(:foobar).conversion!
128
- experiment(:foobar).conversion!
122
+ experiment(:foobar).track!
123
+ experiment(:foobar).track!
129
124
  end
130
125
  alts = experiment(:foobar).alternatives
131
126
  assert_equal 100, alts.map(&:converted).sum
132
127
  end
133
128
 
134
- def test_reset_experiment
135
- experiment :simple do
129
+ def test_destroy_experiment
130
+ ab_test :simple do
136
131
  identify { "me" }
137
132
  complete_if { alternatives.map(&:converted).sum >= 1 }
138
133
  outcome_is { alternative(true) }
139
134
  end
140
135
  experiment(:simple).choose
141
- experiment(:simple).conversion!
136
+ experiment(:simple).track!
142
137
  refute experiment(:simple).active?
143
138
  assert_equal true, experiment(:simple).outcome.value
144
139
 
145
- experiment(:simple).reset!
140
+ experiment(:simple).destroy
146
141
  assert experiment(:simple).active?
147
142
  assert_nil experiment(:simple).outcome
148
143
  assert_nil experiment(:simple).completed_at
@@ -155,13 +150,13 @@ class AbTestTest < ActionController::TestCase
155
150
  # -- A/B helper methods --
156
151
 
157
152
  def test_fail_if_no_experiment
158
- assert_raise MissingSourceFile do
153
+ assert_raise LoadError do
159
154
  get :test_render
160
155
  end
161
156
  end
162
157
 
163
158
  def test_ab_test_chooses_in_render
164
- experiment(:simple) { }
159
+ ab_test(:simple) { }
165
160
  responses = Array.new(100) do
166
161
  @controller = nil ; setup_controller_request_and_response
167
162
  get :test_render
@@ -171,7 +166,7 @@ class AbTestTest < ActionController::TestCase
171
166
  end
172
167
 
173
168
  def test_ab_test_chooses_view_helper
174
- experiment(:simple) { }
169
+ ab_test(:simple) { }
175
170
  responses = Array.new(100) do
176
171
  @controller = nil ; setup_controller_request_and_response
177
172
  get :test_view
@@ -181,7 +176,7 @@ class AbTestTest < ActionController::TestCase
181
176
  end
182
177
 
183
178
  def test_ab_test_with_capture
184
- experiment(:simple) { }
179
+ ab_test(:simple) { }
185
180
  responses = Array.new(100) do
186
181
  @controller = nil ; setup_controller_request_and_response
187
182
  get :test_capture
@@ -190,11 +185,11 @@ class AbTestTest < ActionController::TestCase
190
185
  assert_equal %w{false true}, responses.map(&:strip).uniq.sort
191
186
  end
192
187
 
193
- def test_ab_test_goal
194
- experiment(:simple) { }
188
+ def test_ab_test_track
189
+ ab_test(:simple) { }
195
190
  responses = Array.new(100) do
196
191
  @controller.send(:cookies).clear
197
- get :goal
192
+ get :track
198
193
  @response.body
199
194
  end
200
195
  end
@@ -203,7 +198,7 @@ class AbTestTest < ActionController::TestCase
203
198
  # -- Testing with tests --
204
199
 
205
200
  def test_with_given_choice
206
- experiment(:simple) { alternatives :a, :b, :c }
201
+ ab_test(:simple) { alternatives :a, :b, :c }
207
202
  100.times do |i|
208
203
  @controller = nil ; setup_controller_request_and_response
209
204
  experiment(:simple).chooses(:b)
@@ -213,35 +208,50 @@ class AbTestTest < ActionController::TestCase
213
208
  end
214
209
 
215
210
  def test_which_chooses_non_existent_alternative
216
- experiment(:simple) { }
211
+ ab_test(:simple) { }
217
212
  assert_raises ArgumentError do
218
213
  experiment(:simple).chooses(404)
219
214
  end
220
215
  end
221
216
 
217
+ def test_chooses_cleared_with_nil
218
+ ab_test :simple do
219
+ identify { rand }
220
+ alternatives :a, :b, :c
221
+ end
222
+ responses = Array.new(100) { |i|
223
+ @controller = nil ; setup_controller_request_and_response
224
+ experiment(:simple).chooses(:b)
225
+ experiment(:simple).chooses(nil)
226
+ get :test_render
227
+ @response.body
228
+ }
229
+ assert responses.uniq.size == 3
230
+ end
231
+
222
232
 
223
233
  # -- Scoring --
224
234
 
225
235
  def test_scoring
226
- experiment(:abcd) { alternatives :a, :b, :c, :d }
236
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
227
237
  # participating, conversions, rate, z-score
228
238
  # Control: 182 35 19.23% N/A
229
- 182.times { |i| experiment(:abcd).count i, :a, :participant }
230
- 35.times { |i| experiment(:abcd).count i, :a, :conversion }
239
+ 182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
240
+ 35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
231
241
  # Treatment A: 180 45 25.00% 1.33
232
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
233
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
242
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
243
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
234
244
  # treatment B: 189 28 14.81% -1.13
235
- 189.times { |i| experiment(:abcd).count i, :c, :participant }
236
- 28.times { |i| experiment(:abcd).count i, :c, :conversion }
245
+ 189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
246
+ 28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
237
247
  # treatment C: 188 61 32.45% 2.94
238
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
239
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
248
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
249
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
240
250
 
241
251
  z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
242
252
  assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
243
- confidences = experiment(:abcd).score.alts.map(&:confidence)
244
- assert_equal [90, 0, 99, 90], confidences
253
+ probabilities = experiment(:abcd).score.alts.map(&:probability)
254
+ assert_equal [90, 0, 99, 90], probabilities
245
255
 
246
256
  diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
247
257
  assert_equal [30, 69, nil, 119], diff
@@ -253,9 +263,9 @@ class AbTestTest < ActionController::TestCase
253
263
  end
254
264
 
255
265
  def test_scoring_with_no_performers
256
- experiment(:abcd) { alternatives :a, :b, :c, :d }
266
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
257
267
  assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
258
- assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
268
+ assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
259
269
  assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
260
270
  assert_nil experiment(:abcd).score.best
261
271
  assert_nil experiment(:abcd).score.choice
@@ -263,11 +273,11 @@ class AbTestTest < ActionController::TestCase
263
273
  end
264
274
 
265
275
  def test_scoring_with_one_performer
266
- experiment(:abcd) { alternatives :a, :b, :c, :d }
267
- 10.times { |i| experiment(:abcd).count i, :b, :participant }
268
- 8.times { |i| experiment(:abcd).count i, :b, :conversion }
276
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
277
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
278
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
269
279
  assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
270
- assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
280
+ assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
271
281
  assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
272
282
  assert 1, experiment(:abcd).score.best.id
273
283
  assert_nil experiment(:abcd).score.choice
@@ -276,16 +286,16 @@ class AbTestTest < ActionController::TestCase
276
286
  end
277
287
 
278
288
  def test_scoring_with_some_performers
279
- experiment(:abcd) { alternatives :a, :b, :c, :d }
280
- 10.times { |i| experiment(:abcd).count i, :b, :participant }
281
- 8.times { |i| experiment(:abcd).count i, :b, :conversion }
282
- 12.times { |i| experiment(:abcd).count i, :d, :participant }
283
- 5.times { |i| experiment(:abcd).count i, :d, :conversion }
289
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
290
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
291
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
292
+ 12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
293
+ 5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
284
294
 
285
295
  z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
286
296
  assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
287
- confidences = experiment(:abcd).score.alts.map(&:confidence)
288
- assert_equal [0, 95, 0, 0], confidences
297
+ probabilities = experiment(:abcd).score.alts.map(&:probability)
298
+ assert_equal [0, 95, 0, 0], probabilities
289
299
  diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
290
300
  assert_equal [nil, 92, nil, nil], diff
291
301
  assert_equal 1, experiment(:abcd).score.best.id
@@ -294,26 +304,39 @@ class AbTestTest < ActionController::TestCase
294
304
  assert_equal 3, experiment(:abcd).score.least.id
295
305
  end
296
306
 
307
+ def test_scoring_with_different_probability
308
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
309
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
310
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
311
+ 12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
312
+ 5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
313
+
314
+ assert_equal 1, experiment(:abcd).score(90).choice.id
315
+ assert_equal 1, experiment(:abcd).score(95).choice.id
316
+ assert_nil experiment(:abcd).score(99).choice
317
+ end
318
+
297
319
 
298
320
  # -- Conclusion --
299
321
 
300
322
  def test_conclusion
301
- experiment(:abcd) { alternatives :a, :b, :c, :d }
323
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
302
324
  # participating, conversions, rate, z-score
303
325
  # Control: 182 35 19.23% N/A
304
- 182.times { |i| experiment(:abcd).count i, :a, :participant }
305
- 35.times { |i| experiment(:abcd).count i, :a, :conversion }
326
+ 182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
327
+ 35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
306
328
  # Treatment A: 180 45 25.00% 1.33
307
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
308
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
329
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
330
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
309
331
  # treatment B: 189 28 14.81% -1.13
310
- 189.times { |i| experiment(:abcd).count i, :c, :participant }
311
- 28.times { |i| experiment(:abcd).count i, :c, :conversion }
332
+ 189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
333
+ 28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
312
334
  # treatment C: 188 61 32.45% 2.94
313
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
314
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
335
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
336
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
315
337
 
316
338
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
339
+ There are 739 participants in this experiment.
317
340
  The best choice is option D: it converted at 32.4% (30% better than option B).
318
341
  With 90% probability this result is statistically significant.
319
342
  Option B converted at 25.0%.
@@ -324,15 +347,16 @@ Option D selected as the best alternative.
324
347
  end
325
348
 
326
349
  def test_conclusion_with_some_performers
327
- experiment(:abcd) { alternatives :a, :b, :c, :d }
350
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
328
351
  # Treatment A: 180 45 25.00% 1.33
329
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
330
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
352
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
353
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
331
354
  # treatment C: 188 61 32.45% 2.94
332
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
333
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
355
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
356
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
334
357
 
335
358
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
359
+ There are 368 participants in this experiment.
336
360
  The best choice is option D: it converted at 32.4% (30% better than option B).
337
361
  With 90% probability this result is statistically significant.
338
362
  Option B converted at 25.0%.
@@ -343,15 +367,16 @@ Option D selected as the best alternative.
343
367
  end
344
368
 
345
369
  def test_conclusion_without_clear_winner
346
- experiment(:abcd) { alternatives :a, :b, :c, :d }
370
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
347
371
  # Treatment A: 180 45 25.00% 1.33
348
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
349
- 58.times { |i| experiment(:abcd).count i, :b, :conversion }
372
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
373
+ 58.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
350
374
  # treatment C: 188 61 32.45% 2.94
351
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
352
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
375
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
376
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
353
377
 
354
378
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
379
+ There are 368 participants in this experiment.
355
380
  The best choice is option D: it converted at 32.4% (1% better than option B).
356
381
  This result is not statistically significant, suggest you continue this experiment.
357
382
  Option B converted at 32.2%.
@@ -361,15 +386,16 @@ Option C did not convert.
361
386
  end
362
387
 
363
388
  def test_conclusion_without_close_performers
364
- experiment(:abcd) { alternatives :a, :b, :c, :d }
389
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
365
390
  # Treatment A: 180 45 25.00% 1.33
366
- 186.times { |i| experiment(:abcd).count i, :b, :participant }
367
- 60.times { |i| experiment(:abcd).count i, :b, :conversion }
391
+ 186.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
392
+ 60.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
368
393
  # treatment C: 188 61 32.45% 2.94
369
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
370
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
394
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
395
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
371
396
 
372
397
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
398
+ There are 374 participants in this experiment.
373
399
  The best choice is option D: it converted at 32.4%.
374
400
  This result is not statistically significant, suggest you continue this experiment.
375
401
  Option B converted at 32.3%.
@@ -379,15 +405,16 @@ Option C did not convert.
379
405
  end
380
406
 
381
407
  def test_conclusion_without_equal_performers
382
- experiment(:abcd) { alternatives :a, :b, :c, :d }
408
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
383
409
  # Treatment A: 180 45 25.00% 1.33
384
- 188.times { |i| experiment(:abcd).count i, :b, :participant }
385
- 61.times { |i| experiment(:abcd).count i, :b, :conversion }
410
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
411
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
386
412
  # treatment C: 188 61 32.45% 2.94
387
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
388
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
413
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
414
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
389
415
 
390
416
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
417
+ There are 376 participants in this experiment.
391
418
  Option D converted at 32.4%.
392
419
  Option B converted at 32.4%.
393
420
  Option A did not convert.
@@ -396,24 +423,30 @@ Option C did not convert.
396
423
  end
397
424
 
398
425
  def test_conclusion_with_one_performers
399
- experiment(:abcd) { alternatives :a, :b, :c, :d }
426
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
400
427
  # Treatment A: 180 45 25.00% 1.33
401
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
402
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
428
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
429
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
403
430
 
404
- assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
431
+ assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
432
+ There are 180 participants in this experiment.
433
+ This experiment did not run long enough to find a clear winner.
434
+ TEXT
405
435
  end
406
436
 
407
437
  def test_conclusion_with_no_performers
408
- experiment(:abcd) { alternatives :a, :b, :c, :d }
409
- assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
438
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
439
+ assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
440
+ There are no participants in this experiment yet.
441
+ This experiment did not run long enough to find a clear winner.
442
+ TEXT
410
443
  end
411
444
 
412
445
 
413
446
  # -- Completion --
414
447
 
415
448
  def test_completion_if
416
- experiment :simple do
449
+ ab_test :simple do
417
450
  identify { rand }
418
451
  complete_if { true }
419
452
  end
@@ -422,7 +455,7 @@ Option C did not convert.
422
455
  end
423
456
 
424
457
  def test_completion_if_fails
425
- experiment :simple do
458
+ ab_test :simple do
426
459
  identify { rand }
427
460
  complete_if { fail }
428
461
  end
@@ -432,7 +465,7 @@ Option C did not convert.
432
465
 
433
466
  def test_completion
434
467
  ids = Array.new(100) { |i| i.to_s }.shuffle
435
- experiment :simple do
468
+ ab_test :simple do
436
469
  identify { ids.pop }
437
470
  complete_if { alternatives.map(&:participants).sum >= 100 }
438
471
  end
@@ -447,7 +480,7 @@ Option C did not convert.
447
480
 
448
481
  def test_ab_methods_after_completion
449
482
  ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
450
- experiment :simple do
483
+ ab_test :simple do
451
484
  identify { ids.pop }
452
485
  complete_if { alternatives.map(&:participants).sum >= 100 }
453
486
  outcome_is { alternatives[1] }
@@ -456,7 +489,7 @@ Option C did not convert.
456
489
  results = Set.new
457
490
  100.times do
458
491
  results << experiment(:simple).choose
459
- experiment(:simple).conversion!
492
+ experiment(:simple).track!
460
493
  end
461
494
  assert results.include?(true) && results.include?(false)
462
495
  refute experiment(:simple).active?
@@ -464,7 +497,7 @@ Option C did not convert.
464
497
  # Test that we always get the same choice (true)
465
498
  100.times do
466
499
  assert_equal true, experiment(:simple).choose
467
- experiment(:simple).conversion!
500
+ experiment(:simple).track!
468
501
  end
469
502
  # We don't get to count the 100 participant's conversion, but that's ok.
470
503
  assert_equal 99, experiment(:simple).alternatives.map(&:converted).sum
@@ -475,7 +508,7 @@ Option C did not convert.
475
508
  # -- Outcome --
476
509
 
477
510
  def test_completion_outcome
478
- experiment :quick do
511
+ ab_test :quick do
479
512
  outcome_is { alternatives[1] }
480
513
  end
481
514
  experiment(:quick).complete!
@@ -483,7 +516,7 @@ Option C did not convert.
483
516
  end
484
517
 
485
518
  def test_outcome_is_returns_nil
486
- experiment :quick do
519
+ ab_test :quick do
487
520
  outcome_is { nil }
488
521
  end
489
522
  experiment(:quick).complete!
@@ -491,7 +524,7 @@ Option C did not convert.
491
524
  end
492
525
 
493
526
  def test_outcome_is_returns_something_else
494
- experiment :quick do
527
+ ab_test :quick do
495
528
  outcome_is { "error" }
496
529
  end
497
530
  experiment(:quick).complete!
@@ -499,7 +532,7 @@ Option C did not convert.
499
532
  end
500
533
 
501
534
  def test_outcome_is_fails
502
- experiment :quick do
535
+ ab_test :quick do
503
536
  outcome_is { fail }
504
537
  end
505
538
  experiment(:quick).complete!
@@ -507,29 +540,33 @@ Option C did not convert.
507
540
  end
508
541
 
509
542
  def test_outcome_choosing_best_alternative
510
- experiment :quick do
543
+ ab_test :quick do
511
544
  end
512
- 2.times { |i| experiment(:quick).count i, false, :participant }
513
- 10.times { |i| experiment(:quick).count i, true }
545
+ 2.times { |i| experiment(:quick).send(:count_participant, i, false) }
546
+ 10.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
514
547
  experiment(:quick).complete!
515
548
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
516
549
  end
517
550
 
518
551
  def test_outcome_only_performing_alternative
519
- experiment :quick do
552
+ ab_test :quick do
520
553
  end
521
- 2.times { |i| experiment(:quick).count i, true }
554
+ 2.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
522
555
  experiment(:quick).complete!
523
556
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
524
557
  end
525
558
 
526
559
  def test_outcome_choosing_equal_alternatives
527
- experiment :quick do
560
+ ab_test :quick do
528
561
  end
529
- 8.times { |i| experiment(:quick).count i, false }
530
- 8.times { |i| experiment(:quick).count i, true }
562
+ 8.times { |i| experiment(:quick).send(:count_participant, i, false).send(:count_conversion, i, false) }
563
+ 8.times { |i| experiment(:quick).send(:count_participant, i, true). send(:count_conversion, i, true) }
531
564
  experiment(:quick).complete!
532
565
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
533
566
  end
534
567
 
568
+
569
+ def ab_test(name, &block)
570
+ Vanity.playground.define name, :ab_test, &block
571
+ end
535
572
  end