vanity 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. data/CHANGELOG +34 -19
  2. data/README.rdoc +19 -15
  3. data/lib/vanity/commands/report.rb +11 -4
  4. data/lib/vanity/experiment/ab_test.rb +149 -108
  5. data/lib/vanity/experiment/base.rb +42 -53
  6. data/lib/vanity/playground.rb +48 -24
  7. data/lib/vanity/rails.rb +2 -1
  8. data/lib/vanity/rails/dashboard.rb +15 -0
  9. data/lib/vanity/rails/helpers.rb +19 -32
  10. data/lib/vanity/rails/testing.rb +3 -1
  11. data/lib/vanity/templates/_ab_test.erb +7 -5
  12. data/lib/vanity/templates/_experiment.erb +5 -0
  13. data/lib/vanity/templates/_experiments.erb +2 -7
  14. data/lib/vanity/templates/_report.erb +14 -14
  15. data/lib/vanity/templates/vanity.css +13 -0
  16. data/test/ab_test_test.rb +147 -110
  17. data/test/experiment_test.rb +15 -22
  18. data/test/experiments/age_and_zipcode.rb +17 -2
  19. data/test/experiments/null_abc.rb +1 -1
  20. data/test/playground_test.rb +53 -31
  21. data/test/rails_test.rb +1 -1
  22. data/test/test_helper.rb +2 -0
  23. data/vanity.gemspec +2 -2
  24. metadata +7 -6
  25. data/lib/vanity/rails/console.rb +0 -14
  26. data/lib/vanity/templates/_vanity.css +0 -13
data/lib/vanity/templates/_experiments.erb CHANGED
@@ -1,12 +1,7 @@
1
1
  <ul class="experiments">
2
2
  <% experiments.sort_by(&:created_at).each do |experiment| %>
3
- <li class="experiment" id="experiment_<%= CGI.escape experiment.id.to_s %>">
4
- <h2 class="<%= experiment.type %>"><%= CGI.escape_html experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
5
- <p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
6
- <%= render Vanity.template("ab_test"), experiment: experiment %>
7
- <p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
8
- <%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
9
- </p>
3
+ <li class="experiment <%= experiment.type %>" id="experiment_<%=h experiment.id.to_s %>">
4
+ <%= render Vanity.template("experiment"), experiment: experiment %>
10
5
  </li>
11
6
  <% end %>
12
7
  </ul>
data/lib/vanity/templates/_report.erb CHANGED
@@ -1,16 +1,16 @@
1
1
  <html>
2
- <head>
3
- <title>Experiments (<%= Time.now %>)</title>
4
- <style>
5
- .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
6
- .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
7
- </style>
8
- <%= render Vanity.template("vanity.css") %>
9
- </head>
10
- <body>
11
- <div class="vanity">
12
- <h1>Experiments</h1>
13
- <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
14
- </div>
15
- </body>
2
+ <head>
3
+ <title>Experiments (<%= Time.now %>)</title>
4
+ <style>
5
+ .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
6
+ .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
7
+ <%= File.read(Vanity.template("vanity.css")) %>
8
+ </style>
9
+ </head>
10
+ <body>
11
+ <div class="vanity">
12
+ <h1>Experiments</h1>
13
+ <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
14
+ </div>
15
+ </body>
16
16
  </html>
data/lib/vanity/templates/vanity.css ADDED
@@ -0,0 +1,13 @@
1
+ .vanity .experiments { list-style: none; margin: 0; padding: 0 }
2
+ .vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
3
+ .vanity .experiment .description { padding: .5em; margin: 0 }
4
+ .vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
5
+ .vanity .experiment .meta { color: #444; font-style: italic }
6
+
7
+ .vanity .ab_test table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
8
+ .vanity .ab_test td { padding: .5em; border-top: 1px solid #ccc }
9
+ .vanity .ab_test .choice td { font-weight: bold; background: #f0f0f8 }
10
+ .vanity .ab_test caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
11
+ .vanity .ab_test td.option { width: 5em; white-space: nowrap; overflow: hidden }
12
+ .vanity .ab_test td.value { width: 8em; white-space: nowrap; overflow: hidden }
13
+ .vanity .ab_test td.action { width: 6em; overflow: hidden; text-align: center }
data/test/ab_test_test.rb CHANGED
@@ -16,8 +16,8 @@ class AbTestController < ActionController::Base
16
16
  render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
17
17
  end
18
18
 
19
- def goal
20
- ab_goal! :simple
19
+ def track
20
+ track! :simple
21
21
  render text: ""
22
22
  end
23
23
  end
@@ -28,29 +28,24 @@ class AbTestTest < ActionController::TestCase
28
28
 
29
29
  # -- Experiment definition --
30
30
 
31
- def test_uses_ab_test_when_type_is_ab_test
32
- experiment(:ab, type: :ab_test) { }
33
- assert_instance_of Vanity::Experiment::AbTest, experiment(:ab)
34
- end
35
-
36
31
  def test_requires_at_least_two_alternatives_per_experiment
37
32
  assert_raises RuntimeError do
38
- experiment :none, type: :ab_test do
33
+ ab_test :none do
39
34
  alternatives []
40
35
  end
41
36
  end
42
37
  assert_raises RuntimeError do
43
- experiment :one, type: :ab_test do
38
+ ab_test :one do
44
39
  alternatives "foo"
45
40
  end
46
41
  end
47
- experiment :two, type: :ab_test do
42
+ ab_test :two do
48
43
  alternatives "foo", "bar"
49
44
  end
50
45
  end
51
46
 
52
47
  def test_returning_alternative_by_value
53
- experiment :abcd do
48
+ ab_test :abcd do
54
49
  alternatives :a, :b, :c, :d
55
50
  end
56
51
  assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
@@ -58,7 +53,7 @@ class AbTestTest < ActionController::TestCase
58
53
  end
59
54
 
60
55
  def test_alternative_name
61
- experiment :abcd do
56
+ ab_test :abcd do
62
57
  alternatives :a, :b
63
58
  end
64
59
  assert_equal "option A", experiment(:abcd).alternative(:a).name
@@ -69,7 +64,7 @@ class AbTestTest < ActionController::TestCase
69
64
  # -- Running experiment --
70
65
 
71
66
  def test_returns_the_same_alternative_consistently
72
- experiment :foobar do
67
+ ab_test :foobar do
73
68
  alternatives "foo", "bar"
74
69
  identify { "6e98ec" }
75
70
  end
@@ -81,7 +76,7 @@ class AbTestTest < ActionController::TestCase
81
76
  end
82
77
 
83
78
  def test_returns_different_alternatives_for_each_participant
84
- experiment :foobar do
79
+ ab_test :foobar do
85
80
  alternatives "foo", "bar"
86
81
  identify { rand }
87
82
  end
@@ -92,7 +87,7 @@ class AbTestTest < ActionController::TestCase
92
87
 
93
88
  def test_records_all_participants_in_each_alternative
94
89
  ids = (Array.new(200) { |i| i } * 5).shuffle
95
- experiment :foobar do
90
+ ab_test :foobar do
96
91
  alternatives "foo", "bar"
97
92
  identify { ids.pop }
98
93
  end
@@ -104,13 +99,13 @@ class AbTestTest < ActionController::TestCase
104
99
 
105
100
  def test_records_each_converted_participant_only_once
106
101
  ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
107
- experiment :foobar do
102
+ ab_test :foobar do
108
103
  alternatives "foo", "bar"
109
104
  identify { ids.pop }
110
105
  end
111
106
  500.times do
112
107
  experiment(:foobar).choose
113
- experiment(:foobar).conversion!
108
+ experiment(:foobar).track!
114
109
  end
115
110
  alts = experiment(:foobar).alternatives
116
111
  assert_equal 100, alts.map(&:converted).sum
@@ -118,31 +113,31 @@ class AbTestTest < ActionController::TestCase
118
113
 
119
114
  def test_records_conversion_only_for_participants
120
115
  ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
121
- experiment :foobar do
116
+ ab_test :foobar do
122
117
  alternatives "foo", "bar"
123
118
  identify { ids.pop }
124
119
  end
125
120
  500.times do
126
121
  experiment(:foobar).choose
127
- experiment(:foobar).conversion!
128
- experiment(:foobar).conversion!
122
+ experiment(:foobar).track!
123
+ experiment(:foobar).track!
129
124
  end
130
125
  alts = experiment(:foobar).alternatives
131
126
  assert_equal 100, alts.map(&:converted).sum
132
127
  end
133
128
 
134
- def test_reset_experiment
135
- experiment :simple do
129
+ def test_destroy_experiment
130
+ ab_test :simple do
136
131
  identify { "me" }
137
132
  complete_if { alternatives.map(&:converted).sum >= 1 }
138
133
  outcome_is { alternative(true) }
139
134
  end
140
135
  experiment(:simple).choose
141
- experiment(:simple).conversion!
136
+ experiment(:simple).track!
142
137
  refute experiment(:simple).active?
143
138
  assert_equal true, experiment(:simple).outcome.value
144
139
 
145
- experiment(:simple).reset!
140
+ experiment(:simple).destroy
146
141
  assert experiment(:simple).active?
147
142
  assert_nil experiment(:simple).outcome
148
143
  assert_nil experiment(:simple).completed_at
@@ -155,13 +150,13 @@ class AbTestTest < ActionController::TestCase
155
150
  # -- A/B helper methods --
156
151
 
157
152
  def test_fail_if_no_experiment
158
- assert_raise MissingSourceFile do
153
+ assert_raise LoadError do
159
154
  get :test_render
160
155
  end
161
156
  end
162
157
 
163
158
  def test_ab_test_chooses_in_render
164
- experiment(:simple) { }
159
+ ab_test(:simple) { }
165
160
  responses = Array.new(100) do
166
161
  @controller = nil ; setup_controller_request_and_response
167
162
  get :test_render
@@ -171,7 +166,7 @@ class AbTestTest < ActionController::TestCase
171
166
  end
172
167
 
173
168
  def test_ab_test_chooses_view_helper
174
- experiment(:simple) { }
169
+ ab_test(:simple) { }
175
170
  responses = Array.new(100) do
176
171
  @controller = nil ; setup_controller_request_and_response
177
172
  get :test_view
@@ -181,7 +176,7 @@ class AbTestTest < ActionController::TestCase
181
176
  end
182
177
 
183
178
  def test_ab_test_with_capture
184
- experiment(:simple) { }
179
+ ab_test(:simple) { }
185
180
  responses = Array.new(100) do
186
181
  @controller = nil ; setup_controller_request_and_response
187
182
  get :test_capture
@@ -190,11 +185,11 @@ class AbTestTest < ActionController::TestCase
190
185
  assert_equal %w{false true}, responses.map(&:strip).uniq.sort
191
186
  end
192
187
 
193
- def test_ab_test_goal
194
- experiment(:simple) { }
188
+ def test_ab_test_track
189
+ ab_test(:simple) { }
195
190
  responses = Array.new(100) do
196
191
  @controller.send(:cookies).clear
197
- get :goal
192
+ get :track
198
193
  @response.body
199
194
  end
200
195
  end
@@ -203,7 +198,7 @@ class AbTestTest < ActionController::TestCase
203
198
  # -- Testing with tests --
204
199
 
205
200
  def test_with_given_choice
206
- experiment(:simple) { alternatives :a, :b, :c }
201
+ ab_test(:simple) { alternatives :a, :b, :c }
207
202
  100.times do |i|
208
203
  @controller = nil ; setup_controller_request_and_response
209
204
  experiment(:simple).chooses(:b)
@@ -213,35 +208,50 @@ class AbTestTest < ActionController::TestCase
213
208
  end
214
209
 
215
210
  def test_which_chooses_non_existent_alternative
216
- experiment(:simple) { }
211
+ ab_test(:simple) { }
217
212
  assert_raises ArgumentError do
218
213
  experiment(:simple).chooses(404)
219
214
  end
220
215
  end
221
216
 
217
+ def test_chooses_cleared_with_nil
218
+ ab_test :simple do
219
+ identify { rand }
220
+ alternatives :a, :b, :c
221
+ end
222
+ responses = Array.new(100) { |i|
223
+ @controller = nil ; setup_controller_request_and_response
224
+ experiment(:simple).chooses(:b)
225
+ experiment(:simple).chooses(nil)
226
+ get :test_render
227
+ @response.body
228
+ }
229
+ assert responses.uniq.size == 3
230
+ end
231
+
222
232
 
223
233
  # -- Scoring --
224
234
 
225
235
  def test_scoring
226
- experiment(:abcd) { alternatives :a, :b, :c, :d }
236
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
227
237
  # participating, conversions, rate, z-score
228
238
  # Control: 182 35 19.23% N/A
229
- 182.times { |i| experiment(:abcd).count i, :a, :participant }
230
- 35.times { |i| experiment(:abcd).count i, :a, :conversion }
239
+ 182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
240
+ 35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
231
241
  # Treatment A: 180 45 25.00% 1.33
232
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
233
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
242
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
243
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
234
244
  # treatment B: 189 28 14.81% -1.13
235
- 189.times { |i| experiment(:abcd).count i, :c, :participant }
236
- 28.times { |i| experiment(:abcd).count i, :c, :conversion }
245
+ 189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
246
+ 28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
237
247
  # treatment C: 188 61 32.45% 2.94
238
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
239
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
248
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
249
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
240
250
 
241
251
  z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
242
252
  assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
243
- confidences = experiment(:abcd).score.alts.map(&:confidence)
244
- assert_equal [90, 0, 99, 90], confidences
253
+ probabilities = experiment(:abcd).score.alts.map(&:probability)
254
+ assert_equal [90, 0, 99, 90], probabilities
245
255
 
246
256
  diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
247
257
  assert_equal [30, 69, nil, 119], diff
@@ -253,9 +263,9 @@ class AbTestTest < ActionController::TestCase
253
263
  end
254
264
 
255
265
  def test_scoring_with_no_performers
256
- experiment(:abcd) { alternatives :a, :b, :c, :d }
266
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
257
267
  assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
258
- assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
268
+ assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
259
269
  assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
260
270
  assert_nil experiment(:abcd).score.best
261
271
  assert_nil experiment(:abcd).score.choice
@@ -263,11 +273,11 @@ class AbTestTest < ActionController::TestCase
263
273
  end
264
274
 
265
275
  def test_scoring_with_one_performer
266
- experiment(:abcd) { alternatives :a, :b, :c, :d }
267
- 10.times { |i| experiment(:abcd).count i, :b, :participant }
268
- 8.times { |i| experiment(:abcd).count i, :b, :conversion }
276
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
277
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
278
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
269
279
  assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
270
- assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
280
+ assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
271
281
  assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
272
282
  assert 1, experiment(:abcd).score.best.id
273
283
  assert_nil experiment(:abcd).score.choice
@@ -276,16 +286,16 @@ class AbTestTest < ActionController::TestCase
276
286
  end
277
287
 
278
288
  def test_scoring_with_some_performers
279
- experiment(:abcd) { alternatives :a, :b, :c, :d }
280
- 10.times { |i| experiment(:abcd).count i, :b, :participant }
281
- 8.times { |i| experiment(:abcd).count i, :b, :conversion }
282
- 12.times { |i| experiment(:abcd).count i, :d, :participant }
283
- 5.times { |i| experiment(:abcd).count i, :d, :conversion }
289
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
290
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
291
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
292
+ 12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
293
+ 5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
284
294
 
285
295
  z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
286
296
  assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
287
- confidences = experiment(:abcd).score.alts.map(&:confidence)
288
- assert_equal [0, 95, 0, 0], confidences
297
+ probabilities = experiment(:abcd).score.alts.map(&:probability)
298
+ assert_equal [0, 95, 0, 0], probabilities
289
299
  diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
290
300
  assert_equal [nil, 92, nil, nil], diff
291
301
  assert_equal 1, experiment(:abcd).score.best.id
@@ -294,26 +304,39 @@ class AbTestTest < ActionController::TestCase
294
304
  assert_equal 3, experiment(:abcd).score.least.id
295
305
  end
296
306
 
307
+ def test_scoring_with_different_probability
308
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
309
+ 10.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
310
+ 8.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
311
+ 12.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
312
+ 5.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
313
+
314
+ assert_equal 1, experiment(:abcd).score(90).choice.id
315
+ assert_equal 1, experiment(:abcd).score(95).choice.id
316
+ assert_nil experiment(:abcd).score(99).choice
317
+ end
318
+
297
319
 
298
320
  # -- Conclusion --
299
321
 
300
322
  def test_conclusion
301
- experiment(:abcd) { alternatives :a, :b, :c, :d }
323
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
302
324
  # participating, conversions, rate, z-score
303
325
  # Control: 182 35 19.23% N/A
304
- 182.times { |i| experiment(:abcd).count i, :a, :participant }
305
- 35.times { |i| experiment(:abcd).count i, :a, :conversion }
326
+ 182.times { |i| experiment(:abcd).send(:count_participant, i, :a) }
327
+ 35.times { |i| experiment(:abcd).send(:count_conversion, i, :a) }
306
328
  # Treatment A: 180 45 25.00% 1.33
307
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
308
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
329
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
330
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
309
331
  # treatment B: 189 28 14.81% -1.13
310
- 189.times { |i| experiment(:abcd).count i, :c, :participant }
311
- 28.times { |i| experiment(:abcd).count i, :c, :conversion }
332
+ 189.times { |i| experiment(:abcd).send(:count_participant, i, :c) }
333
+ 28.times { |i| experiment(:abcd).send(:count_conversion, i, :c) }
312
334
  # treatment C: 188 61 32.45% 2.94
313
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
314
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
335
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
336
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
315
337
 
316
338
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
339
+ There are 739 participants in this experiment.
317
340
  The best choice is option D: it converted at 32.4% (30% better than option B).
318
341
  With 90% probability this result is statistically significant.
319
342
  Option B converted at 25.0%.
@@ -324,15 +347,16 @@ Option D selected as the best alternative.
324
347
  end
325
348
 
326
349
  def test_conclusion_with_some_performers
327
- experiment(:abcd) { alternatives :a, :b, :c, :d }
350
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
328
351
  # Treatment A: 180 45 25.00% 1.33
329
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
330
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
352
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
353
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
331
354
  # treatment C: 188 61 32.45% 2.94
332
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
333
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
355
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
356
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
334
357
 
335
358
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
359
+ There are 368 participants in this experiment.
336
360
  The best choice is option D: it converted at 32.4% (30% better than option B).
337
361
  With 90% probability this result is statistically significant.
338
362
  Option B converted at 25.0%.
@@ -343,15 +367,16 @@ Option D selected as the best alternative.
343
367
  end
344
368
 
345
369
  def test_conclusion_without_clear_winner
346
- experiment(:abcd) { alternatives :a, :b, :c, :d }
370
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
347
371
  # Treatment A: 180 45 25.00% 1.33
348
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
349
- 58.times { |i| experiment(:abcd).count i, :b, :conversion }
372
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
373
+ 58.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
350
374
  # treatment C: 188 61 32.45% 2.94
351
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
352
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
375
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
376
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
353
377
 
354
378
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
379
+ There are 368 participants in this experiment.
355
380
  The best choice is option D: it converted at 32.4% (1% better than option B).
356
381
  This result is not statistically significant, suggest you continue this experiment.
357
382
  Option B converted at 32.2%.
@@ -361,15 +386,16 @@ Option C did not convert.
361
386
  end
362
387
 
363
388
  def test_conclusion_without_close_performers
364
- experiment(:abcd) { alternatives :a, :b, :c, :d }
389
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
365
390
  # Treatment A: 180 45 25.00% 1.33
366
- 186.times { |i| experiment(:abcd).count i, :b, :participant }
367
- 60.times { |i| experiment(:abcd).count i, :b, :conversion }
391
+ 186.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
392
+ 60.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
368
393
  # treatment C: 188 61 32.45% 2.94
369
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
370
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
394
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
395
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
371
396
 
372
397
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
398
+ There are 374 participants in this experiment.
373
399
  The best choice is option D: it converted at 32.4%.
374
400
  This result is not statistically significant, suggest you continue this experiment.
375
401
  Option B converted at 32.3%.
@@ -379,15 +405,16 @@ Option C did not convert.
379
405
  end
380
406
 
381
407
  def test_conclusion_without_equal_performers
382
- experiment(:abcd) { alternatives :a, :b, :c, :d }
408
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
383
409
  # Treatment A: 180 45 25.00% 1.33
384
- 188.times { |i| experiment(:abcd).count i, :b, :participant }
385
- 61.times { |i| experiment(:abcd).count i, :b, :conversion }
410
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
411
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
386
412
  # treatment C: 188 61 32.45% 2.94
387
- 188.times { |i| experiment(:abcd).count i, :d, :participant }
388
- 61.times { |i| experiment(:abcd).count i, :d, :conversion }
413
+ 188.times { |i| experiment(:abcd).send(:count_participant, i, :d) }
414
+ 61.times { |i| experiment(:abcd).send(:count_conversion, i, :d) }
389
415
 
390
416
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
417
+ There are 376 participants in this experiment.
391
418
  Option D converted at 32.4%.
392
419
  Option B converted at 32.4%.
393
420
  Option A did not convert.
@@ -396,24 +423,30 @@ Option C did not convert.
396
423
  end
397
424
 
398
425
  def test_conclusion_with_one_performers
399
- experiment(:abcd) { alternatives :a, :b, :c, :d }
426
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
400
427
  # Treatment A: 180 45 25.00% 1.33
401
- 180.times { |i| experiment(:abcd).count i, :b, :participant }
402
- 45.times { |i| experiment(:abcd).count i, :b, :conversion }
428
+ 180.times { |i| experiment(:abcd).send(:count_participant, i, :b) }
429
+ 45.times { |i| experiment(:abcd).send(:count_conversion, i, :b) }
403
430
 
404
- assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
431
+ assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
432
+ There are 180 participants in this experiment.
433
+ This experiment did not run long enough to find a clear winner.
434
+ TEXT
405
435
  end
406
436
 
407
437
  def test_conclusion_with_no_performers
408
- experiment(:abcd) { alternatives :a, :b, :c, :d }
409
- assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
438
+ ab_test(:abcd) { alternatives :a, :b, :c, :d }
439
+ assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
440
+ There are no participants in this experiment yet.
441
+ This experiment did not run long enough to find a clear winner.
442
+ TEXT
410
443
  end
411
444
 
412
445
 
413
446
  # -- Completion --
414
447
 
415
448
  def test_completion_if
416
- experiment :simple do
449
+ ab_test :simple do
417
450
  identify { rand }
418
451
  complete_if { true }
419
452
  end
@@ -422,7 +455,7 @@ Option C did not convert.
422
455
  end
423
456
 
424
457
  def test_completion_if_fails
425
- experiment :simple do
458
+ ab_test :simple do
426
459
  identify { rand }
427
460
  complete_if { fail }
428
461
  end
@@ -432,7 +465,7 @@ Option C did not convert.
432
465
 
433
466
  def test_completion
434
467
  ids = Array.new(100) { |i| i.to_s }.shuffle
435
- experiment :simple do
468
+ ab_test :simple do
436
469
  identify { ids.pop }
437
470
  complete_if { alternatives.map(&:participants).sum >= 100 }
438
471
  end
@@ -447,7 +480,7 @@ Option C did not convert.
447
480
 
448
481
  def test_ab_methods_after_completion
449
482
  ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
450
- experiment :simple do
483
+ ab_test :simple do
451
484
  identify { ids.pop }
452
485
  complete_if { alternatives.map(&:participants).sum >= 100 }
453
486
  outcome_is { alternatives[1] }
@@ -456,7 +489,7 @@ Option C did not convert.
456
489
  results = Set.new
457
490
  100.times do
458
491
  results << experiment(:simple).choose
459
- experiment(:simple).conversion!
492
+ experiment(:simple).track!
460
493
  end
461
494
  assert results.include?(true) && results.include?(false)
462
495
  refute experiment(:simple).active?
@@ -464,7 +497,7 @@ Option C did not convert.
464
497
  # Test that we always get the same choice (true)
465
498
  100.times do
466
499
  assert_equal true, experiment(:simple).choose
467
- experiment(:simple).conversion!
500
+ experiment(:simple).track!
468
501
  end
469
502
  # We don't get to count the 100 participant's conversion, but that's ok.
470
503
  assert_equal 99, experiment(:simple).alternatives.map(&:converted).sum
@@ -475,7 +508,7 @@ Option C did not convert.
475
508
  # -- Outcome --
476
509
 
477
510
  def test_completion_outcome
478
- experiment :quick do
511
+ ab_test :quick do
479
512
  outcome_is { alternatives[1] }
480
513
  end
481
514
  experiment(:quick).complete!
@@ -483,7 +516,7 @@ Option C did not convert.
483
516
  end
484
517
 
485
518
  def test_outcome_is_returns_nil
486
- experiment :quick do
519
+ ab_test :quick do
487
520
  outcome_is { nil }
488
521
  end
489
522
  experiment(:quick).complete!
@@ -491,7 +524,7 @@ Option C did not convert.
491
524
  end
492
525
 
493
526
  def test_outcome_is_returns_something_else
494
- experiment :quick do
527
+ ab_test :quick do
495
528
  outcome_is { "error" }
496
529
  end
497
530
  experiment(:quick).complete!
@@ -499,7 +532,7 @@ Option C did not convert.
499
532
  end
500
533
 
501
534
  def test_outcome_is_fails
502
- experiment :quick do
535
+ ab_test :quick do
503
536
  outcome_is { fail }
504
537
  end
505
538
  experiment(:quick).complete!
@@ -507,29 +540,33 @@ Option C did not convert.
507
540
  end
508
541
 
509
542
  def test_outcome_choosing_best_alternative
510
- experiment :quick do
543
+ ab_test :quick do
511
544
  end
512
- 2.times { |i| experiment(:quick).count i, false, :participant }
513
- 10.times { |i| experiment(:quick).count i, true }
545
+ 2.times { |i| experiment(:quick).send(:count_participant, i, false) }
546
+ 10.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
514
547
  experiment(:quick).complete!
515
548
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
516
549
  end
517
550
 
518
551
  def test_outcome_only_performing_alternative
519
- experiment :quick do
552
+ ab_test :quick do
520
553
  end
521
- 2.times { |i| experiment(:quick).count i, true }
554
+ 2.times { |i| experiment(:quick).send(:count_participant, i, true).send(:count_conversion, i, true) }
522
555
  experiment(:quick).complete!
523
556
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
524
557
  end
525
558
 
526
559
  def test_outcome_choosing_equal_alternatives
527
- experiment :quick do
560
+ ab_test :quick do
528
561
  end
529
- 8.times { |i| experiment(:quick).count i, false }
530
- 8.times { |i| experiment(:quick).count i, true }
562
+ 8.times { |i| experiment(:quick).send(:count_participant, i, false).send(:count_conversion, i, false) }
563
+ 8.times { |i| experiment(:quick).send(:count_participant, i, true). send(:count_conversion, i, true) }
531
564
  experiment(:quick).complete!
532
565
  assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
533
566
  end
534
567
 
568
+
569
+ def ab_test(name, &block)
570
+ Vanity.playground.define name, :ab_test, &block
571
+ end
535
572
  end